?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob3d66f8fd117dbce383a71cad2e06f0e48f834871
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 6/20/2024 Ver. 0.13.10 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
38
39 try:
40 basestring;
41 except NameError:
42 basestring = str;
43
44 # URL Parsing
45 try:
46 from urllib.parse import urlparse, urlunparse;
47 except ImportError:
48 from urlparse import urlparse, urlunparse;
49
50 # Windows-specific setup
51 if os.name == 'nt':
52 if sys.version_info[0] == 2:
53 import codecs;
54 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
55 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
56 else:
57 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
58 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
59
60 hashlib_guaranteed = False;
61 # Environment setup
62 os.environ["PYTHONIOENCODING"] = "UTF-8";
63 os.environ["LC_CTYPE"] = "UTF-8";
64
65 # Reload sys to set default encoding to UTF-8 (Python 2 only)
66 if sys.version_info[0] == 2:
67 try:
68 reload(sys);
69 sys.setdefaultencoding('UTF-8');
70 except (NameError, AttributeError):
71 pass;
72
73 # CRC32 import
74 try:
75 from zlib import crc32;
76 except ImportError:
77 from binascii import crc32;
78
79 # Define FileNotFoundError for Python 2
80 try:
81 FileNotFoundError;
82 except NameError:
83 FileNotFoundError = IOError;
84
85 # RAR file support
86 rarfile_support = False;
87 try:
88 import rarfile;
89 rarfile_support = True;
90 except ImportError:
91 pass;
92
93 # 7z file support
94 py7zr_support = False;
95 try:
96 import py7zr;
97 py7zr_support = True;
98 except ImportError:
99 pass;
100
101 # TAR file checking
102 try:
103 from xtarfile import is_tarfile;
104 except ImportError:
105 try:
106 from safetar import is_tarfile;
107 except ImportError:
108 from tarfile import is_tarfile;
109
110 # TAR file module
111 try:
112 import xtarfile as tarfile;
113 except ImportError:
114 try:
115 import safetar as tarfile;
116 except ImportError:
117 import tarfile;
118
119 # Paramiko support
120 haveparamiko = False;
121 try:
122 import paramiko;
123 haveparamiko = True;
124 except ImportError:
125 pass;
126
127 # PySFTP support
128 havepysftp = False;
129 try:
130 import pysftp;
131 havepysftp = True;
132 except ImportError:
133 pass;
134
135 # Requests support
136 haverequests = False;
137 try:
138 import requests;
139 haverequests = True;
140 import urllib3;
141 logging.getLogger("urllib3").setLevel(logging.WARNING);
142 except ImportError:
143 pass;
144
145 # HTTPX support
146 havehttpx = False;
147 try:
148 import httpx;
149 havehttpx = True;
150 logging.getLogger("httpx").setLevel(logging.WARNING);
151 logging.getLogger("httpcore").setLevel(logging.WARNING);
152 except ImportError:
153 pass;
154
155 # HTTP and URL parsing
156 try:
157 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
158 from urllib.parse import urlparse;
159 except ImportError:
160 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
161 from urlparse import urlparse;
162
163 # StringIO and BytesIO
164 try:
165 from io import StringIO, BytesIO;
166 except ImportError:
167 try:
168 from cStringIO import StringIO;
169 from cStringIO import StringIO as BytesIO;
170 except ImportError:
171 from StringIO import StringIO;
172 from StringIO import StringIO as BytesIO;
173
174 __use_pysftp__ = False;
175 __use_alt_format__ = False;
176 if(not havepysftp):
177 __use_pysftp__ = False;
178 __use_http_lib__ = "httpx";
179 if(__use_http_lib__=="httpx" and haverequests and not havehttpx):
180 __use_http_lib__ = "requests";
181 if(__use_http_lib__=="requests" and havehttpx and not haverequests):
182 __use_http_lib__ = "httpx";
183 if((__use_http_lib__=="httpx" or __use_http_lib__=="requests") and not havehttpx and not haverequests):
184 __use_http_lib__ = "urllib";
185 if(not __use_alt_format__):
186 ''' Format Info by Kazuki Przyborowski '''
187 __file_format_name__ = "CatFile";
188 __program_name__ = "Py"+__file_format_name__;
189 __file_format_lower__ = __file_format_name__.lower();
190 __file_format_magic__ = __file_format_name__;
191 __file_format_len__ = len(__file_format_magic__);
192 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
193 __file_format_delimiter__ = "\x00";
194 __file_format_ver__ = "001";
195 __use_new_style__ = True;
196 __use_advanced_list__ = True;
197 __use_alt_inode__ = False;
198 __file_format_extension__ = ".cat";
199 else:
200 ''' Format Info Generated by ChatGPT '''
201 __file_format_name__ = "FastArchive";
202 __program_name__ = "Py" + __file_format_name__;
203 __file_format_lower__ = __file_format_name__.lower();
204 __file_format_magic__ = "FstArch";
205 __file_format_len__ = len(__file_format_magic__);
206 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
207 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
208 __file_format_ver__ = "001";
209 __use_new_style__ = True;
210 __use_advanced_list__ = False;
211 __use_alt_inode__ = False;
212 __file_format_extension__ = ".fast";
213 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
214 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
215 __project__ = __program_name__;
216 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
217 __version_info__ = (0, 13, 10, "RC 1", 1);
218 __version_date_info__ = (2024, 6, 20, "RC 1", 1);
219 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
220 __revision__ = __version_info__[3];
221 __revision_id__ = "$Id$";
222 if(__version_info__[4] is not None):
223 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
224 if(__version_info__[4] is None):
225 __version_date_plusrc__ = __version_date__;
226 if(__version_info__[3] is not None):
227 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
228 if(__version_info__[3] is None):
229 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
230
231 PyBitness = platform.architecture();
232 if(PyBitness=="32bit" or PyBitness=="32"):
233 PyBitness = "32";
234 elif(PyBitness=="64bit" or PyBitness=="64"):
235 PyBitness = "64";
236 else:
237 PyBitness = "32";
238
239 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
240 if(platform.python_implementation()!=""):
241 py_implementation = platform.python_implementation();
242 if(platform.python_implementation()==""):
243 py_implementation = "Python";
244 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
245 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
246 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
247 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
248 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
249 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
250 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
251
252 compressionsupport = [];
253 try:
254 import gzip;
255 compressionsupport.append("gz");
256 compressionsupport.append("gzip");
257 except ImportError:
258 pass;
259 try:
260 import bz2;
261 compressionsupport.append("bz2");
262 compressionsupport.append("bzip2");
263 except ImportError:
264 pass;
265 try:
266 import lz4, lz4.frame;
267 compressionsupport.append("lz4");
268 except ImportError:
269 pass;
270 try:
271 import lzo;
272 compressionsupport.append("lzo");
273 compressionsupport.append("lzop");
274 except ImportError:
275 pass;
276 try:
277 import zstandard;
278 compressionsupport.append("zstd");
279 compressionsupport.append("zstandard");
280 except ImportError:
281 pass;
282 try:
283 import lzma;
284 compressionsupport.append("lzma");
285 compressionsupport.append("xz");
286 except ImportError:
287 try:
288 from backports import lzma;
289 compressionsupport.append("lzma");
290 compressionsupport.append("xz");
291 except ImportError:
292 pass;
293 compressionsupport.append("zlib");
294
295 compressionlist = ['auto'];
296 compressionlistalt = [];
297 outextlist = [];
298 outextlistwd = [];
299 if('gzip' in compressionsupport):
300 compressionlist.append('gzip');
301 compressionlistalt.append('gzip');
302 outextlist.append('gz');
303 outextlistwd.append('.gz');
304 if('bzip2' in compressionsupport):
305 compressionlist.append('bzip2');
306 compressionlistalt.append('bzip2');
307 outextlist.append('bz2');
308 outextlistwd.append('.bz2');
309 if('zstd' in compressionsupport):
310 compressionlist.append('zstd');
311 compressionlistalt.append('zstd');
312 outextlist.append('zst');
313 outextlistwd.append('.zst');
314 if('lz4' in compressionsupport):
315 compressionlist.append('lz4');
316 compressionlistalt.append('lz4');
317 outextlist.append('lz4');
318 outextlistwd.append('.lz4');
319 if('lzo' in compressionsupport):
320 compressionlist.append('lzo');
321 compressionlistalt.append('lzo');
322 outextlist.append('lzo');
323 outextlistwd.append('.lzo');
324 if('lzop' in compressionsupport):
325 compressionlist.append('lzop');
326 compressionlistalt.append('lzop');
327 outextlist.append('lzop');
328 outextlistwd.append('.lzop');
329 if('lzma' in compressionsupport):
330 compressionlist.append('lzma');
331 compressionlistalt.append('lzma');
332 outextlist.append('lzma');
333 outextlistwd.append('.lzma');
334 if('xz' in compressionsupport):
335 compressionlist.append('xz');
336 compressionlistalt.append('xz');
337 outextlist.append('xz');
338 outextlistwd.append('.xz');
339 if('zlib' in compressionsupport):
340 compressionlist.append('zlib');
341 compressionlistalt.append('zlib');
342 outextlist.append('zz');
343 outextlistwd.append('.zz');
344 outextlist.append('zl');
345 outextlistwd.append('.zl');
346 outextlist.append('zlib');
347 outextlistwd.append('.zlib');
348
349 tarfile_mimetype = "application/tar";
350 tarfile_tar_mimetype = tarfile_mimetype;
351 zipfile_mimetype = "application/zip";
352 zipfile_zip_mimetype = zipfile_mimetype;
353 rarfile_mimetype = "application/rar";
354 rarfile_rar_mimetype = rarfile_mimetype;
355 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
356 archivefile_cat_mimetype = archivefile_mimetype;
357 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
358 archivefile_gz_mimetype = archivefile_gzip_mimetype;
359 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
360 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
361 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
362 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
363 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
364 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
365 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
366 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
367 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
368 archivefile_zlib_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zlib";
369 archivefile_zz_mimetype = archivefile_zlib_mimetype;
370 archivefile_zl_mimetype = archivefile_zlib_mimetype;
371 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz", __file_format_extension__+".zz", __file_format_extension__+".zl", __file_format_extension__+".zlib"];
372
373 if __name__ == "__main__":
374 import subprocess;
375 curscrpath = os.path.dirname(sys.argv[0]);
376 if(curscrpath==""):
377 curscrpath = ".";
378 if(os.sep=="\\"):
379 curscrpath = curscrpath.replace(os.sep, "/");
380 curscrpath = curscrpath + "/";
381 scrfile = curscrpath + "catfile.py";
382 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
383 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
384 scrcmd.wait();
385
386 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
387 if(not dbgenable):
388 return True;
389 log_functions = {
390 "print": print,
391 "log": logging.info,
392 "warning": logging.warning,
393 "error": logging.error,
394 "critical": logging.critical,
395 "exception": logging.exception,
396 "logalt": lambda x: logging.log(dgblevel, x),
397 "debug": logging.debug
398 };
399 log_function = log_functions.get(outtype);
400 if(log_function):
401 log_function(dbgtxt);
402 return True;
403 return False;
404
405 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
406 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
407 return dbgtxt;
408
409 def RemoveWindowsPath(dpath):
410 if(dpath is None):
411 dpath = "";
412 if(os.sep!="/"):
413 dpath = dpath.replace(os.path.sep, "/");
414 dpath = dpath.rstrip("/");
415 if(dpath=="." or dpath==".."):
416 dpath = dpath + "/";
417 return dpath;
418
419 def NormalizeRelativePath(inpath):
420 inpath = RemoveWindowsPath(inpath);
421 if(os.path.isabs(inpath)):
422 outpath = inpath;
423 else:
424 if(inpath.startswith("./") or inpath.startswith("../")):
425 outpath = inpath;
426 else:
427 outpath = "./" + inpath;
428 return outpath;
429
430 def PrependPath(base_dir, child_path):
431 # Check if base_dir is None or empty, if so, return child_path as is
432 if not base_dir:
433 return child_path;
434 # Ensure base_dir ends with exactly one slash
435 if not base_dir.endswith('/'):
436 base_dir += '/';
437 # Check if child_path starts with ./ or ../ (indicating a relative path)
438 if child_path.startswith('./') or child_path.startswith('../'):
439 # For relative paths, we don't alter the child_path
440 return base_dir + child_path;
441 else:
442 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
443 return base_dir + child_path.lstrip('/');
444
445 def ListDir(dirpath, followlink=False, duplicates=False):
446 if isinstance(dirpath, (list, tuple, )):
447 dirpath = list(filter(None, dirpath));
448 elif isinstance(dirpath, str):
449 dirpath = list(filter(None, [dirpath]));
450 retlist = [];
451 fs_encoding = sys.getfilesystemencoding();
452 for mydirfile in dirpath:
453 if not os.path.exists(mydirfile):
454 return False;
455 mydirfile = NormalizeRelativePath(mydirfile);
456 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
457 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
458 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
459 for root, dirs, filenames in os.walk(mydirfile):
460 dpath = root;
461 dpath = RemoveWindowsPath(dpath);
462 if fs_encoding != 'utf-8':
463 dpath = dpath.encode(fs_encoding).decode('utf-8');
464 if dpath not in retlist and not duplicates:
465 retlist.append(dpath);
466 if duplicates:
467 retlist.append(dpath);
468 for file in filenames:
469 fpath = os.path.join(root, file);
470 fpath = RemoveWindowsPath(fpath);
471 if fs_encoding != 'utf-8':
472 fpath = fpath.encode(fs_encoding).decode('utf-8');
473 if fpath not in retlist and not duplicates:
474 retlist.append(fpath);
475 if duplicates:
476 retlist.append(fpath);
477 else:
478 path = RemoveWindowsPath(mydirfile);
479 if fs_encoding != 'utf-8':
480 path = path.encode(fs_encoding).decode('utf-8');
481 retlist.append(path);
482 return retlist;
483
484 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
485 if isinstance(dirpath, (list, tuple, )):
486 dirpath = list(filter(None, dirpath));
487 elif isinstance(dirpath, str):
488 dirpath = list(filter(None, [dirpath]));
489 retlist = [];
490 fs_encoding = sys.getfilesystemencoding();
491 for mydirfile in dirpath:
492 if not os.path.exists(mydirfile):
493 return False;
494 mydirfile = NormalizeRelativePath(mydirfile);
495 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
496 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
497 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
498 for root, dirs, filenames in os.walk(mydirfile):
499 # Sort dirs and filenames alphabetically in place
500 dirs.sort(key=lambda x: x.lower());
501 filenames.sort(key=lambda x: x.lower());
502 dpath = RemoveWindowsPath(root);
503 if fs_encoding != 'utf-8':
504 dpath = dpath.encode(fs_encoding).decode('utf-8');
505 if not duplicates and dpath not in retlist:
506 retlist.append(dpath);
507 elif duplicates:
508 retlist.append(dpath);
509 for file in filenames:
510 fpath = os.path.join(root, file);
511 fpath = RemoveWindowsPath(fpath);
512 if fs_encoding != 'utf-8':
513 fpath = fpath.encode(fs_encoding).decode('utf-8');
514 if not duplicates and fpath not in retlist:
515 retlist.append(fpath);
516 elif duplicates:
517 retlist.append(fpath);
518 else:
519 path = RemoveWindowsPath(mydirfile);
520 if fs_encoding != 'utf-8':
521 path = path.encode(fs_encoding).decode('utf-8');
522 retlist.append(path);
523 return retlist;
524
525 def create_alias_function(prefix, base_name, suffix, target_function):
526 # Define a new function that wraps the target function
527 def alias_function(*args, **kwargs):
528 return target_function(*args, **kwargs);
529 # Create the function name by combining the prefix, base name, and the suffix
530 function_name = "{}{}{}".format(prefix, base_name, suffix);
531 # Add the new function to the global namespace
532 globals()[function_name] = alias_function;
533
534 def create_alias_function_alt(prefix, base_name, suffix, target_function):
535 # Create the function name by combining the prefix, base name, and the suffix
536 # Use the format method for string formatting, compatible with Python 2 and 3
537 function_name = "{}{}{}".format(prefix, base_name, suffix)
538 # Add the new function (alias of the target_function) to the global namespace
539 # This line is compatible as-is with both Python 2 and 3
540 globals()[function_name] = target_function
541
542 def FormatSpecsListToDict(formatspecs=__file_format_list__):
543 if(isinstance(formatspecs, (list, tuple, ))):
544 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
545 elif(isinstance(formatspecs, (dict, ))):
546 return formatspecs;
547 else:
548 return __file_format_dict__;
549 return __file_format_dict__;
550
551 class ZlibFile:
552 def __init__(self, file_path=None, fileobj=None, mode='rb', level=9, wbits=15, encoding=None, errors=None, newline=None):
553 if file_path is None and fileobj is None:
554 raise ValueError("Either file_path or fileobj must be provided");
555 if file_path is not None and fileobj is not None:
556 raise ValueError("Only one of file_path or fileobj should be provided");
557
558 self.file_path = file_path;
559 self.fileobj = fileobj;
560 self.mode = mode;
561 self.level = level;
562 self.wbits = wbits;
563 self.encoding = encoding;
564 self.errors = errors;
565 self.newline = newline;
566 self._compressed_data = b'';
567 self._decompressed_data = b'';
568 self._position = 0;
569 self._text_mode = 't' in mode;
570
571 # Force binary mode for internal handling
572 internal_mode = mode.replace('t', 'b');
573
574 if 'w' in mode or 'a' in mode or 'x' in mode:
575 self.file = open(file_path, internal_mode) if file_path else fileobj;
576 self._compressor = zlib.compressobj(level, zlib.DEFLATED, wbits);
577 elif 'r' in mode:
578 if file_path:
579 if os.path.exists(file_path):
580 self.file = open(file_path, internal_mode);
581 self._load_file();
582 else:
583 raise FileNotFoundError("No such file: '{}'".format(file_path));
584 elif fileobj:
585 self.file = fileobj;
586 self._load_file();
587 else:
588 raise ValueError("Mode should be 'rb' or 'wb'");
589
590 def _load_file(self):
591 self.file.seek(0);
592 self._compressed_data = self.file.read();
593 if not self._compressed_data.startswith((b'\x78\x01', b'\x78\x5E', b'\x78\x9C', b'\x78\xDA')):
594 raise ValueError("Invalid zlib file header");
595 self._decompressed_data = zlib.decompress(self._compressed_data, self.wbits);
596 if self._text_mode:
597 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
598
599 def write(self, data):
600 if self._text_mode:
601 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
602 compressed_data = self._compressor.compress(data) + self._compressor.flush(zlib.Z_SYNC_FLUSH);
603 self.file.write(compressed_data);
604
605 def read(self, size=-1):
606 if size == -1:
607 size = len(self._decompressed_data) - self._position;
608 data = self._decompressed_data[self._position:self._position + size];
609 self._position += size;
610 return data;
611
612 def seek(self, offset, whence=0):
613 if whence == 0: # absolute file positioning
614 self._position = offset;
615 elif whence == 1: # seek relative to the current position
616 self._position += offset;
617 elif whence == 2: # seek relative to the file's end
618 self._position = len(self._decompressed_data) + offset;
619 else:
620 raise ValueError("Invalid value for whence");
621
622 # Ensure the position is within bounds
623 self._position = max(0, min(self._position, len(self._decompressed_data)));
624
625 def tell(self):
626 return self._position;
627
628 def flush(self):
629 self.file.flush();
630
631 def fileno(self):
632 if hasattr(self.file, 'fileno'):
633 return self.file.fileno();
634 raise OSError("The underlying file object does not support fileno()");
635
636 def isatty(self):
637 if hasattr(self.file, 'isatty'):
638 return self.file.isatty();
639 return False;
640
641 def truncate(self, size=None):
642 if hasattr(self.file, 'truncate'):
643 return self.file.truncate(size);
644 raise OSError("The underlying file object does not support truncate()");
645
646 def close(self):
647 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
648 self.file.write(self._compressor.flush(zlib.Z_FINISH));
649 if self.file_path:
650 self.file.close();
651
652 def __enter__(self):
653 return self;
654
655 def __exit__(self, exc_type, exc_value, traceback):
656 self.close();
657
658 class GzipFile:
659 def __init__(self, file_path=None, fileobj=None, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None):
660 if file_path is None and fileobj is None:
661 raise ValueError("Either file_path or fileobj must be provided");
662 if file_path is not None and fileobj is not None:
663 raise ValueError("Only one of file_path or fileobj should be provided");
664
665 self.file_path = file_path;
666 self.fileobj = fileobj;
667 self.mode = mode;
668 self.compresslevel = compresslevel;
669 self.encoding = encoding;
670 self.errors = errors;
671 self.newline = newline;
672 self._compressed_data = b'';
673 self._decompressed_data = b'';
674 self._position = 0;
675 self._text_mode = 't' in mode;
676
677 # Force binary mode for internal handling
678 internal_mode = mode.replace('t', 'b');
679
680 if 'w' in mode or 'a' in mode or 'x' in mode:
681 self.file = gzip.open(file_path, internal_mode, compresslevel=compresslevel) if file_path else gzip.GzipFile(fileobj=fileobj, mode=internal_mode, compresslevel=compresslevel);
682 self._compressor = gzip.GzipFile(fileobj=self.file, mode=internal_mode, compresslevel=compresslevel);
683 elif 'r' in mode:
684 if file_path:
685 if os.path.exists(file_path):
686 self.file = gzip.open(file_path, internal_mode);
687 self._load_file();
688 else:
689 raise FileNotFoundError("No such file: '{}'".format(file_path));
690 elif fileobj:
691 self.file = gzip.GzipFile(fileobj=fileobj, mode=internal_mode);
692 self._load_file();
693 else:
694 raise ValueError("Mode should be 'rb' or 'wb'");
695
696 def _load_file(self):
697 self.file.seek(0);
698 self._compressed_data = self.file.read();
699 if not self._compressed_data.startswith(b'\x1f\x8b'):
700 raise ValueError("Invalid gzip file header");
701 self._decompressed_data = gzip.decompress(self._compressed_data);
702 if self._text_mode:
703 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
704
705 def write(self, data):
706 if self._text_mode:
707 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
708 compressed_data = self._compressor.compress(data);
709 self.file.write(compressed_data);
710 self.file.flush();
711
712 def read(self, size=-1):
713 if size == -1:
714 size = len(self._decompressed_data) - self._position;
715 data = self._decompressed_data[self._position:self._position + size];
716 self._position += size;
717 return data;
718
719 def seek(self, offset, whence=0):
720 if whence == 0: # absolute file positioning
721 self._position = offset;
722 elif whence == 1: # seek relative to the current position
723 self._position += offset;
724 elif whence == 2: # seek relative to the file's end
725 self._position = len(self._decompressed_data) + offset;
726 else:
727 raise ValueError("Invalid value for whence");
728
729 # Ensure the position is within bounds
730 self._position = max(0, min(self._position, len(self._decompressed_data)));
731
732 def tell(self):
733 return self._position;
734
735 def flush(self):
736 self.file.flush();
737
738 def fileno(self):
739 if hasattr(self.file, 'fileno'):
740 return self.file.fileno();
741 raise OSError("The underlying file object does not support fileno()");
742
743 def isatty(self):
744 if hasattr(self.file, 'isatty'):
745 return self.file.isatty();
746 return False;
747
748 def truncate(self, size=None):
749 if hasattr(self.file, 'truncate'):
750 return self.file.truncate(size);
751 raise OSError("The underlying file object does not support truncate()");
752
753 def close(self):
754 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
755 self.file.write(self._compressor.flush());
756 if self.file_path:
757 self.file.close();
758
759 def __enter__(self):
760 return self;
761
762 def __exit__(self, exc_type, exc_value, traceback):
763 self.close();
764
765 '''
766 class BloscFile:
767 def __init__(self, file_path=None, fileobj=None, mode='rb', level=9, encoding=None, errors=None, newline=None):
768 if file_path is None and fileobj is None:
769 raise ValueError("Either file_path or fileobj must be provided");
770 if file_path is not None and fileobj is not None:
771 raise ValueError("Only one of file_path or fileobj should be provided");
772
773 self.file_path = file_path;
774 self.fileobj = fileobj;
775 self.mode = mode;
776 self.level = level;
777 self.encoding = encoding;
778 self.errors = errors;
779 self.newline = newline;
780 self._compressed_data = b'';
781 self._decompressed_data = b'';
782 self._position = 0;
783 self._text_mode = 't' in mode;
784
785 # Force binary mode for internal handling
786 internal_mode = mode.replace('t', 'b');
787
788 if 'w' in mode or 'a' in mode or 'x' in mode:
789 self.file = open(file_path, internal_mode) if file_path else fileobj;
790 self._compressor = blosc.Blosc(level);
791 elif 'r' in mode:
792 if file_path:
793 if os.path.exists(file_path):
794 self.file = open(file_path, internal_mode);
795 self._load_file();
796 else:
797 raise FileNotFoundError("No such file: '{}'".format(file_path));
798 elif fileobj:
799 self.file = fileobj;
800 self._load_file();
801 else:
802 raise ValueError("Mode should be 'rb' or 'wb'");
803
804 def _load_file(self):
805 self.file.seek(0);
806 self._compressed_data = self.file.read();
807 if not self._compressed_data:
808 raise ValueError("Invalid blosc file header");
809 self._decompressed_data = blosc.decompress(self._compressed_data);
810 if self._text_mode:
811 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
812
813 def write(self, data):
814 if self._text_mode:
815 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
816 compressed_data = blosc.compress(data, cname='blosclz', clevel=self.level);
817 self.file.write(compressed_data);
818 self.file.flush();
819
820 def read(self, size=-1):
821 if size == -1:
822 size = len(self._decompressed_data) - self._position;
823 data = self._decompressed_data[self._position:self._position + size];
824 self._position += size;
825 return data;
826
827 def seek(self, offset, whence=0):
828 if whence == 0: # absolute file positioning
829 self._position = offset;
830 elif whence == 1: # seek relative to the current position
831 self._position += offset;
832 elif whence == 2: # seek relative to the file's end
833 self._position = len(self._decompressed_data) + offset;
834 else:
835 raise ValueError("Invalid value for whence");
836
837 # Ensure the position is within bounds
838 self._position = max(0, min(self._position, len(self._decompressed_data)));
839
840 def tell(self):
841 return self._position;
842
843 def flush(self):
844 self.file.flush();
845
846 def fileno(self):
847 if hasattr(self.file, 'fileno'):
848 return self.file.fileno();
849 raise OSError("The underlying file object does not support fileno()");
850
851 def isatty(self):
852 if hasattr(self.file, 'isatty'):
853 return self.file.isatty();
854 return False;
855
856 def truncate(self, size=None):
857 if hasattr(self.file, 'truncate'):
858 return self.file.truncate(size);
859 raise OSError("The underlying file object does not support truncate()");
860
861 def close(self):
862 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
863 self.file.write(blosc.compress(self._compressor.flush(), cname='blosclz', clevel=self.level));
864 if self.file_path:
865 self.file.close();
866
867 def __enter__(self):
868 return self;
869
870 def __exit__(self, exc_type, exc_value, traceback):
871 self.close();
872
873 class BrotliFile:
874 def __init__(self, file_path=None, fileobj=None, mode='rb', level=11, encoding=None, errors=None, newline=None):
875 if file_path is None and fileobj is None:
876 raise ValueError("Either file_path or fileobj must be provided");
877 if file_path is not None and fileobj is not None:
878 raise ValueError("Only one of file_path or fileobj should be provided");
879
880 self.file_path = file_path;
881 self.fileobj = fileobj;
882 self.mode = mode;
883 self.level = level;
884 self.encoding = encoding;
885 self.errors = errors;
886 self.newline = newline;
887 self._compressed_data = b'';
888 self._decompressed_data = b'';
889 self._position = 0;
890 self._text_mode = 't' in mode;
891
892 # Force binary mode for internal handling
893 internal_mode = mode.replace('t', 'b');
894
895 if 'w' in mode or 'a' in mode or 'x' in mode:
896 self.file = open(file_path, internal_mode) if file_path else fileobj;
897 self._compressor = brotli.Compressor(quality=self.level);
898 elif 'r' in mode:
899 if file_path:
900 if os.path.exists(file_path):
901 self.file = open(file_path, internal_mode);
902 self._load_file();
903 else:
904 raise FileNotFoundError("No such file: '{}'".format(file_path));
905 elif fileobj:
906 self.file = fileobj;
907 self._load_file();
908 else:
909 raise ValueError("Mode should be 'rb' or 'wb'");
910
911 def _load_file(self):
912 self.file.seek(0);
913 self._compressed_data = self.file.read();
914 if not self._compressed_data:
915 raise ValueError("Invalid brotli file header");
916 self._decompressed_data = brotli.decompress(self._compressed_data);
917 if self._text_mode:
918 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
919
920 def write(self, data):
921 if self._text_mode:
922 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
923 compressed_data = self._compressor.process(data);
924 self.file.write(compressed_data);
925 self.file.flush();
926
927 def read(self, size=-1):
928 if size == -1:
929 size = len(self._decompressed_data) - self._position;
930 data = self._decompressed_data[self._position:self._position + size];
931 self._position += size;
932 return data;
933
934 def seek(self, offset, whence=0):
935 if whence == 0: # absolute file positioning
936 self._position = offset;
937 elif whence == 1: # seek relative to the current position
938 self._position += offset;
939 elif whence == 2: # seek relative to the file's end
940 self._position = len(self._decompressed_data) + offset;
941 else:
942 raise ValueError("Invalid value for whence");
943
944 # Ensure the position is within bounds
945 self._position = max(0, min(self._position, len(self._decompressed_data)));
946
947 def tell(self):
948 return self._position;
949
950 def flush(self):
951 self.file.flush();
952
953 def fileno(self):
954 if hasattr(self.file, 'fileno'):
955 return self.file.fileno();
956 raise OSError("The underlying file object does not support fileno()");
957
958 def isatty(self):
959 if hasattr(self.file, 'isatty'):
960 return self.file.isatty();
961 return False;
962
963 def truncate(self, size=None):
964 if hasattr(self.file, 'truncate'):
965 return self.file.truncate(size);
966 raise OSError("The underlying file object does not support truncate()");
967
968 def close(self):
969 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
970 self.file.write(self._compressor.finish());
971 if self.file_path:
972 self.file.close();
973
974 def __enter__(self):
975 return self;
976
977 def __exit__(self, exc_type, exc_value, traceback):
978 self.close();
979 '''
980
981 def TarFileCheck(infile):
982 try:
983 if is_tarfile(infile):
984 return True;
985 else:
986 return False;
987 except TypeError:
988 try:
989 # Check if the input is a file object
990 if hasattr(infile, 'read'):
991 # Save the current file position
992 current_position = infile.tell();
993 # Attempt to open the file object as a tar file
994 tar = tarfile.open(fileobj=infile);
995 tar.close();
996 # Restore the file position
997 infile.seek(current_position);
998 return True;
999 else:
1000 # Assume it's a filename
1001 tar = tarfile.open(name=infile);
1002 tar.close();
1003 return True;
1004 except tarfile.TarError:
1005 return False;
1006
1007 def TarFileCheckAlt(infile):
1008 try:
1009 if is_tarfile(infile):
1010 return True;
1011 except TypeError:
1012 pass;
1013 try:
1014 # Check if the input is a file-like object
1015 if hasattr(infile, 'read'):
1016 # Save the current file position
1017 current_position = infile.tell();
1018 # Attempt to open the file object as a tar file
1019 with tarfile.open(fileobj=infile) as tar:
1020 pass;
1021 # Restore the file position
1022 infile.seek(current_position);
1023 else:
1024 # Assume it's a filename and attempt to open it as a tar file
1025 with tarfile.open(name=infile) as tar:
1026 pass;
1027 return True;
1028 except (tarfile.TarError, AttributeError, IOError):
1029 return False;
1030
1031 def ZipFileCheck(infile):
1032 try:
1033 if zipfile.is_zipfile(infile):
1034 return True;
1035 except TypeError:
1036 pass;
1037 try:
1038 # Check if the input is a file-like object
1039 if hasattr(infile, 'read'):
1040 # Save the current file position
1041 current_position = infile.tell();
1042 # Attempt to open the file object as a zip file
1043 with zipfile.ZipFile(infile) as zipf:
1044 pass;
1045 # Restore the file position
1046 infile.seek(current_position);
1047 else:
1048 # Assume it's a filename and attempt to open it as a zip file
1049 with zipfile.ZipFile(infile) as zipf:
1050 pass;
1051 return True;
1052 except (zipfile.BadZipFile, AttributeError, IOError):
1053 return False;
1054
1055 def RarFileCheck(infile):
1056 try:
1057 if rarfile.is_rarfile(infile):
1058 return True;
1059 except TypeError:
1060 pass;
1061 try:
1062 # Check if the input is a file-like object
1063 if hasattr(infile, 'read'):
1064 # Save the current file position
1065 current_position = infile.tell();
1066 # Attempt to open the file object as a rar file
1067 with rarfile.RarFile(infile) as rarf:
1068 pass;
1069 # Restore the file position
1070 infile.seek(current_position);
1071 else:
1072 # Assume it's a filename and attempt to open it as a rar file
1073 with rarfile.RarFile(infile) as rarf:
1074 pass;
1075 return True;
1076 except (rarfile.Error, AttributeError, IOError):
1077 return False;
1078
1079 def SevenZipFileCheck(infile):
1080 try:
1081 # Check if the input is a file-like object
1082 if hasattr(infile, 'read'):
1083 # Save the current file position
1084 current_position = infile.tell();
1085 # Attempt to open the file object as a 7z file
1086 with py7zr.SevenZipFile(infile, 'r') as archive:
1087 pass;
1088 # Restore the file position
1089 infile.seek(current_position);
1090 else:
1091 # Assume it's a filename and attempt to open it as a 7z file
1092 with py7zr.SevenZipFile(infile, 'r') as archive:
1093 pass;
1094 return True;
1095 except (py7zr.Bad7zFile, AttributeError, IOError):
1096 return False;
1097
1098 # initial_value can be 0xFFFF or 0x0000
1099 def crc16_ansi(msg, initial_value=0xFFFF):
1100 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
1101 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
1102 crc = initial_value; # Initial value
1103 for b in msg:
1104 crc ^= b << 8; # XOR byte into CRC top byte
1105 for _ in range(8): # Process each bit
1106 if crc & 0x8000: # If the top bit is set
1107 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
1108 else:
1109 crc = crc << 1; # Just shift left
1110 crc &= 0xFFFF; # Ensure CRC remains 16-bit
1111 return crc;
1112
1113 # initial_value can be 0xFFFF or 0x0000
1114 def crc16_ibm(msg, initial_value=0xFFFF):
1115 return crc16_ansi(msg, initial_value);
1116
1117 # initial_value is 0xFFFF
1118 def crc16(msg):
1119 return crc16_ansi(msg, 0xFFFF);
1120
1121 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
1122 def crc16_ccitt(msg, initial_value=0xFFFF):
1123 # CRC-16-CCITT polynomial
1124 poly = 0x1021; # Polynomial for CRC-16-CCITT
1125 # Use the specified initial value
1126 crc = initial_value;
1127 for b in msg:
1128 crc ^= b << 8; # XOR byte into CRC top byte
1129 for _ in range(8): # Process each bit
1130 if crc & 0x8000: # If the top bit is set
1131 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
1132 else:
1133 crc = crc << 1; # Just shift left
1134 crc &= 0xFFFF; # Ensure CRC remains 16-bit
1135 return crc;
1136
1137 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
1138 def crc64_ecma(msg, initial_value=0x0000000000000000):
1139 # CRC-64-ECMA polynomial and initial value
1140 poly = 0x42F0E1EBA9EA3693;
1141 crc = initial_value; # Initial value for CRC-64-ECMA
1142 for b in msg:
1143 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
1144 for _ in range(8): # Process each bit
1145 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
1146 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
1147 else:
1148 crc <<= 1; # Just shift left if the MSB is 0
1149 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
1150 return crc;
1151
1152 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
1153 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
1154 # CRC-64-ISO polynomial and initial value
1155 poly = 0x000000000000001B;
1156 crc = initial_value; # Common initial value for CRC-64-ISO
1157 for b in msg:
1158 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
1159 for _ in range(8): # Process each bit
1160 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
1161 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
1162 else:
1163 crc <<= 1; # Just shift left if the MSB is 0
1164 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
1165 return crc;
1166
1167 def GetDataFromArray(data, path, default=None):
1168 element = data;
1169 try:
1170 for key in path:
1171 element = element[key];
1172 return element;
1173 except (KeyError, TypeError, IndexError):
1174 return default;
1175
1176 def GetDataFromArrayAlt(structure, path, default=None):
1177 element = structure;
1178 for key in path:
1179 if isinstance(element, dict) and key in element:
1180 element = element[key];
1181 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
1182 element = element[key];
1183 else:
1184 return default;
1185 return element;
1186
1187 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
1188 formatspecs = FormatSpecsListToDict(formatspecs);
1189 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
1190 if encodedata:
1191 fileheader = fileheader.encode('UTF-8');
1192 checksum_methods = {
1193 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1194 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1195 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1196 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
1197 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
1198 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
1199 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
1200 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1201 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1202 };
1203 if checksumtype in checksum_methods:
1204 return checksum_methods[checksumtype](fileheader);
1205 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
1206 checksumoutstr = hashlib.new(checksumtype);
1207 checksumoutstr.update(fileheader);
1208 return checksumoutstr.hexdigest().lower();
1209 return format(0, 'x').lower();
1210
1211 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
1212 formatspecs = FormatSpecsListToDict(formatspecs);
1213 if encodedata:
1214 instr = instr.encode('UTF-8');
1215 checksum_methods = {
1216 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1217 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1218 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1219 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
1220 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
1221 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
1222 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
1223 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1224 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1225 };
1226 if checksumtype in checksum_methods:
1227 return checksum_methods[checksumtype](instr);
1228 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
1229 checksumoutstr = hashlib.new(checksumtype);
1230 checksumoutstr.update(instr);
1231 return checksumoutstr.hexdigest().lower();
1232 return format(0, 'x').lower();
1233
1234 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
1235 formatspecs = FormatSpecsListToDict(formatspecs);
1236 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
1237 return inchecksum.lower() == catfileheadercshex;
1238
1239 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
1240 formatspecs = FormatSpecsListToDict(formatspecs);
1241 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
1242 return inchecksum.lower() == catinfilecshex;
1243
1244 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
1245 curbyte = b"";
1246 curfullbyte = b"";
1247 nullbyte = delimiter.encode("UTF-8");
1248 while(True):
1249 curbyte = fp.read(1);
1250 if(curbyte==nullbyte or not curbyte):
1251 break;
1252 curfullbyte = curfullbyte + curbyte;
1253 return curfullbyte.decode('UTF-8');
1254
1255 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
1256 return ReadTillNullByteOld(fp, delimiter);
1257
1258 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
1259 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
1260 buffer = bytearray();
1261 total_read = 0;
1262 delimiter_length = len(delimiter);
1263 while True:
1264 chunk = fp.read(chunk_size)
1265 if not chunk:
1266 # End of file reached without finding the delimiter
1267 break;
1268 buffer.extend(chunk);
1269 total_read += len(chunk);
1270 if delimiter in buffer:
1271 # Delimiter found, calculate where to reset the file pointer
1272 index = buffer.find(delimiter);
1273 # Calculate how many extra bytes were read after the delimiter
1274 extra_bytes_read = len(buffer) - (index + delimiter_length);
1275 # Move the file pointer back to just after the delimiter
1276 fp.seek(-extra_bytes_read, 1);
1277 buffer = buffer[:index];
1278 break;
1279 if total_read >= max_read:
1280 # Stop reading if max limit is reached to prevent excessive memory usage
1281 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1282 # Check for incomplete UTF-8 sequences at the end of the buffer
1283 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
1284 # This suggests that the last byte might be the start of a multi-byte character
1285 # Try to read one more byte to complete the character
1286 extra_byte = fp.read(1);
1287 if extra_byte:
1288 buffer.extend(extra_byte);
1289 else:
1290 # No more data available
1291 break;
1292 try:
1293 return buffer.decode('UTF-8', errors='replace');
1294 except UnicodeDecodeError:
1295 return buffer.decode('UTF-8', errors='replace');
1296
1297 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
1298 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
1299
1300 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
1301 curfullbyte = bytearray();
1302 nullbyte = delimiter.encode("UTF-8");
1303 total_read = 0; # Track the total number of bytes read
1304 while True:
1305 curbyte = fp.read(1);
1306 if curbyte == nullbyte or not curbyte:
1307 break;
1308 curfullbyte.extend(curbyte);
1309 total_read += 1;
1310 if total_read >= max_read:
1311 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1312 # Decode the full byte array to string once out of the loop
1313 try:
1314 return curfullbyte.decode('UTF-8');
1315 except UnicodeDecodeError:
1316 # Handle potential partial UTF-8 characters
1317 for i in range(1, 4):
1318 try:
1319 return curfullbyte[:-i].decode('UTF-8');
1320 except UnicodeDecodeError:
1321 continue;
1322 raise; # Re-raise if decoding fails even after trimming
1323
1324 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
1325 return ReadTillNullByte(fp, delimiter, max_read);
1326
1327 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
1328 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
1329 buffer = bytearray();
1330 total_read = 0;
1331 delimiter_length = len(delimiter);
1332 results = [];
1333 while len(results) < num_delimiters:
1334 chunk = fp.read(chunk_size);
1335 if not chunk:
1336 # End of file reached; decode whatever is collected if it's the last needed part
1337 if len(buffer) > 0:
1338 results.append(buffer.decode('UTF-8', errors='replace'));
1339 break;
1340 buffer.extend(chunk)
1341 total_read += len(chunk);
1342 # Check if we have found the delimiter
1343 while delimiter in buffer:
1344 index = buffer.find(delimiter);
1345 # Decode the section before the delimiter
1346 results.append(buffer[:index].decode('UTF-8', errors='replace'));
1347 # Remove the processed part from the buffer
1348 buffer = buffer[index + delimiter_length:];
1349 if len(results) == num_delimiters:
1350 # If reached the required number of delimiters, adjust the file pointer and stop
1351 fp.seek(-len(buffer), 1);
1352 return results;
1353 if total_read >= max_read:
1354 # Stop reading if max limit is reached to prevent excessive memory usage
1355 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1356 # Check for incomplete UTF-8 sequences at the end of the buffer
1357 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
1358 # This suggests that the last byte might be the start of a multi-byte character
1359 # Try to read one more byte to complete the character
1360 extra_byte = fp.read(1);
1361 if extra_byte:
1362 buffer.extend(extra_byte);
1363 else:
1364 # No more data available
1365 break;
1366 # Process remaining buffer if less than the required number of delimiters were found
1367 if len(buffer) > 0 and len(results) < num_delimiters:
1368 results.append(buffer.decode('UTF-8', errors='replace'));
1369 return results;
1370
1371 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
1372 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
1373
1374 def SeekToEndOfFile(fp):
1375 lasttell = 0;
1376 while(True):
1377 fp.seek(1, 1);
1378 if(lasttell==fp.tell()):
1379 break;
1380 lasttell = fp.tell();
1381 return True;
1382
1383 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
1384 rocount = 0;
1385 roend = int(rounds);
1386 HeaderOut = [];
1387 while(rocount<roend):
1388 HeaderOut.append(ReadTillNullByte(fp, delimiter));
1389 rocount = rocount + 1;
1390 return HeaderOut;
1391
1392 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
1393 headerpresize = ReadTillNullByte(fp, delimiter);
1394 headersize = int(headerpresize, 16);
1395 if(headersize<=0):
1396 return [];
1397 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
1398 fp.seek(1, 1);
1399 rocount = 0;
1400 roend = int(len(headercontent));
1401 HeaderOut = [headerpresize];
1402 while(rocount<roend):
1403 HeaderOut.append(headercontent[rocount]);
1404 rocount = rocount + 1;
1405 return HeaderOut;
1406
1407 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
1408 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
1409 headersize = int(preheaderdata[0], 16);
1410 headernumfields = int(preheaderdata[1], 16);
1411 if(headersize<=0 or headernumfields<=0):
1412 return [];
1413 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
1414 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
1415 HeaderOut = preheaderdata + headerdata;
1416 return HeaderOut;
1417
1418 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1419 formatspecs = FormatSpecsListToDict(formatspecs);
1420 delimiter = formatspecs['format_delimiter'];
1421 fheaderstart = fp.tell();
1422 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1423 if(len(HeaderOut)==0):
1424 return False;
1425 if(re.findall(r"^[.|/]", HeaderOut[3])):
1426 fname = HeaderOut[3];
1427 else:
1428 fname = "./"+HeaderOut[3];
1429 fcs = HeaderOut[-2].lower();
1430 fccs = HeaderOut[-1].lower();
1431 fsize = int(HeaderOut[5], 16);
1432 fcompression = HeaderOut[12];
1433 fcsize = int(HeaderOut[13], 16);
1434 fseeknextfile = HeaderOut[25];
1435 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1436 if(fcs!=newfcs and not skipchecksum):
1437 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1438 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1439 return False;
1440 fhend = fp.tell() - 1;
1441 fcontentstart = fp.tell();
1442 fcontents = BytesIO();
1443 if(fsize>0 and not listonly):
1444 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1445 fcontents.write(fp.read(fsize));
1446 else:
1447 fcontents.write(fp.read(fcsize));
1448 elif(fsize>0 and listonly):
1449 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1450 fp.seek(fsize, 1);
1451 else:
1452 fp.seek(fcsize, 1);
1453 fcontents.seek(0, 0);
1454 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1455 if(fccs!=newfccs and not skipchecksum and not listonly):
1456 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1457 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1458 return False;
1459 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1460 pass;
1461 else:
1462 fcontents.seek(0, 0);
1463 if(uncompress):
1464 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1465 fcontentend = fp.tell();
1466 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1467 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1468 if(abs(fseeknextasnum)==0):
1469 pass;
1470 fp.seek(fseeknextasnum, 1);
1471 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1472 fseeknextasnum = int(fseeknextfile);
1473 if(abs(fseeknextasnum)==0):
1474 pass;
1475 fp.seek(fseeknextasnum, 1);
1476 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1477 fseeknextasnum = int(fseeknextfile);
1478 if(abs(fseeknextasnum)==0):
1479 pass;
1480 fp.seek(fseeknextasnum, 0);
1481 else:
1482 return False;
1483 HeaderOut.append(fcontents);
1484 return HeaderOut;
1485
1486 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1487 formatspecs = FormatSpecsListToDict(formatspecs);
1488 delimiter = formatspecs['format_delimiter'];
1489 fheaderstart = fp.tell();
1490 if(formatspecs['new_style']):
1491 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1492 else:
1493 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1494 if(len(HeaderOut)==0):
1495 return False;
1496 fheadsize = int(HeaderOut[0], 16);
1497 fnumfields = int(HeaderOut[1], 16);
1498 ftype = int(HeaderOut[2], 16);
1499 if(re.findall(r"^[.|/]", HeaderOut[3])):
1500 fname = HeaderOut[3];
1501 else:
1502 fname = "./"+HeaderOut[3];
1503 fbasedir = os.path.dirname(fname);
1504 flinkname = HeaderOut[4];
1505 fsize = int(HeaderOut[5], 16);
1506 fatime = int(HeaderOut[6], 16);
1507 fmtime = int(HeaderOut[7], 16);
1508 fctime = int(HeaderOut[8], 16);
1509 fbtime = int(HeaderOut[9], 16);
1510 fmode = int(HeaderOut[10], 16);
1511 fchmode = stat.S_IMODE(fmode);
1512 ftypemod = stat.S_IFMT(fmode);
1513 fwinattributes = int(HeaderOut[11], 16);
1514 fcompression = HeaderOut[12];
1515 fcsize = int(HeaderOut[13], 16);
1516 fuid = int(HeaderOut[14], 16);
1517 funame = HeaderOut[15];
1518 fgid = int(HeaderOut[16], 16);
1519 fgname = HeaderOut[17];
1520 fid = int(HeaderOut[18], 16);
1521 finode = int(HeaderOut[19], 16);
1522 flinkcount = int(HeaderOut[20], 16);
1523 fdev_minor = int(HeaderOut[21], 16);
1524 fdev_major = int(HeaderOut[22], 16);
1525 frdev_minor = int(HeaderOut[23], 16);
1526 frdev_major = int(HeaderOut[24], 16);
1527 fseeknextfile = HeaderOut[25];
1528 fextrasize = int(HeaderOut[26], 16);
1529 fextrafields = int(HeaderOut[27], 16);
1530 extrafieldslist = [];
1531 extrastart = 28
1532 extraend = extrastart + fextrafields;
1533 extrafieldslist = [];
1534 if(extrastart<extraend):
1535 extrafieldslist.append(HeaderOut[extrastart]);
1536 extrastart = extrastart + 1;
1537 fcs = HeaderOut[-2].lower();
1538 fccs = HeaderOut[-1].lower();
1539 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1540 if(fcs!=newfcs and not skipchecksum):
1541 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1542 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1543 return False;
1544 fhend = fp.tell() - 1;
1545 fcontentstart = fp.tell();
1546 fcontents = BytesIO();
1547 pyhascontents = False;
1548 if(fsize>0 and not listonly):
1549 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1550 fcontents.write(fp.read(fsize));
1551 else:
1552 fcontents.write(fp.read(fcsize));
1553 pyhascontents = True;
1554 elif(fsize>0 and listonly):
1555 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1556 fp.seek(fsize, 1);
1557 else:
1558 fp.seek(fcsize, 1);
1559 pyhascontents = False;
1560 fcontents.seek(0, 0);
1561 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1562 if(fccs!=newfccs and not skipchecksum and not listonly):
1563 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1564 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1565 return False;
1566 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1567 pass;
1568 else:
1569 fcontents.seek(0, 0);
1570 if(uncompress):
1571 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1572 fcontents.seek(0, 0);
1573 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1574 fcontentend = fp.tell() - 1;
1575 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1576 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1577 if(abs(fseeknextasnum)==0):
1578 pass;
1579 fp.seek(fseeknextasnum, 1);
1580 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1581 fseeknextasnum = int(fseeknextfile);
1582 if(abs(fseeknextasnum)==0):
1583 pass;
1584 fp.seek(fseeknextasnum, 1);
1585 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1586 fseeknextasnum = int(fseeknextfile);
1587 if(abs(fseeknextasnum)==0):
1588 pass;
1589 fp.seek(fseeknextasnum, 0);
1590 else:
1591 return False;
1592 fcontents.seek(0, 0);
1593 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1594 return catlist;
1595
1596 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1597 formatspecs = FormatSpecsListToDict(formatspecs);
1598 delimiter = formatspecs['format_delimiter'];
1599 fheaderstart = fp.tell();
1600 if(formatspecs['new_style']):
1601 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1602 else:
1603 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1604 if(len(HeaderOut)==0):
1605 return False;
1606 fheadsize = int(HeaderOut[0], 16);
1607 fnumfields = int(HeaderOut[1], 16);
1608 ftype = int(HeaderOut[2], 16);
1609 if(re.findall(r"^[.|/]", HeaderOut[3])):
1610 fname = HeaderOut[3];
1611 else:
1612 fname = "./"+HeaderOut[3];
1613 fbasedir = os.path.dirname(fname);
1614 flinkname = HeaderOut[4];
1615 fsize = int(HeaderOut[5], 16);
1616 fatime = int(HeaderOut[6], 16);
1617 fmtime = int(HeaderOut[7], 16);
1618 fctime = int(HeaderOut[8], 16);
1619 fbtime = int(HeaderOut[9], 16);
1620 fmode = int(HeaderOut[10], 16);
1621 fchmode = stat.S_IMODE(fmode);
1622 ftypemod = stat.S_IFMT(fmode);
1623 fwinattributes = int(HeaderOut[11], 16);
1624 fcompression = HeaderOut[12];
1625 fcsize = int(HeaderOut[13], 16);
1626 fuid = int(HeaderOut[14], 16);
1627 funame = HeaderOut[15];
1628 fgid = int(HeaderOut[16], 16);
1629 fgname = HeaderOut[17];
1630 fid = int(HeaderOut[18], 16);
1631 finode = int(HeaderOut[19], 16);
1632 flinkcount = int(HeaderOut[20], 16);
1633 fdev_minor = int(HeaderOut[21], 16);
1634 fdev_major = int(HeaderOut[22], 16);
1635 frdev_minor = int(HeaderOut[23], 16);
1636 frdev_major = int(HeaderOut[24], 16);
1637 fseeknextfile = HeaderOut[25];
1638 fextrasize = int(HeaderOut[26], 16);
1639 fextrafields = int(HeaderOut[27], 16);
1640 extrafieldslist = [];
1641 extrastart = 28;
1642 extraend = extrastart + fextrafields;
1643 extrafieldslist = [];
1644 if(extrastart<extraend):
1645 extrafieldslist.append(HeaderOut[extrastart]);
1646 extrastart = extrastart + 1;
1647 fheaderchecksumtype = HeaderOut[extrastart].lower();
1648 fcontentchecksumtype = HeaderOut[extrastart + 1].lower();
1649 fcs = HeaderOut[-2].lower();
1650 fccs = HeaderOut[-1].lower();
1651 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1652 if(fcs!=newfcs and not skipchecksum):
1653 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1654 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1655 return False;
1656 fhend = fp.tell() - 1;
1657 fcontentstart = fp.tell();
1658 fcontents = BytesIO();
1659 pyhascontents = False;
1660 if(fsize>0 and not listonly):
1661 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1662 fcontents.write(fp.read(fsize));
1663 else:
1664 fcontents.write(fp.read(fcsize));
1665 pyhascontents = True;
1666 elif(fsize>0 and listonly):
1667 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1668 fp.seek(fsize, 1);
1669 else:
1670 fp.seek(fcsize, 1);
1671 pyhascontents = False;
1672 fcontents.seek(0, 0);
1673 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1674 if(fccs!=newfccs and not skipchecksum and not listonly):
1675 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1676 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1677 return False;
1678 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1679 pass;
1680 else:
1681 fcontents.seek(0, 0);
1682 if(uncompress):
1683 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1684 fcontents.seek(0, 0);
1685 fcontentend = fp.tell() - 1;
1686 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1687 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1688 if(abs(fseeknextasnum)==0):
1689 pass;
1690 fp.seek(fseeknextasnum, 1);
1691 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1692 fseeknextasnum = int(fseeknextfile);
1693 if(abs(fseeknextasnum)==0):
1694 pass;
1695 fp.seek(fseeknextasnum, 1);
1696 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1697 fseeknextasnum = int(fseeknextfile);
1698 if(abs(fseeknextasnum)==0):
1699 pass;
1700 fp.seek(fseeknextasnum, 0);
1701 else:
1702 return False;
1703 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1704 return catlist;
1705
1706 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1707 formatspecs = FormatSpecsListToDict(formatspecs);
1708 delimiter = formatspecs['format_delimiter'];
1709 curloc = fp.tell();
1710 if(curloc>0):
1711 fp.seek(0, 0);
1712 catheader = ReadFileHeaderData(fp, 4, delimiter);
1713 if(curloc>0):
1714 fp.seek(curloc, 0);
1715 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1716 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1717 if(not headercheck and not skipchecksum):
1718 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1719 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1720 return False;
1721 fnumfiles = int(catheader[1], 16);
1722 countnum = 0;
1723 flist = [];
1724 while(countnum < fnumfiles):
1725 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1726 if(len(HeaderOut)==0):
1727 break;
1728 flist.append(HeaderOut);
1729 countnum = countnum + 1;
1730 return flist;
1731
1732 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1733 formatspecs = FormatSpecsListToDict(formatspecs);
1734 delimiter = formatspecs['format_delimiter'];
1735 curloc = fp.tell();
1736 if(curloc>0):
1737 fp.seek(0, 0);
1738 catheader = ReadFileHeaderData(fp, 4, delimiter);
1739 if(curloc>0):
1740 fp.seek(curloc, 0);
1741 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1742 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1743 if(not headercheck and not skipchecksum):
1744 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1745 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1746 return False;
1747 catstring = catheader[0];
1748 catversion = re.findall(r"([\d]+)", catstring);
1749 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1750 fprenumfiles = catheader[1];
1751 fnumfiles = int(fprenumfiles, 16);
1752 fprechecksumtype = catheader[2];
1753 fprechecksum = catheader[3];
1754 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1755 if(seekstart<0 and seekstart>fnumfiles):
1756 seekstart = 0;
1757 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1758 seekend = fnumfiles;
1759 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1760 seekend = fnumfiles - abs(seekend);
1761 if(seekstart>0):
1762 il = 0;
1763 while(il < seekstart):
1764 prefhstart = fp.tell();
1765 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1766 if(len(preheaderdata)==0):
1767 break;
1768 prefsize = int(preheaderdata[5], 16);
1769 prefseeknextfile = preheaderdata[25];
1770 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1771 prefcs = preheaderdata[-2];
1772 if(prefcs!=prenewfcs and not skipchecksum):
1773 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1774 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1775 return False;
1776 valid_archive = False;
1777 invalid_archive = True;
1778 prefhend = fp.tell() - 1;
1779 prefcontentstart = fp.tell();
1780 prefcontents = BytesIO();
1781 pyhascontents = False;
1782 if(prefsize>0):
1783 prefcontents.write(fp.read(prefsize));
1784 prefcontents.seek(0, 0);
1785 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1786 prefccs = preheaderdata[-1];
1787 pyhascontents = True;
1788 if(prefccs!=prenewfccs and not skipchecksum):
1789 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1790 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1791 return False;
1792 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1793 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1794 if(abs(fseeknextasnum)==0):
1795 pass;
1796 fp.seek(fseeknextasnum, 1);
1797 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1798 fseeknextasnum = int(prefseeknextfile);
1799 if(abs(fseeknextasnum)==0):
1800 pass;
1801 fp.seek(fseeknextasnum, 1);
1802 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1803 fseeknextasnum = int(prefseeknextfile);
1804 if(abs(fseeknextasnum)==0):
1805 pass;
1806 fp.seek(fseeknextasnum, 0);
1807 else:
1808 return False;
1809 il = il + 1;
1810 realidnum = 0;
1811 countnum = seekstart;
1812 while(countnum < seekend):
1813 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1814 if(len(HeaderOut)==0):
1815 break;
1816 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1817 catlist['ffilelist'].append(HeaderOut);
1818 countnum = countnum + 1;
1819 realidnum = realidnum + 1;
1820 return catlist;
1821
1822 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1823 formatspecs = FormatSpecsListToDict(formatspecs);
1824 delimiter = formatspecs['format_delimiter'];
1825 curloc = fp.tell();
1826 if(curloc>0):
1827 fp.seek(0, 0);
1828 catheader = ReadFileHeaderData(fp, 4, delimiter);
1829 if(curloc>0):
1830 fp.seek(curloc, 0);
1831 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1832 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1833 if(not headercheck and not skipchecksum):
1834 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1835 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1836 return False;
1837 catstring = catheader[0];
1838 catversion = re.findall(r"([\d]+)", catstring);
1839 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1840 fprenumfiles = catheader[1];
1841 fnumfiles = int(fprenumfiles, 16);
1842 fprechecksumtype = catheader[2];
1843 fprechecksum = catheader[3];
1844 catlist = [];
1845 if(seekstart<0 and seekstart>fnumfiles):
1846 seekstart = 0;
1847 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1848 seekend = fnumfiles;
1849 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1850 seekend = fnumfiles - abs(seekend);
1851 if(seekstart>0):
1852 il = 0;
1853 while(il < seekstart):
1854 prefhstart = fp.tell();
1855 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1856 if(len(preheaderdata)==0):
1857 break;
1858 prefsize = int(preheaderdata[5], 16);
1859 prefcompression = preheaderdata[12];
1860 prefcsize = int(preheaderdata[13], 16);
1861 prefseeknextfile = HeaderOut[25];
1862 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1863 prefcs = preheaderdata[-2];
1864 if(prefcs!=prenewfcs and not skipchecksum):
1865 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1866 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1867 return False;
1868 valid_archive = False;
1869 invalid_archive = True;
1870 prefhend = fp.tell() - 1;
1871 prefcontentstart = fp.tell();
1872 prefcontents = "";
1873 pyhascontents = False;
1874 if(prefsize>0):
1875 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1876 prefcontents = catfp.read(prefsize);
1877 else:
1878 prefcontents = catfp.read(prefcsize);
1879 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1880 prefccs = preheaderdata[-1];
1881 pyhascontents = True;
1882 if(prefccs!=prenewfccs and not skipchecksum):
1883 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1884 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1885 return False;
1886 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1887 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1888 if(abs(fseeknextasnum)==0):
1889 pass;
1890 catfp.seek(fseeknextasnum, 1);
1891 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1892 fseeknextasnum = int(prefseeknextfile);
1893 if(abs(fseeknextasnum)==0):
1894 pass;
1895 catfp.seek(fseeknextasnum, 1);
1896 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1897 fseeknextasnum = int(prefseeknextfile);
1898 if(abs(fseeknextasnum)==0):
1899 pass;
1900 catfp.seek(fseeknextasnum, 0);
1901 else:
1902 return False;
1903 il = il + 1;
1904 realidnum = 0;
1905 countnum = seekstart;
1906 while(countnum < seekend):
1907 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1908 if(len(HeaderOut)==0):
1909 break;
1910 catlist.append(HeaderOut);
1911 countnum = countnum + 1;
1912 realidnum = realidnum + 1;
1913 return catlist;
1914
1915 def ReadInFileBySizeWithContentToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1916 formatspecs = FormatSpecsListToDict(formatspecs);
1917 delimiter = formatspecs['format_delimiter'];
1918 if(hasattr(infile, "read") or hasattr(infile, "write")):
1919 fp = infile;
1920 fp.seek(0, 0);
1921 fp = UncompressArchiveFile(fp, formatspecs);
1922 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1923 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1924 return False;
1925 if(not fp):
1926 return False;
1927 fp.seek(0, 0);
1928 elif(infile=="-"):
1929 fp = BytesIO();
1930 if(hasattr(sys.stdin, "buffer")):
1931 shutil.copyfileobj(sys.stdin.buffer, fp);
1932 else:
1933 shutil.copyfileobj(sys.stdin, fp);
1934 fp.seek(0, 0);
1935 fp = UncompressArchiveFile(fp, formatspecs);
1936 if(not fp):
1937 return False;
1938 fp.seek(0, 0);
1939 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1940 fp = download_file_from_internet_file(infile);
1941 fp = UncompressArchiveFile(fp, formatspecs);
1942 fp.seek(0, 0);
1943 if(not fp):
1944 return False;
1945 fp.seek(0, 0);
1946 else:
1947 infile = RemoveWindowsPath(infile);
1948 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1949 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1950 return False;
1951 compresscheck = CheckCompressionType(infile, formatspecs, True);
1952 if(not compresscheck):
1953 fextname = os.path.splitext(infile)[1];
1954 if(fextname==".gz"):
1955 compresscheck = "gzip";
1956 elif(fextname==".bz2"):
1957 compresscheck = "bzip2";
1958 elif(fextname==".zst"):
1959 compresscheck = "zstd";
1960 elif(fextname==".lz4" or fextname==".clz4"):
1961 compresscheck = "lz4";
1962 elif(fextname==".lzo" or fextname==".lzop"):
1963 compresscheck = "lzo";
1964 elif(fextname==".lzma"):
1965 compresscheck = "lzma";
1966 elif(fextname==".xz"):
1967 compresscheck = "xz";
1968 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
1969 compresscheck = "zlib";
1970 else:
1971 return False;
1972 if(not compresscheck):
1973 return False;
1974 fp = UncompressFile(infile, formatspecs, "rb");
1975 return ReadFileDataBySizeWithContentToArray(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1976
1977 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1978 formatspecs = FormatSpecsListToDict(formatspecs);
1979 delimiter = formatspecs['format_delimiter'];
1980 if(hasattr(infile, "read") or hasattr(infile, "write")):
1981 fp = infile;
1982 fp.seek(0, 0);
1983 fp = UncompressArchiveFile(fp, formatspecs);
1984 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1985 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1986 return False;
1987 if(not fp):
1988 return False;
1989 fp.seek(0, 0);
1990 elif(infile=="-"):
1991 fp = BytesIO();
1992 if(hasattr(sys.stdin, "buffer")):
1993 shutil.copyfileobj(sys.stdin.buffer, fp);
1994 else:
1995 shutil.copyfileobj(sys.stdin, fp);
1996 fp.seek(0, 0);
1997 fp = UncompressArchiveFile(fp, formatspecs);
1998 if(not fp):
1999 return False;
2000 fp.seek(0, 0);
2001 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2002 fp = download_file_from_internet_file(infile);
2003 fp = UncompressArchiveFile(fp, formatspecs);
2004 fp.seek(0, 0);
2005 if(not fp):
2006 return False;
2007 fp.seek(0, 0);
2008 else:
2009 infile = RemoveWindowsPath(infile);
2010 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
2011 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
2012 return False;
2013 compresscheck = CheckCompressionType(infile, formatspecs, True);
2014 if(not compresscheck):
2015 fextname = os.path.splitext(infile)[1];
2016 if(fextname==".gz"):
2017 compresscheck = "gzip";
2018 elif(fextname==".bz2"):
2019 compresscheck = "bzip2";
2020 elif(fextname==".zst"):
2021 compresscheck = "zstd";
2022 elif(fextname==".lz4" or fextname==".clz4"):
2023 compresscheck = "lz4";
2024 elif(fextname==".lzo" or fextname==".lzop"):
2025 compresscheck = "lzo";
2026 elif(fextname==".lzma"):
2027 compresscheck = "lzma";
2028 elif(fextname==".xz"):
2029 compresscheck = "xz";
2030 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
2031 compresscheck = "zlib";
2032 else:
2033 return False;
2034 if(not compresscheck):
2035 return False;
2036 fp = UncompressFile(infile, formatspecs, "rb");
2037 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
2038
2039 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
2040 outdata = str(indata) + delimiter;
2041 return outdata;
2042
2043 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
2044 outdata = "";
2045 inum = 0;
2046 il = len(indata);
2047 while(inum < il):
2048 outdata = outdata + AppendNullByte(indata[inum], delimiter);
2049 inum = inum + 1;
2050 return outdata;
2051
2052 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
2053 formatspecs = FormatSpecsListToDict(formatspecs);
2054 delimiter = formatspecs['format_delimiter'];
2055 catver = formatspecs['format_ver'];
2056 fileheaderver = str(int(catver.replace(".", "")));
2057 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
2058 fp.write(fileheader.encode('UTF-8'));
2059 fnumfiles = format(int(numfiles), 'x').lower();
2060 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
2061 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2062 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
2063 fp.write(fnumfilesa.encode('UTF-8'));
2064 try:
2065 fp.flush();
2066 os.fsync(fp.fileno());
2067 except io.UnsupportedOperation:
2068 pass;
2069 except AttributeError:
2070 pass;
2071 except OSError as e:
2072 pass;
2073 return fp;
2074
2075 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
2076 formatspecs = FormatSpecsListToDict(formatspecs);
2077 AppendFileHeader(fp, 0, checksumtype, formatspecs);
2078 return fp;
2079
2080 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
2081 formatspecs = FormatSpecsListToDict(formatspecs);
2082 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2083 if(os.path.exists(outfile)):
2084 try:
2085 os.unlink(outfile);
2086 except OSError as e:
2087 pass;
2088 if(outfile=="-"):
2089 verbose = False;
2090 catfpfp = BytesIO();
2091 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2092 catfp = outfile;
2093 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2094 catfp = BytesIO();
2095 else:
2096 fbasename = os.path.splitext(outfile)[0];
2097 fextname = os.path.splitext(outfile)[1];
2098 if(not compresswholefile and fextname in outextlistwd):
2099 compresswholefile = True;
2100 catfp = CompressOpenFile(outfile, True, compressionlevel);
2101 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
2102 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2103 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2104 try:
2105 catfp.flush();
2106 os.fsync(catfp.fileno());
2107 except io.UnsupportedOperation:
2108 pass;
2109 except AttributeError:
2110 pass;
2111 except OSError as e:
2112 pass;
2113 if(outfile=="-"):
2114 catfp.seek(0, 0);
2115 if(hasattr(sys.stdout, "buffer")):
2116 shutil.copyfileobj(catfp, sys.stdout.buffer);
2117 else:
2118 shutil.copyfileobj(catfp, sys.stdout);
2119 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2120 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2121 catfp.seek(0, 0);
2122 upload_file_to_internet_file(catfp, outfile);
2123 if(returnfp):
2124 catfp.seek(0, 0);
2125 return catfp;
2126 else:
2127 catfp.close();
2128 return True;
2129
2130 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
2131 formatspecs = FormatSpecsListToDict(formatspecs);
2132 extrafields = format(len(extradata), 'x').lower();
2133 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
2134 if(len(extradata)>0):
2135 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
2136 extrasizelen = format(len(extrasizestr), 'x').lower();
2137 catoutlen = len(filevalues) + len(extradata) + 5;
2138 catoutlenhex = format(catoutlen, 'x').lower();
2139 catoutlist = filevalues;
2140 catoutlist.insert(0, catoutlenhex);
2141 catoutlist.append(extrasizelen);
2142 catoutlist.append(extrafields);
2143 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
2144 if(len(extradata)>0):
2145 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
2146 if(len(filecontent)==0):
2147 checksumlist = [checksumtype, "none"];
2148 else:
2149 checksumlist = [checksumtype, checksumtype];
2150 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
2151 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2152 if(len(filecontent)==0):
2153 catfilecontentcshex = GetFileChecksum(filecontent, "none", False, formatspecs);
2154 else:
2155 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
2156 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
2157 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2158 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
2159 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2160 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
2161 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2162 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
2163 catfileout = catfileoutstrecd + filecontent + nullstrecd;
2164 fp.write(catfileout);
2165 try:
2166 fp.flush();
2167 os.fsync(fp.fileno());
2168 except io.UnsupportedOperation:
2169 pass;
2170 except AttributeError:
2171 pass;
2172 except OSError as e:
2173 pass;
2174 return fp;
2175
2176 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2177 formatspecs = FormatSpecsListToDict(formatspecs);
2178 advancedlist = formatspecs['use_advanced_list'];
2179 altinode = formatspecs['use_alt_inode'];
2180 if(verbose):
2181 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2182 if(infiles=="-"):
2183 for line in sys.stdin:
2184 infilelist.append(line.strip());
2185 infilelist = list(filter(None, infilelist));
2186 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2187 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2188 return False;
2189 with UncompressFile(infiles, formatspecs, "r") as finfile:
2190 for line in finfile:
2191 infilelist.append(line.strip());
2192 infilelist = list(filter(None, infilelist));
2193 else:
2194 if(isinstance(infiles, (list, tuple, ))):
2195 infilelist = list(filter(None, infiles));
2196 elif(isinstance(infiles, (str, ))):
2197 infilelist = list(filter(None, [infiles]));
2198 if(advancedlist):
2199 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2200 else:
2201 GetDirList = ListDir(infilelist, followlink, False);
2202 if(not GetDirList):
2203 return False;
2204 curinode = 0;
2205 curfid = 0;
2206 inodelist = [];
2207 inodetofile = {};
2208 filetoinode = {};
2209 inodetocatinode = {};
2210 numfiles = int(len(GetDirList));
2211 fnumfiles = format(numfiles, 'x').lower();
2212 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
2213 for curfname in GetDirList:
2214 if(re.findall(r"^[.|/]", curfname)):
2215 fname = curfname;
2216 else:
2217 fname = "./"+curfname;
2218 if(verbose):
2219 VerbosePrintOut(fname);
2220 if(not followlink or followlink is None):
2221 fstatinfo = os.lstat(fname);
2222 else:
2223 fstatinfo = os.stat(fname);
2224 fpremode = fstatinfo.st_mode;
2225 finode = fstatinfo.st_ino;
2226 flinkcount = fstatinfo.st_nlink;
2227 ftype = 0;
2228 if(stat.S_ISREG(fpremode)):
2229 ftype = 0;
2230 elif(stat.S_ISLNK(fpremode)):
2231 ftype = 2;
2232 elif(stat.S_ISCHR(fpremode)):
2233 ftype = 3;
2234 elif(stat.S_ISBLK(fpremode)):
2235 ftype = 4;
2236 elif(stat.S_ISDIR(fpremode)):
2237 ftype = 5;
2238 elif(stat.S_ISFIFO(fpremode)):
2239 ftype = 6;
2240 elif(stat.S_ISSOCK(fpremode)):
2241 ftype = 8;
2242 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2243 ftype = 9;
2244 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2245 ftype = 10;
2246 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2247 ftype = 11;
2248 else:
2249 ftype = 0;
2250 flinkname = "";
2251 fcurfid = format(int(curfid), 'x').lower();
2252 if(not followlink and finode!=0):
2253 if(ftype!=1):
2254 if(finode in inodelist):
2255 ftype = 1;
2256 flinkname = inodetofile[finode];
2257 if(altinode):
2258 fcurinode = format(int(finode), 'x').lower();
2259 else:
2260 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2261 if(finode not in inodelist):
2262 inodelist.append(finode);
2263 inodetofile.update({finode: fname});
2264 inodetocatinode.update({finode: curinode});
2265 if(altinode):
2266 fcurinode = format(int(finode), 'x').lower();
2267 else:
2268 fcurinode = format(int(curinode), 'x').lower();
2269 curinode = curinode + 1;
2270 else:
2271 fcurinode = format(int(curinode), 'x').lower();
2272 curinode = curinode + 1;
2273 curfid = curfid + 1;
2274 if(ftype==2):
2275 flinkname = os.readlink(fname);
2276 fdev = fstatinfo.st_dev;
2277 getfdev = GetDevMajorMinor(fdev);
2278 fdev_minor = getfdev[0];
2279 fdev_major = getfdev[1];
2280 frdev = fstatinfo.st_dev;
2281 if(hasattr(fstatinfo, "st_rdev")):
2282 frdev = fstatinfo.st_rdev;
2283 else:
2284 frdev = fstatinfo.st_dev;
2285 getfrdev = GetDevMajorMinor(frdev);
2286 frdev_minor = getfrdev[0];
2287 frdev_major = getfrdev[1];
2288 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2289 fsize = format(int("0"), 'x').lower();
2290 elif(ftype==0 or ftype==7):
2291 fsize = format(int(fstatinfo.st_size), 'x').lower();
2292 else:
2293 fsize = format(int(fstatinfo.st_size)).lower();
2294 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2295 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2296 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2297 if(hasattr(fstatinfo, "st_birthtime")):
2298 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2299 else:
2300 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2301 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2302 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2303 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2304 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2305 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2306 funame = "";
2307 try:
2308 import pwd;
2309 try:
2310 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2311 funame = userinfo.pw_name;
2312 except KeyError:
2313 funame = "";
2314 except ImportError:
2315 funame = "";
2316 fgname = "";
2317 try:
2318 import grp;
2319 try:
2320 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2321 fgname = groupinfo.gr_name;
2322 except KeyError:
2323 fgname = "";
2324 except ImportError:
2325 fgname = "";
2326 fdev_minor = format(int(fdev_minor), 'x').lower();
2327 fdev_major = format(int(fdev_major), 'x').lower();
2328 frdev_minor = format(int(frdev_minor), 'x').lower();
2329 frdev_major = format(int(frdev_major), 'x').lower();
2330 finode = format(int(finode), 'x').lower();
2331 flinkcount = format(int(flinkcount), 'x').lower();
2332 if(hasattr(fstatinfo, "st_file_attributes")):
2333 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2334 else:
2335 fwinattributes = format(int(0), 'x').lower();
2336 fcompression = "";
2337 fcsize = format(int(0), 'x').lower();
2338 fcontents = BytesIO();
2339 chunk_size = 1024;
2340 if(ftype==0 or ftype==7):
2341 with open(fname, "rb") as fpc:
2342 shutil.copyfileobj(fpc, fcontents);
2343 if(not compresswholefile):
2344 fcontents.seek(0, 2);
2345 ucfsize = fcontents.tell();
2346 fcontents.seek(0, 0);
2347 if(compression=="auto"):
2348 ilsize = len(compressionlistalt);
2349 ilmin = 0;
2350 ilcsize = [];
2351 while(ilmin < ilsize):
2352 cfcontents = BytesIO();
2353 shutil.copyfileobj(fcontents, cfcontents);
2354 fcontents.seek(0, 0);
2355 cfcontents.seek(0, 0);
2356 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2357 if(cfcontents):
2358 cfcontents.seek(0, 2);
2359 ilcsize.append(cfcontents.tell());
2360 cfcontents.close();
2361 else:
2362 try:
2363 ilcsize.append(sys.maxint);
2364 except AttributeError:
2365 ilcsize.append(sys.maxsize);
2366 ilmin = ilmin + 1;
2367 ilcmin = ilcsize.index(min(ilcsize));
2368 compression = compressionlistalt[ilcmin];
2369 fcontents.seek(0, 0);
2370 cfcontents = BytesIO();
2371 shutil.copyfileobj(fcontents, cfcontents);
2372 cfcontents.seek(0, 0);
2373 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2374 cfcontents.seek(0, 2);
2375 cfsize = cfcontents.tell();
2376 if(ucfsize > cfsize):
2377 fcsize = format(int(cfsize), 'x').lower();
2378 fcompression = compression;
2379 fcontents.close();
2380 fcontents = cfcontents;
2381 if(followlink and (ftype==1 or ftype==2)):
2382 flstatinfo = os.stat(flinkname);
2383 with open(flinkname, "rb") as fpc:
2384 shutil.copyfileobj(fpc, fcontents);
2385 if(not compresswholefile):
2386 fcontents.seek(0, 2);
2387 ucfsize = fcontents.tell();
2388 fcontents.seek(0, 0);
2389 if(compression=="auto"):
2390 ilsize = len(compressionlistalt);
2391 ilmin = 0;
2392 ilcsize = [];
2393 while(ilmin < ilsize):
2394 cfcontents = BytesIO();
2395 shutil.copyfileobj(fcontents, cfcontents);
2396 fcontents.seek(0, 0);
2397 cfcontents.seek(0, 0);
2398 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2399 if(cfcontents):
2400 cfcontents.seek(0, 2);
2401 ilcsize.append(cfcontents.tell());
2402 cfcontents.close();
2403 else:
2404 try:
2405 ilcsize.append(sys.maxint);
2406 except AttributeError:
2407 ilcsize.append(sys.maxsize);
2408 ilmin = ilmin + 1;
2409 ilcmin = ilcsize.index(min(ilcsize));
2410 compression = compressionlistalt[ilcmin];
2411 fcontents.seek(0, 0);
2412 cfcontents = BytesIO();
2413 shutil.copyfileobj(fcontents, cfcontents);
2414 cfcontents.seek(0, 0);
2415 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2416 cfcontents.seek(0, 2);
2417 cfsize = cfcontents.tell();
2418 if(ucfsize > cfsize):
2419 fcsize = format(int(cfsize), 'x').lower();
2420 fcompression = compression;
2421 fcontents.close();
2422 fcontents = cfcontents;
2423 if(fcompression=="none"):
2424 fcompression = "";
2425 fcontents.seek(0, 0);
2426 ftypehex = format(ftype, 'x').lower();
2427 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
2428 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2429 if(numfiles>0):
2430 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2431 fp.seek(0, 0);
2432 return fp;
2433
2434 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2435 formatspecs = FormatSpecsListToDict(formatspecs);
2436 if(verbose):
2437 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2438 GetDirList = inlist;
2439 if(not GetDirList):
2440 return False;
2441 curinode = 0;
2442 curfid = 0;
2443 inodelist = [];
2444 inodetofile = {};
2445 filetoinode = {};
2446 inodetocatinode = {};
2447 numfiles = int(len(GetDirList));
2448 fnumfiles = format(numfiles, 'x').lower();
2449 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
2450 for curfname in GetDirList:
2451 ftype = format(curfname[0], 'x').lower();
2452 if(re.findall(r"^[.|/]", curfname[1])):
2453 fname = curfname[1];
2454 else:
2455 fname = "./"+curfname[1];
2456 fbasedir = os.path.dirname(fname);
2457 flinkname = curfname[2];
2458 fsize = format(curfname[3], 'x').lower();
2459 fatime = format(curfname[4], 'x').lower();
2460 fmtime = format(curfname[5], 'x').lower();
2461 fctime = format(curfname[6], 'x').lower();
2462 fbtime = format(curfname[7], 'x').lower();
2463 fmode = format(curfname[8], 'x').lower();
2464 fwinattributes = format(curfname[9], 'x').lower();
2465 fcompression = curfname[10];
2466 fcsize = format(curfname[11], 'x').lower();
2467 fuid = format(curfname[12], 'x').lower();
2468 funame = curfname[13];
2469 fgid = format(curfname[14], 'x').lower();
2470 fgname = curfname[15];
2471 fid = format(curfname[16], 'x').lower();
2472 finode = format(curfname[17], 'x').lower();
2473 flinkcount = format(curfname[18], 'x').lower();
2474 fdev_minor = format(curfname[19], 'x').lower();
2475 fdev_major = format(curfname[20], 'x').lower();
2476 frdev_minor = format(curfname[21], 'x').lower();
2477 frdev_major = format(curfname[22], 'x').lower();
2478 fseeknextfile = curfname[23];
2479 extradata = curfname[24];
2480 fheaderchecksumtype = curfname[25];
2481 fcontentchecksumtype = curfname[26];
2482 fcontents = curfname[27];
2483 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
2484 fcontents.seek(0, 0);
2485 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2486 if(numfiles>0):
2487 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2488 return fp;
2489
2490 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2491 formatspecs = FormatSpecsListToDict(formatspecs);
2492 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2493 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
2494
2495 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2496 formatspecs = FormatSpecsListToDict(formatspecs);
2497 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2498 if(os.path.exists(outfile)):
2499 try:
2500 os.unlink(outfile);
2501 except OSError as e:
2502 pass;
2503 if(outfile=="-"):
2504 verbose = False;
2505 catfpfp = BytesIO();
2506 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2507 catfp = outfile;
2508 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2509 catfp = BytesIO();
2510 else:
2511 fbasename = os.path.splitext(outfile)[0];
2512 fextname = os.path.splitext(outfile)[1];
2513 if(not compresswholefile and fextname in outextlistwd):
2514 compresswholefile = True;
2515 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2516 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2517 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2518 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2519 try:
2520 catfp.flush();
2521 os.fsync(catfp.fileno());
2522 except io.UnsupportedOperation:
2523 pass;
2524 except AttributeError:
2525 pass;
2526 except OSError as e:
2527 pass;
2528 if(outfile=="-"):
2529 catfp.seek(0, 0);
2530 if(hasattr(sys.stdout, "buffer")):
2531 shutil.copyfileobj(catfp, sys.stdout.buffer);
2532 else:
2533 shutil.copyfileobj(catfp, sys.stdout);
2534 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2535 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2536 catfp.seek(0, 0);
2537 upload_file_to_internet_file(catfp, outfile);
2538 if(returnfp):
2539 catfp.seek(0, 0);
2540 return catfp;
2541 else:
2542 catfp.close();
2543 return True;
2544
2545 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2546 formatspecs = FormatSpecsListToDict(formatspecs);
2547 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2548 if(os.path.exists(outfile)):
2549 try:
2550 os.unlink(outfile);
2551 except OSError as e:
2552 pass;
2553 if(outfile=="-"):
2554 verbose = False;
2555 catfpfp = BytesIO();
2556 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2557 catfp = outfile;
2558 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2559 catfp = BytesIO();
2560 else:
2561 fbasename = os.path.splitext(outfile)[0];
2562 fextname = os.path.splitext(outfile)[1];
2563 if(not compresswholefile and fextname in outextlistwd):
2564 compresswholefile = True;
2565 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2566 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2567 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2568 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2569 try:
2570 catfp.flush();
2571 os.fsync(catfp.fileno());
2572 except io.UnsupportedOperation:
2573 pass;
2574 except AttributeError:
2575 pass;
2576 except OSError as e:
2577 pass;
2578 if(outfile=="-"):
2579 catfp.seek(0, 0);
2580 if(hasattr(sys.stdout, "buffer")):
2581 shutil.copyfileobj(catfp, sys.stdout.buffer);
2582 else:
2583 shutil.copyfileobj(catfp, sys.stdout);
2584 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2585 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2586 catfp.seek(0, 0);
2587 upload_file_to_internet_file(catfp, outfile);
2588 if(returnfp):
2589 catfp.seek(0, 0);
2590 return catfp;
2591 else:
2592 catfp.close();
2593 return True;
2594
2595 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2596 formatspecs = FormatSpecsListToDict(formatspecs);
2597 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2598 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2599
2600 def PrintPermissionString(fchmode, ftype):
2601 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2602 permissionstr = "";
2603 for fmodval in str(oct(fchmode))[-3:]:
2604 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2605 if(ftype==0 or ftype==7):
2606 permissionstr = "-" + permissionstr;
2607 if(ftype==1):
2608 permissionstr = "h" + permissionstr;
2609 if(ftype==2):
2610 permissionstr = "l" + permissionstr;
2611 if(ftype==3):
2612 permissionstr = "c" + permissionstr;
2613 if(ftype==4):
2614 permissionstr = "b" + permissionstr;
2615 if(ftype==5):
2616 permissionstr = "d" + permissionstr;
2617 if(ftype==6):
2618 permissionstr = "f" + permissionstr;
2619 if(ftype==8):
2620 permissionstr = "D" + permissionstr;
2621 if(ftype==9):
2622 permissionstr = "p" + permissionstr;
2623 if(ftype==10):
2624 permissionstr = "w" + permissionstr;
2625 try:
2626 permissionoutstr = stat.filemode(fchmode);
2627 except AttributeError:
2628 permissionoutstr = permissionstr;
2629 except KeyError:
2630 permissionoutstr = permissionstr;
2631 return permissionoutstr;
2632
2633 def PrintPermissionStringAlt(fchmode, ftype):
2634 permissions = {
2635 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2636 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2637 };
2638 # Translate file mode into permission string
2639 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2640 # Append file type indicator
2641 type_indicators = {
2642 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2643 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2644 };
2645 file_type = type_indicators.get(ftype, '-');
2646 permissionstr = file_type + permissionstr;
2647 try:
2648 permissionoutstr = stat.filemode(fchmode);
2649 except AttributeError:
2650 permissionoutstr = permissionstr;
2651 return permissionoutstr;
2652
2653 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2654 formatspecs = FormatSpecsListToDict(formatspecs);
2655 if(hasattr(infile, "read") or hasattr(infile, "write")):
2656 catfp = infile;
2657 else:
2658 try:
2659 catfp = open(infile, "rb");
2660 except FileNotFoundError:
2661 return False;
2662 filetype = False;
2663 catfp.seek(0, 0);
2664 prefp = catfp.read(2);
2665 if(prefp==binascii.unhexlify("1f8b")):
2666 filetype = "gzip";
2667 if(prefp==binascii.unhexlify("7801")):
2668 filetype = "zlib";
2669 if(prefp==binascii.unhexlify("785e")):
2670 filetype = "zlib";
2671 if(prefp==binascii.unhexlify("789c")):
2672 filetype = "zlib";
2673 if(prefp==binascii.unhexlify("78da")):
2674 filetype = "zlib";
2675 catfp.seek(0, 0);
2676 prefp = catfp.read(3);
2677 if(prefp==binascii.unhexlify("425a68")):
2678 filetype = "bzip2";
2679 if(prefp==binascii.unhexlify("5d0000")):
2680 filetype = "lzma";
2681 catfp.seek(0, 0);
2682 prefp = catfp.read(4);
2683 if(prefp==binascii.unhexlify("28b52ffd")):
2684 filetype = "zstd";
2685 if(prefp==binascii.unhexlify("04224d18")):
2686 filetype = "lz4";
2687 if(prefp==binascii.unhexlify("504B0304")):
2688 filetype = "zipfile";
2689 catfp.seek(0, 0);
2690 prefp = catfp.read(5);
2691 if(prefp==binascii.unhexlify("7573746172")):
2692 filetype = "tarfile";
2693 catfp.seek(0, 0);
2694 prefp = catfp.read(6);
2695 if(prefp==binascii.unhexlify("fd377a585a00")):
2696 filetype = "lzma";
2697 if(prefp==binascii.unhexlify("377abcaf271c")):
2698 filetype = "7zipfile";
2699 catfp.seek(0, 0);
2700 prefp = catfp.read(7);
2701 if(prefp==binascii.unhexlify("526172211a0700")):
2702 filetype = "rarfile";
2703 if(prefp==binascii.unhexlify("43617446696c65")):
2704 filetype = "catfile";
2705 catfp.seek(0, 0);
2706 prefp = catfp.read(8);
2707 if(prefp==binascii.unhexlify("526172211a070100")):
2708 filetype = "rarfile";
2709 catfp.seek(0, 0);
2710 prefp = catfp.read(formatspecs['format_len']);
2711 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2712 filetype = formatspecs['format_lower'];
2713 catfp.seek(0, 0);
2714 prefp = catfp.read(9);
2715 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2716 filetype = "lzo";
2717 catfp.seek(0, 0);
2718 prefp = catfp.read(10);
2719 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2720 filetype = "tarfile";
2721 catfp.seek(0, 0);
2722 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4" or filetype=="zlib"):
2723 if(TarFileCheck(catfp)):
2724 filetype = "tarfile";
2725 if(not filetype):
2726 if(TarFileCheck(catfp)):
2727 filetype = "tarfile";
2728 elif(zipfile.is_zipfile(catfp)):
2729 filetype = "zipfile";
2730 elif(rarfile_support and (rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp))):
2731 filetype = "rarile";
2732 elif(py7zr_support and py7zr.is_7zfile(catfp)):
2733 return "7zipfile";
2734 else:
2735 filetype = False;
2736 catfp.seek(0, 0);
2737 if(closefp):
2738 catfp.close();
2739 return filetype;
2740
2741 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2742 formatspecs = FormatSpecsListToDict(formatspecs);
2743 try:
2744 instringsfile = BytesIO(instring);
2745 except TypeError:
2746 instringsfile = BytesIO(instring.encode("UTF-8"));
2747 return CheckCompressionType(instringsfile, formatspecs, closefp);
2748
2749 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2750 formatspecs = FormatSpecsListToDict(formatspecs);
2751 compresscheck = CheckCompressionType(fp, formatspecs, False);
2752 if(compresscheck=="gzip" or compresscheck=="gz"):
2753 return archivefile_gzip_mimetype;
2754 if(compresscheck=="zlib" or (compresscheck=="zz" or compresscheck=="zl" or compresscheck=="zlib")):
2755 return archivefile_zlib_mimetype;
2756 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2757 return archivefile_bzip2_mimetype;
2758 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2759 return archivefile_zstandard_mimetype;
2760 if(compresscheck=="lz4"):
2761 return archivefile_lz4_mimetype;
2762 if(compresscheck=="lzo" or compresscheck=="lzop"):
2763 return archivefile_lzop_mimetype;
2764 if(compresscheck=="lzma"):
2765 return archivefile_lzma_mimetype;
2766 if(compresscheck=="xz"):
2767 return archivefile_xz_mimetype;
2768 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2769 return archivefile_cat_mimetype;
2770 if(not compresscheck):
2771 return False;
2772 return False;
2773
2774 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2775 formatspecs = FormatSpecsListToDict(formatspecs);
2776 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2777 return False;
2778 compresscheck = CheckCompressionType(fp, formatspecs, False);
2779 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2780 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2781 elif(compresscheck=="bzip2" and compresscheck in compressionsupport):
2782 catfp = bz2.BZ2File(fp);
2783 elif(compresscheck=="zstd" and compresscheck in compressionsupport):
2784 catfp = zstd.ZstdDecompressor().stream_reader(fp);
2785 elif(compresscheck=="lz4" and compresscheck in compressionsupport):
2786 catfp = lz4.frame.open_fp(fp, mode='rb');
2787 elif((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2788 catfp = BytesIO();
2789 catfp.write(lzo.decompress(fp.read()));
2790 elif((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2791 catfp = lzma.LZMAFile(fp);
2792 elif(compresscheck=="zlib" and compresscheck in compressionsupport):
2793 catfp = ZlibFile(fileobj=fp, mode="rb");
2794 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2795 catfp = fp;
2796 if(not compresscheck):
2797 try:
2798 import lzma;
2799 except ImportError:
2800 try:
2801 from backports import lzma
2802 except ImportError:
2803 return False;
2804 catfp = BytesIO();
2805 with fp as fpcontent:
2806 try:
2807 catfp.write(lzma.decompress(fp.read()));
2808 except lzma.LZMAError:
2809 return False;
2810 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2811 fp.close();
2812 return catfp;
2813
2814 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2815
2816 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2817 formatspecs = FormatSpecsListToDict(formatspecs);
2818 compresscheck = CheckCompressionType(infile, formatspecs, False);
2819 if(sys.version_info[0]==2 and compresscheck):
2820 if(mode=="rt"):
2821 mode = "r";
2822 if(mode=="wt"):
2823 mode = "w";
2824 try:
2825 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2826 try:
2827 filefp = gzip.open(infile, mode, encoding="UTF-8");
2828 except (ValueError, TypeError) as e:
2829 filefp = gzip.open(infile, mode);
2830 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2831 try:
2832 filefp = bz2.open(infile, mode, encoding="UTF-8");
2833 except (ValueError, TypeError) as e:
2834 filefp = bz2.open(infile, mode);
2835 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2836 try:
2837 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2838 except (ValueError, TypeError) as e:
2839 filefp = zstandard.open(infile, mode);
2840 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2841 try:
2842 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2843 except (ValueError, TypeError) as e:
2844 filefp = lz4.frame.open(infile, mode);
2845 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2846 try:
2847 filefp = lzo.open(infile, mode, encoding="UTF-8");
2848 except (ValueError, TypeError) as e:
2849 filefp = lzo.open(infile, mode);
2850 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2851 try:
2852 filefp = lzma.open(infile, mode, encoding="UTF-8");
2853 except (ValueError, TypeError) as e:
2854 filefp = lzma.open(infile, mode);
2855 if(compresscheck=="zlib" and compresscheck in compressionsupport):
2856 filefp = ZlibFile(infile, mode=mode);
2857 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2858 try:
2859 filefp = open(infile, mode, encoding="UTF-8");
2860 except (ValueError, TypeError) as e:
2861 filefp = open(infile, mode);
2862 if(not compresscheck):
2863 try:
2864 filefp = open(infile, mode, encoding="UTF-8");
2865 except (ValueError, TypeError) as e:
2866 filefp = open(infile, mode);
2867 except FileNotFoundError:
2868 return False;
2869 try:
2870 filefp.write_through = True;
2871 except AttributeError:
2872 pass;
2873 return filefp;
2874
2875 def UncompressString(infile):
2876 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2877 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2878 fileuz = gzip.decompress(infile);
2879 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2880 fileuz = bz2.decompress(infile);
2881 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2882 try:
2883 import zstandard;
2884 except ImportError:
2885 return False;
2886 fileuz = zstandard.decompress(infile);
2887 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2888 fileuz = lz4.frame.decompress(infile);
2889 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2890 fileuz = lzo.decompress(infile);
2891 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2892 fileuz = lzma.decompress(infile);
2893 if(compresscheck=="zlib" and compresscheck in compressionsupport):
2894 fileuz = zlib.decompress(infile);
2895 if(not compresscheck):
2896 fileuz = infile;
2897 if(hasattr(fileuz, 'decode')):
2898 fileuz = fileuz.decode("UTF-8");
2899 return fileuz;
2900
2901 def UncompressStringAlt(infile):
2902 filefp = StringIO();
2903 outstring = UncompressString(infile);
2904 filefp.write(outstring);
2905 filefp.seek(0, 0);
2906 return filefp;
2907
2908 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2909 formatspecs = FormatSpecsListToDict(formatspecs);
2910 compresscheck = CheckCompressionType(infile, formatspecs, False);
2911 if(not compresscheck):
2912 fextname = os.path.splitext(infile)[1];
2913 if(fextname==".gz"):
2914 compresscheck = "gzip";
2915 elif(fextname==".bz2"):
2916 compresscheck = "bzip2";
2917 elif(fextname==".zst"):
2918 compresscheck = "zstd";
2919 elif(fextname==".lz4"):
2920 compresscheck = "lz4";
2921 elif(fextname==".lzo" or fextname==".lzop"):
2922 compresscheck = "lzo";
2923 elif(fextname==".lzma"):
2924 compresscheck = "lzma";
2925 elif(fextname==".xz"):
2926 compresscheck = "xz";
2927 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
2928 compresscheck = "zlib";
2929 else:
2930 return False;
2931 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4" or compresscheck=="zlib"):
2932 if(TarFileCheck(infile)):
2933 filetype = "tarfile";
2934 if(not compresscheck):
2935 if(TarFileCheck(infile)):
2936 return "tarfile";
2937 elif(zipfile.is_zipfile(infile)):
2938 return "zipfile";
2939 elif(rarfile_support and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
2940 return "rarile";
2941 elif(py7zr_support and py7zr.is_7zfile(infile)):
2942 return "7zipfile";
2943 else:
2944 return False;
2945 return False;
2946 if(compresscheck=="catfile"):
2947 return "catfile";
2948 if(compresscheck==formatspecs['format_lower']):
2949 return formatspecs['format_lower'];
2950 if(compresscheck=="tarfile"):
2951 return "tarfile";
2952 if(compresscheck=="zipfile"):
2953 return "zipfile";
2954 if(rarfile_support and compresscheck=="rarfile"):
2955 return "rarfile";
2956 if(py7zr_support and compresscheck=="7zipfile" and py7zr.is_7zfile(infile)):
2957 return "7zipfile";
2958 if(hasattr(infile, "read") or hasattr(infile, "write")):
2959 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2960 else:
2961 try:
2962 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2963 catfp = gzip.GzipFile(infile, "rb");
2964 elif(compresscheck=="bzip2" and compresscheck in compressionsupport):
2965 catfp = bz2.BZ2File(infile, "rb");
2966 elif(compresscheck=="lz4" and compresscheck in compressionsupport):
2967 catfp = lz4.frame.open(infile, "rb");
2968 elif(compresscheck=="zstd" and compresscheck in compressionsupport):
2969 catfp = zstandard.open(infile, "rb");
2970 elif((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2971 catfp = lzo.open(infile, "rb");
2972 elif((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2973 catfp = lzma.open(infile, "rb");
2974 elif(compresscheck=="zlib" and compresscheck in compressionsupport):
2975 catfp = ZlibFile(infile, mode="rb");
2976 else:
2977 catfp = open(infile, "rb");
2978 except FileNotFoundError:
2979 return False;
2980 filetype = False;
2981 prefp = catfp.read(5);
2982 if(prefp==binascii.unhexlify("7573746172")):
2983 filetype = "tarfile";
2984 catfp.seek(0, 0);
2985 prefp = catfp.read(7);
2986 if(prefp==binascii.unhexlify("43617446696c65")):
2987 filetype = "catfile";
2988 catfp.seek(0, 0);
2989 prefp = catfp.read(formatspecs['format_len']);
2990 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2991 filetype = formatspecs['format_lower'];
2992 catfp.seek(0, 0);
2993 prefp = catfp.read(10);
2994 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2995 filetype = "tarfile";
2996 catfp.seek(0, 0);
2997 if(closefp):
2998 catfp.close();
2999 return filetype;
3000
3001 def GZipCompress(data, compresslevel=9):
3002 if("gzip" not in compressionsupport):
3003 return False;
3004 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
3005 tmpfp.close();
3006 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
3007 tmpfp.write(data);
3008 tmpfp.close();
3009 try:
3010 catfp = open(tmpfp.name, "rb");
3011 except FileNotFoundError:
3012 return False;
3013 catdata = catfp.read();
3014 catfp.close();
3015 return catdata;
3016
3017 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
3018 formatspecs = FormatSpecsListToDict(formatspecs);
3019 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
3020 return False;
3021 fp.seek(0, 0);
3022 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3023 compression = "auto";
3024 if(compression not in compressionlist and compression is None):
3025 compression = "auto";
3026 if(compression=="gzip" and compression in compressionsupport):
3027 catfp = BytesIO();
3028 if(compressionlevel is None):
3029 compressionlevel = 9;
3030 else:
3031 compressionlevel = int(compressionlevel);
3032 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
3033 if(compression=="bzip2" and compression in compressionsupport):
3034 catfp = BytesIO();
3035 if(compressionlevel is None):
3036 compressionlevel = 9;
3037 else:
3038 compressionlevel = int(compressionlevel);
3039 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
3040 if(compression=="lz4" and compression in compressionsupport):
3041 catfp = BytesIO();
3042 if(compressionlevel is None):
3043 compressionlevel = 9;
3044 else:
3045 compressionlevel = int(compressionlevel);
3046 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
3047 if((compression=="lzo" or compression=="lzop") and compression in compressionsupport):
3048 catfp = BytesIO();
3049 if(compressionlevel is None):
3050 compressionlevel = 9;
3051 else:
3052 compressionlevel = int(compressionlevel);
3053 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
3054 if(compression=="zstd" and compression in compressionsupport):
3055 catfp = BytesIO();
3056 if(compressionlevel is None):
3057 compressionlevel = 10;
3058 else:
3059 compressionlevel = int(compressionlevel);
3060 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
3061 if(compression=="lzma" and compression in compressionsupport):
3062 catfp = BytesIO();
3063 if(compressionlevel is None):
3064 compressionlevel = 9;
3065 else:
3066 compressionlevel = int(compressionlevel);
3067 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
3068 if(compression=="xz" and compression in compressionsupport):
3069 catfp = BytesIO();
3070 if(compressionlevel is None):
3071 compressionlevel = 9;
3072 else:
3073 compressionlevel = int(compressionlevel);
3074 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
3075 if(compression=="zlib" and compression in compressionsupport):
3076 catfp = BytesIO();
3077 if(compressionlevel is None):
3078 compressionlevel = 9;
3079 else:
3080 compressionlevel = int(compressionlevel);
3081 catfp.write(zlib.compress(fp.read(), level=compressionlevel));
3082 if(compression=="auto" or compression is None):
3083 catfp = fp;
3084 catfp.seek(0, 0);
3085 return catfp;
3086
3087 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
3088
3089 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
3090 if(outfile is None):
3091 return False;
3092 fbasename = os.path.splitext(outfile)[0];
3093 fextname = os.path.splitext(outfile)[1];
3094 if(compressionlevel is None and fextname!=".zst"):
3095 compressionlevel = 9;
3096 elif(compressionlevel is None and fextname==".zst"):
3097 compressionlevel = 10;
3098 else:
3099 compressionlevel = int(compressionlevel);
3100 if(sys.version_info[0]==2):
3101 mode = "w";
3102 else:
3103 mode = "wb";
3104 try:
3105 if(fextname not in outextlistwd or not compressionenable):
3106 try:
3107 outfp = open(outfile, "wb", encoding="UTF-8");
3108 except (ValueError, TypeError) as e:
3109 outfp = open(outfile, "wb");
3110 elif(fextname==".gz" and "gzip" in compressionsupport):
3111 try:
3112 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
3113 except (ValueError, TypeError) as e:
3114 outfp = gzip.open(outfile, mode, compressionlevel);
3115 elif(fextname==".bz2" and "bzip2" in compressionsupport):
3116 try:
3117 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
3118 except (ValueError, TypeError) as e:
3119 outfp = bz2.open(outfile, mode, compressionlevel);
3120 elif(fextname==".zst" and "zstandard" in compressionsupport):
3121 try:
3122 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
3123 except (ValueError, TypeError) as e:
3124 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
3125 elif(fextname==".xz" and "xz" in compressionsupport):
3126 try:
3127 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
3128 except (ValueError, TypeError) as e:
3129 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
3130 elif(fextname==".lz4" and "lz4" in compressionsupport):
3131 try:
3132 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
3133 except (ValueError, TypeError) as e:
3134 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
3135 elif(fextname==".lzo" and "lzop" in compressionsupport):
3136 try:
3137 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
3138 except (ValueError, TypeError) as e:
3139 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
3140 elif(fextname==".lzma" and "lzma" in compressionsupport):
3141 try:
3142 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
3143 except (ValueError, TypeError) as e:
3144 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
3145 elif((fextname==".zz" or fextname==".zl" or fextname==".zlib") and "zlib" in compressionsupport):
3146 outfp = ZlibFile(outfile, mode=mode, level=compressionlevel);
3147 except FileNotFoundError:
3148 return False;
3149 try:
3150 outfp.write_through = True;
3151 except AttributeError:
3152 pass;
3153 return outfp;
3154
3155 def GetDevMajorMinor(fdev):
3156 retdev = [];
3157 if(hasattr(os, "minor")):
3158 retdev.append(os.minor(fdev));
3159 else:
3160 retdev.append(0);
3161 if(hasattr(os, "major")):
3162 retdev.append(os.major(fdev));
3163 else:
3164 retdev.append(0);
3165 return retdev;
3166
3167 def CheckSumSupport(checkfor, guaranteed=True):
3168 if(guaranteed):
3169 try:
3170 hash_list = sorted(list(hashlib.algorithms_guaranteed));
3171 except AttributeError:
3172 hash_list = sorted(list(hashlib.algorithms));
3173 else:
3174 try:
3175 hash_list = sorted(list(hashlib.algorithms_available));
3176 except AttributeError:
3177 hash_list = sorted(list(hashlib.algorithms));
3178 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
3179 if(checkfor in checklistout):
3180 return True;
3181 else:
3182 return False;
3183
3184 def CheckSumSupportAlt(checkfor, guaranteed=True):
3185 if(guaranteed):
3186 try:
3187 hash_list = sorted(list(hashlib.algorithms_guaranteed));
3188 except AttributeError:
3189 hash_list = sorted(list(hashlib.algorithms));
3190 else:
3191 try:
3192 hash_list = sorted(list(hashlib.algorithms_available));
3193 except AttributeError:
3194 hash_list = sorted(list(hashlib.algorithms));
3195 checklistout = hash_list;
3196 if(checkfor in checklistout):
3197 return True;
3198 else:
3199 return False;
3200
3201 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3202 formatspecs = FormatSpecsListToDict(formatspecs);
3203 advancedlist = formatspecs['use_advanced_list'];
3204 altinode = formatspecs['use_alt_inode'];
3205 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3206 outfile = RemoveWindowsPath(outfile);
3207 checksumtype = checksumtype.lower();
3208 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3209 checksumtype="crc32";
3210 if(checksumtype=="none"):
3211 checksumtype = "";
3212 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3213 compression = "auto";
3214 if(compression not in compressionlist and compression is None):
3215 compression = "auto";
3216 if(verbose):
3217 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3218 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3219 if(os.path.exists(outfile)):
3220 try:
3221 os.unlink(outfile);
3222 except OSError as e:
3223 pass;
3224 if(outfile=="-"):
3225 verbose = False;
3226 catfp = BytesIO();
3227 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3228 catfp = outfile;
3229 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3230 catfp = BytesIO();
3231 else:
3232 fbasename = os.path.splitext(outfile)[0];
3233 fextname = os.path.splitext(outfile)[1];
3234 if(not compresswholefile and fextname in outextlistwd):
3235 compresswholefile = True;
3236 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3237 catver = formatspecs['format_ver'];
3238 fileheaderver = str(int(catver.replace(".", "")));
3239 infilelist = [];
3240 if(infiles=="-"):
3241 for line in sys.stdin:
3242 infilelist.append(line.strip());
3243 infilelist = list(filter(None, infilelist));
3244 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
3245 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
3246 return False;
3247 with UncompressFile(infiles, formatspecs, "r") as finfile:
3248 for line in finfile:
3249 infilelist.append(line.strip());
3250 infilelist = list(filter(None, infilelist));
3251 else:
3252 if(isinstance(infiles, (list, tuple, ))):
3253 infilelist = list(filter(None, infiles));
3254 elif(isinstance(infiles, (str, ))):
3255 infilelist = list(filter(None, [infiles]));
3256 if(advancedlist):
3257 GetDirList = ListDirAdvanced(infilelist, followlink, False);
3258 else:
3259 GetDirList = ListDir(infilelist, followlink, False);
3260 if(not GetDirList):
3261 return False;
3262 curinode = 0;
3263 curfid = 0;
3264 inodelist = [];
3265 inodetofile = {};
3266 filetoinode = {};
3267 inodetocatinode = {};
3268 numfiles = int(len(GetDirList));
3269 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3270 for curfname in GetDirList:
3271 if(re.findall(r"^[.|/]", curfname)):
3272 fname = curfname;
3273 else:
3274 fname = "./"+curfname;
3275 if(verbose):
3276 VerbosePrintOut(fname);
3277 if(not followlink or followlink is None):
3278 fstatinfo = os.lstat(fname);
3279 else:
3280 fstatinfo = os.stat(fname);
3281 fpremode = fstatinfo.st_mode;
3282 finode = fstatinfo.st_ino;
3283 flinkcount = fstatinfo.st_nlink;
3284 ftype = 0;
3285 if(stat.S_ISREG(fpremode)):
3286 ftype = 0;
3287 elif(stat.S_ISLNK(fpremode)):
3288 ftype = 2;
3289 elif(stat.S_ISCHR(fpremode)):
3290 ftype = 3;
3291 elif(stat.S_ISBLK(fpremode)):
3292 ftype = 4;
3293 elif(stat.S_ISDIR(fpremode)):
3294 ftype = 5;
3295 elif(stat.S_ISFIFO(fpremode)):
3296 ftype = 6;
3297 elif(stat.S_ISSOCK(fpremode)):
3298 ftype = 8;
3299 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
3300 ftype = 9;
3301 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
3302 ftype = 10;
3303 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
3304 ftype = 11;
3305 else:
3306 ftype = 0;
3307 flinkname = "";
3308 fcurfid = format(int(curfid), 'x').lower();
3309 if(not followlink and finode!=0):
3310 if(ftype!=1):
3311 if(finode in inodelist):
3312 ftype = 1;
3313 flinkname = inodetofile[finode];
3314 if(altinode):
3315 fcurinode = format(int(finode), 'x').lower();
3316 else:
3317 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
3318 if(finode not in inodelist):
3319 inodelist.append(finode);
3320 inodetofile.update({finode: fname});
3321 inodetocatinode.update({finode: curinode});
3322 if(altinode):
3323 fcurinode = format(int(finode), 'x').lower();
3324 else:
3325 fcurinode = format(int(curinode), 'x').lower();
3326 curinode = curinode + 1;
3327 else:
3328 fcurinode = format(int(curinode), 'x').lower();
3329 curinode = curinode + 1;
3330 curfid = curfid + 1;
3331 if(ftype==2):
3332 flinkname = os.readlink(fname);
3333 fdev = fstatinfo.st_dev;
3334 getfdev = GetDevMajorMinor(fdev);
3335 fdev_minor = getfdev[0];
3336 fdev_major = getfdev[1];
3337 frdev = fstatinfo.st_dev;
3338 if(hasattr(fstatinfo, "st_rdev")):
3339 frdev = fstatinfo.st_rdev;
3340 else:
3341 frdev = fstatinfo.st_dev;
3342 getfrdev = GetDevMajorMinor(frdev);
3343 frdev_minor = getfrdev[0];
3344 frdev_major = getfrdev[1];
3345 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3346 fsize = format(int("0"), 'x').lower();
3347 elif(ftype==0 or ftype==7):
3348 fsize = format(int(fstatinfo.st_size), 'x').lower();
3349 else:
3350 fsize = format(int(fstatinfo.st_size)).lower();
3351 fatime = format(int(fstatinfo.st_atime), 'x').lower();
3352 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
3353 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
3354 if(hasattr(fstatinfo, "st_birthtime")):
3355 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
3356 else:
3357 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
3358 fmode = format(int(fstatinfo.st_mode), 'x').lower();
3359 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
3360 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
3361 fuid = format(int(fstatinfo.st_uid), 'x').lower();
3362 fgid = format(int(fstatinfo.st_gid), 'x').lower();
3363 funame = "";
3364 try:
3365 import pwd;
3366 try:
3367 userinfo = pwd.getpwuid(fstatinfo.st_uid);
3368 funame = userinfo.pw_name;
3369 except KeyError:
3370 funame = "";
3371 except ImportError:
3372 funame = "";
3373 fgname = "";
3374 try:
3375 import grp;
3376 try:
3377 groupinfo = grp.getgrgid(fstatinfo.st_gid);
3378 fgname = groupinfo.gr_name;
3379 except KeyError:
3380 fgname = "";
3381 except ImportError:
3382 fgname = "";
3383 fdev_minor = format(int(fdev_minor), 'x').lower();
3384 fdev_major = format(int(fdev_major), 'x').lower();
3385 frdev_minor = format(int(frdev_minor), 'x').lower();
3386 frdev_major = format(int(frdev_major), 'x').lower();
3387 finode = format(int(finode), 'x').lower();
3388 flinkcount = format(int(flinkcount), 'x').lower();
3389 if(hasattr(fstatinfo, "st_file_attributes")):
3390 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
3391 else:
3392 fwinattributes = format(int(0), 'x').lower();
3393 fcompression = "";
3394 fcsize = format(int(0), 'x').lower();
3395 fcontents = BytesIO();
3396 if(ftype==0 or ftype==7):
3397 with open(fname, "rb") as fpc:
3398 shutil.copyfileobj(fpc, fcontents);
3399 if(not compresswholefile):
3400 fcontents.seek(0, 2);
3401 ucfsize = fcontents.tell();
3402 fcontents.seek(0, 0);
3403 if(compression=="auto"):
3404 ilsize = len(compressionlistalt);
3405 ilmin = 0;
3406 ilcsize = [];
3407 while(ilmin < ilsize):
3408 cfcontents = BytesIO();
3409 shutil.copyfileobj(fcontents, cfcontents);
3410 fcontents.seek(0, 0);
3411 cfcontents.seek(0, 0);
3412 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3413 if(cfcontents):
3414 cfcontents.seek(0, 2);
3415 ilcsize.append(cfcontents.tell());
3416 cfcontents.close();
3417 else:
3418 try:
3419 ilcsize.append(sys.maxint);
3420 except AttributeError:
3421 ilcsize.append(sys.maxsize);
3422 ilmin = ilmin + 1;
3423 ilcmin = ilcsize.index(min(ilcsize));
3424 compression = compressionlistalt[ilcmin];
3425 fcontents.seek(0, 0);
3426 cfcontents = BytesIO();
3427 shutil.copyfileobj(fcontents, cfcontents);
3428 cfcontents.seek(0, 0);
3429 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3430 cfcontents.seek(0, 2);
3431 cfsize = cfcontents.tell();
3432 if(ucfsize > cfsize):
3433 fcsize = format(int(cfsize), 'x').lower();
3434 fcompression = compression;
3435 fcontents.close();
3436 fcontents = cfcontents;
3437 if(fcompression=="none"):
3438 fcompression = "";
3439 if(followlink and (ftype==1 or ftype==2)):
3440 flstatinfo = os.stat(flinkname);
3441 with open(flinkname, "rb") as fpc:
3442 shutil.copyfileobj(fpc, fcontents);
3443 if(not compresswholefile):
3444 fcontents.seek(0, 2);
3445 ucfsize = fcontents.tell();
3446 fcontents.seek(0, 0);
3447 if(compression=="auto"):
3448 ilsize = len(compressionlistalt);
3449 ilmin = 0;
3450 ilcsize = [];
3451 while(ilmin < ilsize):
3452 cfcontents = BytesIO();
3453 shutil.copyfileobj(fcontents, cfcontents);
3454 fcontents.seek(0, 0);
3455 cfcontents.seek(0, 0);
3456 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3457 if(cfcontents):
3458 cfcontents.seek(0, 2);
3459 ilcsize.append(cfcontents.tell());
3460 cfcontents.close();
3461 else:
3462 try:
3463 ilcsize.append(sys.maxint);
3464 except AttributeError:
3465 ilcsize.append(sys.maxsize);
3466 ilmin = ilmin + 1;
3467 ilcmin = ilcsize.index(min(ilcsize));
3468 compression = compressionlistalt[ilcmin];
3469 fcontents.seek(0, 0);
3470 cfcontents = BytesIO();
3471 shutil.copyfileobj(fcontents, cfcontents);
3472 cfcontents.seek(0, 0);
3473 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3474 cfcontents.seek(0, 2);
3475 cfsize = cfcontents.tell();
3476 if(ucfsize > cfsize):
3477 fcsize = format(int(cfsize), 'x').lower();
3478 fcompression = compression;
3479 fcontents.close();
3480 fcontents = cfcontents;
3481 fcontents.seek(0, 0);
3482 ftypehex = format(ftype, 'x').lower();
3483 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3484 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3485 fcontents.close();
3486 if(numfiles>0):
3487 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3488 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3489 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3490 try:
3491 catfp.flush();
3492 os.fsync(catfp.fileno());
3493 except io.UnsupportedOperation:
3494 pass;
3495 except AttributeError:
3496 pass;
3497 except OSError as e:
3498 pass;
3499 if(outfile=="-"):
3500 catfp.seek(0, 0);
3501 if(hasattr(sys.stdout, "buffer")):
3502 shutil.copyfileobj(catfp, sys.stdout.buffer);
3503 else:
3504 shutil.copyfileobj(catfp, sys.stdout);
3505 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3506 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3507 catfp.seek(0, 0);
3508 upload_file_to_internet_file(catfp, outfile);
3509 if(returnfp):
3510 catfp.seek(0, 0);
3511 return catfp;
3512 else:
3513 catfp.close();
3514 return True;
3515
3516 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3517
3518 if(hasattr(shutil, "register_archive_format")):
3519 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3520 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3521 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3522
3523 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3524 formatspecs = FormatSpecsListToDict(formatspecs);
3525 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3526
3527 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3528
3529 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3530 formatspecs = FormatSpecsListToDict(formatspecs);
3531 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3532 outfile = RemoveWindowsPath(outfile);
3533 checksumtype = checksumtype.lower();
3534 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3535 checksumtype="crc32";
3536 if(checksumtype=="none"):
3537 checksumtype = "";
3538 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3539 compression = "auto";
3540 if(compression not in compressionlist and compression is None):
3541 compression = "auto";
3542 if(verbose):
3543 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3544 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3545 if(os.path.exists(outfile)):
3546 try:
3547 os.unlink(outfile);
3548 except OSError as e:
3549 pass;
3550 if(outfile=="-"):
3551 verbose = False;
3552 catfp = BytesIO();
3553 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3554 catfp = outfile;
3555 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3556 catfp = BytesIO();
3557 else:
3558 fbasename = os.path.splitext(outfile)[0];
3559 fextname = os.path.splitext(outfile)[1];
3560 if(not compresswholefile and fextname in outextlistwd):
3561 compresswholefile = True;
3562 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3563 catver = formatspecs['format_ver'];
3564 fileheaderver = str(int(catver.replace(".", "")));
3565 curinode = 0;
3566 curfid = 0;
3567 inodelist = [];
3568 inodetofile = {};
3569 filetoinode = {};
3570 inodetocatinode = {};
3571 if(infile=="-"):
3572 infile = BytesIO();
3573 if(hasattr(sys.stdin, "buffer")):
3574 shutil.copyfileobj(sys.stdin.buffer, infile);
3575 else:
3576 shutil.copyfileobj(sys.stdin, infile);
3577 infile.seek(0, 0);
3578 if(not infile):
3579 return False;
3580 infile.seek(0, 0);
3581 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3582 infile = download_file_from_internet_file(infile);
3583 infile.seek(0, 0);
3584 if(not infile):
3585 return False;
3586 infile.seek(0, 0);
3587 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3588 return False;
3589 elif(os.path.exists(infile) and os.path.isfile(infile)):
3590 try:
3591 if(not tarfile.TarFileCheck(infile)):
3592 return False;
3593 except AttributeError:
3594 if(not TarFileCheck(infile)):
3595 return False;
3596 try:
3597 if(hasattr(infile, "read") or hasattr(infile, "write")):
3598 tarfp = tarfile.open(fileobj=infile, mode="r");
3599 else:
3600 tarfp = tarfile.open(infile, "r");
3601 except FileNotFoundError:
3602 return False;
3603 numfiles = int(len(tarfp.getmembers()));
3604 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3605 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3606 if(re.findall(r"^[.|/]", member.name)):
3607 fname = member.name;
3608 else:
3609 fname = "./"+member.name;
3610 if(verbose):
3611 VerbosePrintOut(fname);
3612 fpremode = member.mode;
3613 ffullmode = member.mode;
3614 flinkcount = 0;
3615 ftype = 0;
3616 if(member.isreg()):
3617 ffullmode = member.mode + stat.S_IFREG;
3618 ftype = 0;
3619 elif(member.isdev()):
3620 ffullmode = member.mode;
3621 ftype = 7;
3622 elif(member.islnk()):
3623 ffullmode = member.mode + stat.S_IFREG;
3624 ftype = 1;
3625 elif(member.issym()):
3626 ffullmode = member.mode + stat.S_IFLNK;
3627 ftype = 2;
3628 elif(member.ischr()):
3629 ffullmode = member.mode + stat.S_IFCHR;
3630 ftype = 3;
3631 elif(member.isblk()):
3632 ffullmode = member.mode + stat.S_IFBLK;
3633 ftype = 4;
3634 elif(member.isdir()):
3635 ffullmode = member.mode + stat.S_IFDIR;
3636 ftype = 5;
3637 elif(member.isfifo()):
3638 ffullmode = member.mode + stat.S_IFIFO;
3639 ftype = 6;
3640 elif(member.issparse()):
3641 ffullmode = member.mode;
3642 ftype = 12;
3643 else:
3644 ffullmode = member.mode;
3645 ftype = 0;
3646 flinkname = "";
3647 fcurfid = format(int(curfid), 'x').lower();
3648 fcurinode = format(int(curfid), 'x').lower();
3649 curfid = curfid + 1;
3650 if(ftype==2):
3651 flinkname = member.linkname;
3652 fdev_minor = format(int(member.devminor), 'x').lower();
3653 fdev_major = format(int(member.devmajor), 'x').lower();
3654 frdev_minor = format(int(member.devminor), 'x').lower();
3655 frdev_major = format(int(member.devmajor), 'x').lower();
3656 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3657 fsize = format(int("0"), 'x').lower();
3658 elif(ftype==0 or ftype==7):
3659 fsize = format(int(member.size), 'x').lower();
3660 else:
3661 fsize = format(int(member.size), 'x').lower();
3662 fatime = format(int(member.mtime), 'x').lower();
3663 fmtime = format(int(member.mtime), 'x').lower();
3664 fctime = format(int(member.mtime), 'x').lower();
3665 fbtime = format(int(member.mtime), 'x').lower();
3666 fmode = format(int(ffullmode), 'x').lower();
3667 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3668 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3669 fuid = format(int(member.uid), 'x').lower();
3670 fgid = format(int(member.gid), 'x').lower();
3671 funame = member.uname;
3672 fgname = member.gname;
3673 flinkcount = format(int(flinkcount), 'x').lower();
3674 fwinattributes = format(int(0), 'x').lower();
3675 fcompression = "";
3676 fcsize = format(int(0), 'x').lower();
3677 fcontents = BytesIO();
3678 if(ftype==0 or ftype==7):
3679 with tarfp.extractfile(member) as fpc:
3680 shutil.copyfileobj(fpc, fcontents);
3681 if(not compresswholefile):
3682 fcontents.seek(0, 2);
3683 ucfsize = fcontents.tell();
3684 fcontents.seek(0, 0);
3685 if(compression=="auto"):
3686 ilsize = len(compressionlistalt);
3687 ilmin = 0;
3688 ilcsize = [];
3689 while(ilmin < ilsize):
3690 cfcontents = BytesIO();
3691 shutil.copyfileobj(fcontents, cfcontents);
3692 fcontents.seek(0, 0);
3693 cfcontents.seek(0, 0);
3694 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3695 if(cfcontents):
3696 cfcontents.seek(0, 2);
3697 ilcsize.append(cfcontents.tell());
3698 cfcontents.close();
3699 else:
3700 try:
3701 ilcsize.append(sys.maxint);
3702 except AttributeError:
3703 ilcsize.append(sys.maxsize);
3704 ilmin = ilmin + 1;
3705 ilcmin = ilcsize.index(min(ilcsize));
3706 compression = compressionlistalt[ilcmin];
3707 fcontents.seek(0, 0);
3708 cfcontents = BytesIO();
3709 shutil.copyfileobj(fcontents, cfcontents);
3710 cfcontents.seek(0, 0);
3711 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3712 cfcontents.seek(0, 2);
3713 cfsize = cfcontents.tell();
3714 if(ucfsize > cfsize):
3715 fcsize = format(int(cfsize), 'x').lower();
3716 fcompression = compression;
3717 fcontents.close();
3718 fcontents = cfcontents;
3719 if(fcompression=="none"):
3720 fcompression = "";
3721 fcontents.seek(0, 0);
3722 ftypehex = format(ftype, 'x').lower();
3723 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3724 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3725 fcontents.close();
3726 if(numfiles>0):
3727 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3728 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3729 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3730 try:
3731 catfp.flush();
3732 os.fsync(catfp.fileno());
3733 except io.UnsupportedOperation:
3734 pass;
3735 except AttributeError:
3736 pass;
3737 except OSError as e:
3738 pass;
3739 if(outfile=="-"):
3740 catfp.seek(0, 0);
3741 if(hasattr(sys.stdout, "buffer")):
3742 shutil.copyfileobj(catfp, sys.stdout.buffer);
3743 else:
3744 shutil.copyfileobj(catfp, sys.stdout);
3745 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3746 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3747 catfp.seek(0, 0);
3748 upload_file_to_internet_file(catfp, outfile);
3749 if(returnfp):
3750 catfp.seek(0, 0);
3751 return catfp;
3752 else:
3753 catfp.close();
3754 return True;
3755
3756 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3757
3758 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3759 formatspecs = FormatSpecsListToDict(formatspecs);
3760 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3761 outfile = RemoveWindowsPath(outfile);
3762 checksumtype = checksumtype.lower();
3763 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3764 checksumtype="crc32";
3765 if(checksumtype=="none"):
3766 checksumtype = "";
3767 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3768 compression = "auto";
3769 if(compression not in compressionlist and compression is None):
3770 compression = "auto";
3771 if(verbose):
3772 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3773 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3774 if(os.path.exists(outfile)):
3775 try:
3776 os.unlink(outfile);
3777 except OSError as e:
3778 pass;
3779 if(outfile=="-"):
3780 verbose = False;
3781 catfp = BytesIO();
3782 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3783 catfp = outfile;
3784 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3785 catfp = BytesIO();
3786 else:
3787 fbasename = os.path.splitext(outfile)[0];
3788 fextname = os.path.splitext(outfile)[1];
3789 if(not compresswholefile and fextname in outextlistwd):
3790 compresswholefile = True;
3791 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3792 catver = formatspecs['format_ver'];
3793 fileheaderver = str(int(catver.replace(".", "")));
3794 curinode = 0;
3795 curfid = 0;
3796 inodelist = [];
3797 inodetofile = {};
3798 filetoinode = {};
3799 inodetocatinode = {};
3800 if(infile=="-"):
3801 infile = BytesIO();
3802 if(hasattr(sys.stdin, "buffer")):
3803 shutil.copyfileobj(sys.stdin.buffer, infile);
3804 else:
3805 shutil.copyfileobj(sys.stdin, infile);
3806 infile.seek(0, 0);
3807 if(not infile):
3808 return False;
3809 infile.seek(0, 0);
3810 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3811 infile = download_file_from_internet_file(infile);
3812 infile.seek(0, 0);
3813 if(not infile):
3814 return False;
3815 infile.seek(0, 0);
3816 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3817 return False;
3818 if(not zipfile.is_zipfile(infile)):
3819 return False;
3820 try:
3821 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3822 except FileNotFoundError:
3823 return False;
3824 ziptest = zipfp.testzip();
3825 if(ziptest):
3826 VerbosePrintOut("Bad file found!");
3827 numfiles = int(len(zipfp.infolist()));
3828 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3829 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3830 if(re.findall(r"^[.|/]", member.filename)):
3831 fname = member.filename;
3832 else:
3833 fname = "./"+member.filename;
3834 zipinfo = zipfp.getinfo(member.filename);
3835 if(verbose):
3836 VerbosePrintOut(fname);
3837 if(not member.is_dir()):
3838 fpremode = int(stat.S_IFREG + 438);
3839 elif(member.is_dir()):
3840 fpremode = int(stat.S_IFDIR + 511);
3841 flinkcount = 0;
3842 ftype = 0;
3843 if(not member.is_dir()):
3844 ftype = 0;
3845 elif(member.is_dir()):
3846 ftype = 5;
3847 flinkname = "";
3848 fcurfid = format(int(curfid), 'x').lower();
3849 fcurinode = format(int(curfid), 'x').lower();
3850 curfid = curfid + 1;
3851 fdev_minor = format(int(0), 'x').lower();
3852 fdev_major = format(int(0), 'x').lower();
3853 frdev_minor = format(int(0), 'x').lower();
3854 frdev_major = format(int(0), 'x').lower();
3855 if(ftype==5):
3856 fsize = format(int("0"), 'x').lower();
3857 elif(ftype==0):
3858 fsize = format(int(member.file_size), 'x').lower();
3859 else:
3860 fsize = format(int(member.file_size), 'x').lower();
3861 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3862 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3863 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3864 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3865 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3866 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3867 if(not member.is_dir()):
3868 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3869 fchmode = stat.S_IMODE(int(stat.S_IFREG + 438));
3870 ftypemod = stat.S_IFMT(int(stat.S_IFREG + 438));
3871 elif(member.is_dir()):
3872 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3873 fchmode = stat.S_IMODE(int(stat.S_IFDIR + 511));
3874 ftypemod = stat.S_IFMT(int(stat.S_IFDIR + 511));
3875 elif(zipinfo.create_system==3):
3876 fwinattributes = format(int(0), 'x').lower();
3877 try:
3878 fmode = format(int(zipinfo.external_attr), 'x').lower();
3879 prefmode = int(zipinfo.external_attr);
3880 fchmode = stat.S_IMODE(prefmode);
3881 ftypemod = stat.S_IFMT(prefmode);
3882 except OverflowError:
3883 fmode = format(int(zipinfo.external_attr >> 16), 'x').lower();
3884 prefmode = int(zipinfo.external_attr >> 16);
3885 fchmode = stat.S_IMODE(prefmode);
3886 ftypemod = stat.S_IFMT(prefmode);
3887 else:
3888 fwinattributes = format(int(0), 'x').lower();
3889 if(not member.is_dir()):
3890 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3891 prefmode = int(stat.S_IFREG + 438);
3892 fchmode = stat.S_IMODE(prefmode);
3893 ftypemod = stat.S_IFMT(prefmode);
3894 elif(member.is_dir()):
3895 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3896 prefmode = int(stat.S_IFDIR + 511);
3897 fchmode = stat.S_IMODE(prefmode);
3898 ftypemod = stat.S_IFMT(prefmode);
3899 fcompression = "";
3900 fcsize = format(int(0), 'x').lower();
3901 try:
3902 fuid = format(int(os.getuid()), 'x').lower();
3903 except AttributeError:
3904 fuid = format(int(0), 'x').lower();
3905 except KeyError:
3906 fuid = format(int(0), 'x').lower();
3907 try:
3908 fgid = format(int(os.getgid()), 'x').lower();
3909 except AttributeError:
3910 fgid = format(int(0), 'x').lower();
3911 except KeyError:
3912 fgid = format(int(0), 'x').lower();
3913 try:
3914 import pwd;
3915 try:
3916 userinfo = pwd.getpwuid(os.getuid());
3917 funame = userinfo.pw_name;
3918 except KeyError:
3919 funame = "";
3920 except AttributeError:
3921 funame = "";
3922 except ImportError:
3923 funame = "";
3924 fgname = "";
3925 try:
3926 import grp;
3927 try:
3928 groupinfo = grp.getgrgid(os.getgid());
3929 fgname = groupinfo.gr_name;
3930 except KeyError:
3931 fgname = "";
3932 except AttributeError:
3933 fgname = "";
3934 except ImportError:
3935 fgname = "";
3936 fcontents = BytesIO();
3937 if(ftype==0):
3938 fcontents.write(zipfp.read(member.filename));
3939 if(not compresswholefile):
3940 fcontents.seek(0, 2);
3941 ucfsize = fcontents.tell();
3942 fcontents.seek(0, 0);
3943 if(compression=="auto"):
3944 ilsize = len(compressionlistalt);
3945 ilmin = 0;
3946 ilcsize = [];
3947 while(ilmin < ilsize):
3948 cfcontents = BytesIO();
3949 shutil.copyfileobj(fcontents, cfcontents);
3950 fcontents.seek(0, 0);
3951 cfcontents.seek(0, 0);
3952 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3953 cfcontents.seek(0, 2);
3954 ilcsize.append(cfcontents.tell());
3955 cfcontents.close();
3956 ilmin = ilmin + 1;
3957 ilcmin = ilcsize.index(min(ilcsize));
3958 compression = compressionlistalt[ilcmin];
3959 fcontents.seek(0, 0);
3960 cfcontents = BytesIO();
3961 shutil.copyfileobj(fcontents, cfcontents);
3962 cfcontents.seek(0, 0);
3963 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3964 cfcontents.seek(0, 2);
3965 cfsize = cfcontents.tell();
3966 if(ucfsize > cfsize):
3967 fcsize = format(int(cfsize), 'x').lower();
3968 fcompression = compression;
3969 fcontents.close();
3970 fcontents = cfcontents;
3971 if(fcompression=="none"):
3972 fcompression = "";
3973 fcontents.seek(0, 0);
3974 ftypehex = format(ftype, 'x').lower();
3975 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3976 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3977 fcontents.close();
3978 if(numfiles>0):
3979 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3980 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3981 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3982 try:
3983 catfp.flush();
3984 os.fsync(catfp.fileno());
3985 except io.UnsupportedOperation:
3986 pass;
3987 except AttributeError:
3988 pass;
3989 except OSError as e:
3990 pass;
3991 if(outfile=="-"):
3992 catfp.seek(0, 0);
3993 if(hasattr(sys.stdout, "buffer")):
3994 shutil.copyfileobj(catfp, sys.stdout.buffer);
3995 else:
3996 shutil.copyfileobj(catfp, sys.stdout);
3997 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3998 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3999 catfp.seek(0, 0);
4000 upload_file_to_internet_file(catfp, outfile);
4001 if(returnfp):
4002 catfp.seek(0, 0);
4003 return catfp;
4004 else:
4005 catfp.close();
4006 return True;
4007
4008 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
4009
4010 if(not rarfile_support):
4011 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4012 return False
4013
4014 if(rarfile_support):
4015 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4016 formatspecs = FormatSpecsListToDict(formatspecs);
4017 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4018 outfile = RemoveWindowsPath(outfile);
4019 checksumtype = checksumtype.lower();
4020 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
4021 checksumtype="crc32";
4022 if(checksumtype=="none"):
4023 checksumtype = "";
4024 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
4025 compression = "auto";
4026 if(compression not in compressionlist and compression is None):
4027 compression = "auto";
4028 if(verbose):
4029 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4030 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4031 if(os.path.exists(outfile)):
4032 try:
4033 os.unlink(outfile);
4034 except OSError as e:
4035 pass;
4036 if(outfile=="-"):
4037 verbose = False;
4038 catfp = BytesIO();
4039 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
4040 catfp = outfile;
4041 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4042 catfp = BytesIO();
4043 else:
4044 fbasename = os.path.splitext(outfile)[0];
4045 fextname = os.path.splitext(outfile)[1];
4046 if(not compresswholefile and fextname in outextlistwd):
4047 compresswholefile = True;
4048 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
4049 catver = formatspecs['format_ver'];
4050 fileheaderver = str(int(catver.replace(".", "")));
4051 curinode = 0;
4052 curfid = 0;
4053 inodelist = [];
4054 inodetofile = {};
4055 filetoinode = {};
4056 inodetocatinode = {};
4057 if(not os.path.exists(infile) or not os.path.isfile(infile)):
4058 return False;
4059 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
4060 return False;
4061 rarfp = rarfile.RarFile(infile, "r");
4062 rartest = rarfp.testrar();
4063 if(rartest):
4064 VerbosePrintOut("Bad file found!");
4065 numfiles = int(len(rarfp.infolist()));
4066 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
4067 try:
4068 catfp.flush();
4069 os.fsync(catfp.fileno());
4070 except io.UnsupportedOperation:
4071 pass;
4072 except AttributeError:
4073 pass;
4074 except OSError as e:
4075 pass;
4076 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
4077 is_unix = False;
4078 is_windows = False;
4079 if(member.host_os==rarfile.RAR_OS_UNIX):
4080 is_windows = False;
4081 try:
4082 member.external_attr
4083 is_unix = True;
4084 except AttributeError:
4085 is_unix = False;
4086 elif(member.host_os==rarfile.RAR_OS_WIN32):
4087 is_unix = False;
4088 try:
4089 member.external_attr
4090 is_windows = True;
4091 except AttributeError:
4092 is_windows = False;
4093 else:
4094 is_unix = False;
4095 is_windows = False;
4096 if(re.findall(r"^[.|/]", member.filename)):
4097 fname = member.filename;
4098 else:
4099 fname = "./"+member.filename;
4100 rarinfo = rarfp.getinfo(member.filename);
4101 if(verbose):
4102 VerbosePrintOut(fname);
4103 if(is_unix and member.external_attr !=0):
4104 fpremode = int(member.external_attr);
4105 elif(member.is_file()):
4106 fpremode = int(stat.S_IFREG + 438);
4107 elif(member.is_symlink()):
4108 fpremode = int(stat.S_IFLNK + 438);
4109 elif(member.is_dir()):
4110 fpremode = int(stat.S_IFDIR + 511);
4111 if(is_windows and member.external_attr !=0):
4112 fwinattributes = format(int(member.external_attr), 'x').lower();
4113 else:
4114 fwinattributes = format(int(0), 'x').lower();
4115 fcompression = "";
4116 fcsize = format(int(0), 'x').lower();
4117 flinkcount = 0;
4118 ftype = 0;
4119 if(member.is_file()):
4120 ftype = 0;
4121 elif(member.is_symlink()):
4122 ftype = 2;
4123 elif(member.is_dir()):
4124 ftype = 5;
4125 flinkname = "";
4126 if(ftype==2):
4127 flinkname = rarfp.read(member.filename).decode("UTF-8");
4128 fcurfid = format(int(curfid), 'x').lower();
4129 fcurinode = format(int(curfid), 'x').lower();
4130 curfid = curfid + 1;
4131 fdev_minor = format(int(0), 'x').lower();
4132 fdev_major = format(int(0), 'x').lower();
4133 frdev_minor = format(int(0), 'x').lower();
4134 frdev_major = format(int(0), 'x').lower();
4135 if(ftype==5):
4136 fsize = format(int("0"), 'x').lower();
4137 elif(ftype==0):
4138 fsize = format(int(member.file_size), 'x').lower();
4139 else:
4140 fsize = format(int(member.file_size), 'x').lower();
4141 try:
4142 if(member.atime):
4143 fatime = format(int(member.atime.timestamp()), 'x').lower();
4144 else:
4145 fatime = format(int(member.mtime.timestamp()), 'x').lower();
4146 except AttributeError:
4147 fatime = format(int(member.mtime.timestamp()), 'x').lower();
4148 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
4149 try:
4150 if(member.ctime):
4151 fctime = format(int(member.ctime.timestamp()), 'x').lower();
4152 else:
4153 fctime = format(int(member.mtime.timestamp()), 'x').lower();
4154 except AttributeError:
4155 fctime = format(int(member.mtime.timestamp()), 'x').lower();
4156 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
4157 if(is_unix and member.external_attr !=0):
4158 fmode = format(int(member.external_attr), 'x').lower();
4159 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
4160 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
4161 elif(member.is_file()):
4162 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4163 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4164 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4165 elif(member.is_symlink()):
4166 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
4167 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4168 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4169 elif(member.is_dir()):
4170 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4171 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4172 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4173 try:
4174 fuid = format(int(os.getuid()), 'x').lower();
4175 except AttributeError:
4176 fuid = format(int(0), 'x').lower();
4177 except KeyError:
4178 fuid = format(int(0), 'x').lower();
4179 try:
4180 fgid = format(int(os.getgid()), 'x').lower();
4181 except AttributeError:
4182 fgid = format(int(0), 'x').lower();
4183 except KeyError:
4184 fgid = format(int(0), 'x').lower();
4185 try:
4186 import pwd;
4187 try:
4188 userinfo = pwd.getpwuid(os.getuid());
4189 funame = userinfo.pw_name;
4190 except KeyError:
4191 funame = "";
4192 except AttributeError:
4193 funame = "";
4194 except ImportError:
4195 funame = "";
4196 fgname = "";
4197 try:
4198 import grp;
4199 try:
4200 groupinfo = grp.getgrgid(os.getgid());
4201 fgname = groupinfo.gr_name;
4202 except KeyError:
4203 fgname = "";
4204 except AttributeError:
4205 fgname = "";
4206 except ImportError:
4207 fgname = "";
4208 fcontents = BytesIO();
4209 if(ftype==0):
4210 fcontents.write(rarfp.read(member.filename));
4211 if(not compresswholefile):
4212 fcontents.seek(0, 2);
4213 ucfsize = fcontents.tell();
4214 fcontents.seek(0, 0);
4215 if(compression=="auto"):
4216 ilsize = len(compressionlistalt);
4217 ilmin = 0;
4218 ilcsize = [];
4219 while(ilmin < ilsize):
4220 cfcontents = BytesIO();
4221 shutil.copyfileobj(fcontents, cfcontents);
4222 fcontents.seek(0, 0);
4223 cfcontents.seek(0, 0);
4224 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
4225 if(cfcontents):
4226 cfcontents.seek(0, 2);
4227 ilcsize.append(cfcontents.tell());
4228 cfcontents.close();
4229 else:
4230 try:
4231 ilcsize.append(sys.maxint);
4232 except AttributeError:
4233 ilcsize.append(sys.maxsize);
4234 ilmin = ilmin + 1;
4235 ilcmin = ilcsize.index(min(ilcsize));
4236 compression = compressionlistalt[ilcmin];
4237 fcontents.seek(0, 0);
4238 cfcontents = BytesIO();
4239 shutil.copyfileobj(fcontents, cfcontents);
4240 cfcontents.seek(0, 0);
4241 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4242 cfcontents.seek(0, 2);
4243 cfsize = cfcontents.tell();
4244 if(ucfsize > cfsize):
4245 fcsize = format(int(cfsize), 'x').lower();
4246 fcompression = compression;
4247 fcontents.close();
4248 fcontents = cfcontents;
4249 if(fcompression=="none"):
4250 fcompression = "";
4251 fcontents.seek(0, 0);
4252 ftypehex = format(ftype, 'x').lower();
4253 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4254 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4255 fcontents.close();
4256 if(numfiles>0):
4257 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4258 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4259 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4260 try:
4261 catfp.flush();
4262 os.fsync(catfp.fileno());
4263 except io.UnsupportedOperation:
4264 pass;
4265 except AttributeError:
4266 pass;
4267 except OSError as e:
4268 pass;
4269 if(outfile=="-"):
4270 catfp.seek(0, 0)
4271 if(hasattr(sys.stdout, "buffer")):
4272 shutil.copyfileobj(catfp, sys.stdout.buffer);
4273 else:
4274 shutil.copyfileobj(catfp, sys.stdout);
4275 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4276 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4277 catfp.seek(0, 0);
4278 upload_file_to_internet_file(catfp, outfile);
4279 if(returnfp):
4280 catfp.seek(0, 0)
4281 return catfp
4282 else:
4283 catfp.close()
4284 return True;
4285
4286 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
4287
4288 if(not py7zr_support):
4289 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4290 return False
4291
4292 if(py7zr_support):
4293 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4294 formatspecs = FormatSpecsListToDict(formatspecs);
4295 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4296 outfile = RemoveWindowsPath(outfile);
4297 checksumtype = checksumtype.lower();
4298 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
4299 checksumtype="crc32";
4300 if(checksumtype=="none"):
4301 checksumtype = "";
4302 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
4303 compression = "auto";
4304 if(compression not in compressionlist and compression is None):
4305 compression = "auto";
4306 if(verbose):
4307 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4308 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4309 if(os.path.exists(outfile)):
4310 try:
4311 os.unlink(outfile);
4312 except OSError as e:
4313 pass;
4314 if(outfile=="-"):
4315 verbose = False;
4316 catfp = BytesIO();
4317 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
4318 catfp = outfile;
4319 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4320 catfp = BytesIO();
4321 else:
4322 fbasename = os.path.splitext(outfile)[0];
4323 fextname = os.path.splitext(outfile)[1];
4324 if(not compresswholefile and fextname in outextlistwd):
4325 compresswholefile = True;
4326 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
4327 catver = formatspecs['format_ver'];
4328 fileheaderver = str(int(catver.replace(".", "")));
4329 curinode = 0;
4330 curfid = 0;
4331 inodelist = [];
4332 inodetofile = {};
4333 filetoinode = {};
4334 inodetocatinode = {};
4335 if(not os.path.exists(infile) or not os.path.isfile(infile)):
4336 return False;
4337 szpfp = py7zr.SevenZipFile(infile, mode="r");
4338 file_content = szpfp.readall();
4339 #sztest = szpfp.testzip();
4340 sztestalt = szpfp.test();
4341 if(sztestalt):
4342 VerbosePrintOut("Bad file found!");
4343 numfiles = int(len(szpfp.list()));
4344 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
4345 for member in sorted(szpfp.list(), key=lambda x: x.filename):
4346 if(re.findall(r"^[.|/]", member.filename)):
4347 fname = member.filename;
4348 else:
4349 fname = "./"+member.filename;
4350 if(verbose):
4351 VerbosePrintOut(fname);
4352 if(not member.is_directory):
4353 fpremode = int(stat.S_IFREG + 438);
4354 elif(member.is_directory):
4355 fpremode = int(stat.S_IFDIR + 511);
4356 fwinattributes = format(int(0), 'x').lower();
4357 fcompression = "";
4358 fcsize = format(int(0), 'x').lower();
4359 flinkcount = 0;
4360 ftype = 0;
4361 if(member.is_directory):
4362 ftype = 5;
4363 else:
4364 ftype = 0;
4365 flinkname = "";
4366 fcurfid = format(int(curfid), 'x').lower();
4367 fcurinode = format(int(curfid), 'x').lower();
4368 curfid = curfid + 1;
4369 fdev_minor = format(int(0), 'x').lower();
4370 fdev_major = format(int(0), 'x').lower();
4371 frdev_minor = format(int(0), 'x').lower();
4372 frdev_major = format(int(0), 'x').lower();
4373 if(ftype==5):
4374 fsize = format(int("0"), 'x').lower();
4375 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
4376 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
4377 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
4378 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
4379 if(member.is_directory):
4380 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4381 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4382 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4383 else:
4384 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4385 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4386 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4387 try:
4388 fuid = format(int(os.getuid()), 'x').lower();
4389 except AttributeError:
4390 fuid = format(int(0), 'x').lower();
4391 except KeyError:
4392 fuid = format(int(0), 'x').lower();
4393 try:
4394 fgid = format(int(os.getgid()), 'x').lower();
4395 except AttributeError:
4396 fgid = format(int(0), 'x').lower();
4397 except KeyError:
4398 fgid = format(int(0), 'x').lower();
4399 try:
4400 import pwd;
4401 try:
4402 userinfo = pwd.getpwuid(os.getuid());
4403 funame = userinfo.pw_name;
4404 except KeyError:
4405 funame = "";
4406 except AttributeError:
4407 funame = "";
4408 except ImportError:
4409 funame = "";
4410 fgname = "";
4411 try:
4412 import grp;
4413 try:
4414 groupinfo = grp.getgrgid(os.getgid());
4415 fgname = groupinfo.gr_name;
4416 except KeyError:
4417 fgname = "";
4418 except AttributeError:
4419 fgname = "";
4420 except ImportError:
4421 fgname = "";
4422 fcontents = BytesIO();
4423 if(ftype==0):
4424 fcontents.write(file_content[member.filename].read());
4425 fsize = format(fcontents.tell(), 'x').lower();
4426 file_content[member.filename].close();
4427 if(not compresswholefile):
4428 fcontents.seek(0, 2);
4429 ucfsize = fcontents.tell();
4430 fcontents.seek(0, 0);
4431 if(compression=="auto"):
4432 ilsize = len(compressionlistalt);
4433 ilmin = 0;
4434 ilcsize = [];
4435 while(ilmin < ilsize):
4436 cfcontents = BytesIO();
4437 shutil.copyfileobj(fcontents, cfcontents);
4438 fcontents.seek(0, 0);
4439 cfcontents.seek(0, 0);
4440 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
4441 if(cfcontents):
4442 cfcontents.seek(0, 2);
4443 ilcsize.append(cfcontents.tell());
4444 cfcontents.close();
4445 else:
4446 try:
4447 ilcsize.append(sys.maxint);
4448 except AttributeError:
4449 ilcsize.append(sys.maxsize);
4450 ilmin = ilmin + 1;
4451 ilcmin = ilcsize.index(min(ilcsize));
4452 compression = compressionlistalt[ilcmin];
4453 fcontents.seek(0, 0);
4454 cfcontents = BytesIO();
4455 shutil.copyfileobj(fcontents, cfcontents);
4456 cfcontents.seek(0, 0);
4457 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4458 cfcontents.seek(0, 2);
4459 cfsize = cfcontents.tell();
4460 if(ucfsize > cfsize):
4461 fcsize = format(int(cfsize), 'x').lower();
4462 fcompression = compression;
4463 fcontents.close();
4464 fcontents = cfcontents;
4465 if(fcompression=="none"):
4466 fcompression = "";
4467 fcontents.seek(0, 0);
4468 ftypehex = format(ftype, 'x').lower();
4469 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4470 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4471 fcontents.close();
4472 if(numfiles>0):
4473 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4474 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4475 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4476 try:
4477 catfp.flush();
4478 os.fsync(catfp.fileno());
4479 except io.UnsupportedOperation:
4480 pass;
4481 except AttributeError:
4482 pass;
4483 except OSError as e:
4484 pass;
4485 if(outfile=="-"):
4486 catfp.seek(0, 0);
4487 if(hasattr(sys.stdout, "buffer")):
4488 shutil.copyfileobj(catfp, sys.stdout.buffer);
4489 else:
4490 shutil.copyfileobj(catfp, sys.stdout);
4491 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4492 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4493 catfp.seek(0, 0);
4494 upload_file_to_internet_file(catfp, outfile);
4495 if(returnfp):
4496 catfp.seek(0, 0)
4497 return catfp
4498 else:
4499 catfp.close()
4500 return True;
4501
4502 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4503
4504 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4505 formatspecs = FormatSpecsListToDict(formatspecs);
4506 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4507 if(verbose):
4508 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4509 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4510 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4511 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4512 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4513 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4514 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4515 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4516 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4517 elif(checkcompressfile=="catfile"):
4518 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4519 else:
4520 return False;
4521 return False;
4522
4523 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4524
4525 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4526 formatspecs = FormatSpecsListToDict(formatspecs);
4527 if(hasattr(infile, "read") or hasattr(infile, "write")):
4528 catfp = infile;
4529 catfp.seek(0, 0);
4530 catfp = UncompressArchiveFile(catfp, formatspecs);
4531 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4532 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4533 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4534 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4535 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4536 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4537 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4538 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4539 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4540 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4541 return False;
4542 if(not catfp):
4543 return False;
4544 catfp.seek(0, 0);
4545 elif(infile=="-"):
4546 catfp = BytesIO();
4547 if(hasattr(sys.stdin, "buffer")):
4548 shutil.copyfileobj(sys.stdin.buffer, catfp);
4549 else:
4550 shutil.copyfileobj(sys.stdin, catfp);
4551 catfp.seek(0, 0);
4552 catfp = UncompressArchiveFile(catfp, formatspecs);
4553 if(not catfp):
4554 return False;
4555 catfp.seek(0, 0);
4556 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4557 catfp = download_file_from_internet_file(infile);
4558 catfp.seek(0, 0);
4559 catfp = UncompressArchiveFile(catfp, formatspecs);
4560 if(not catfp):
4561 return False;
4562 catfp.seek(0, 0);
4563 else:
4564 infile = RemoveWindowsPath(infile);
4565 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4566 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4567 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4568 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4569 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4570 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4571 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4572 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4573 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4574 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4575 return False;
4576 compresscheck = CheckCompressionType(infile, formatspecs, True);
4577 if(not compresscheck):
4578 fextname = os.path.splitext(infile)[1];
4579 if(fextname==".gz"):
4580 compresscheck = "gzip";
4581 elif(fextname==".bz2"):
4582 compresscheck = "bzip2";
4583 elif(fextname==".zst"):
4584 compresscheck = "zstd";
4585 elif(fextname==".lz4" or fextname==".clz4"):
4586 compresscheck = "lz4";
4587 elif(fextname==".lzo" or fextname==".lzop"):
4588 compresscheck = "lzo";
4589 elif(fextname==".lzma"):
4590 compresscheck = "lzma";
4591 elif(fextname==".xz"):
4592 compresscheck = "xz";
4593 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
4594 compresscheck = "zlib";
4595 else:
4596 return False;
4597 if(not compresscheck):
4598 return False;
4599 catfp = UncompressFile(infile, formatspecs, "rb");
4600 '''
4601 try:
4602 catfp.seek(0, 2);
4603 except OSError:
4604 SeekToEndOfFile(catfp);
4605 except ValueError:
4606 SeekToEndOfFile(catfp);
4607 CatSize = catfp.tell();
4608 CatSizeEnd = CatSize;
4609 '''
4610 try:
4611 catfp.seek(0, 0);
4612 except OSError:
4613 return False;
4614 except ValueError:
4615 return False;
4616 curloc = catfp.tell();
4617 if(curloc>0):
4618 catfp.seek(0, 0);
4619 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4620 if(curloc>0):
4621 catfp.seek(curloc, 0);
4622 catstring = catheader[0];
4623 catversion = re.findall(r"([\d]+)", catstring);
4624 fprenumfiles = catheader[1];
4625 fnumfiles = int(fprenumfiles, 16);
4626 fprechecksumtype = catheader[2];
4627 fprechecksum = catheader[3];
4628 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4629 fnumfileshex = format(int(fnumfiles), 'x').lower();
4630 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4631 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4632 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4633 fheadtell = len(fileheader);
4634 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4635 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4636 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4637 return False;
4638 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4639 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4640 if(seekto>=fnumfiles):
4641 seekto = fnumfiles - 1;
4642 if(seekto<0):
4643 seekto = 0;
4644 if(seekto>=0):
4645 il = -1;
4646 while(il < seekto):
4647 prefhstart = catfp.tell();
4648 if(formatspecs['new_style']):
4649 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4650 else:
4651 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4652 if(len(preheaderdata)==0):
4653 break;
4654 prefheadsize = int(preheaderdata[0], 16);
4655 prefnumfields = int(preheaderdata[1], 16);
4656 preftype = int(preheaderdata[2], 16);
4657 if(re.findall(r"^[.|/]", preheaderdata[3])):
4658 prefname = preheaderdata[3];
4659 else:
4660 prefname = "./"+preheaderdata[3];
4661 prefbasedir = os.path.dirname(prefname);
4662 preflinkname = preheaderdata[4];
4663 prefsize = int(preheaderdata[5], 16);
4664 prefatime = int(preheaderdata[6], 16);
4665 prefmtime = int(preheaderdata[7], 16);
4666 prefctime = int(preheaderdata[8], 16);
4667 prefbtime = int(preheaderdata[9], 16);
4668 prefmode = int(preheaderdata[10], 16);
4669 prefchmode = stat.S_IMODE(prefmode);
4670 preftypemod = stat.S_IFMT(prefmode);
4671 prefwinattributes = int(preheaderdata[11], 16);
4672 prefcompression = preheaderdata[12];
4673 prefcsize = int(preheaderdata[13], 16);
4674 prefuid = int(preheaderdata[14], 16);
4675 prefuname = preheaderdata[15];
4676 prefgid = int(preheaderdata[16], 16);
4677 prefgname = preheaderdata[17];
4678 fid = int(preheaderdata[18], 16);
4679 finode = int(preheaderdata[19], 16);
4680 flinkcount = int(preheaderdata[20], 16);
4681 prefdev_minor = int(preheaderdata[21], 16);
4682 prefdev_major = int(preheaderdata[22], 16);
4683 prefrdev_minor = int(preheaderdata[23], 16);
4684 prefrdev_major = int(preheaderdata[24], 16);
4685 prefseeknextfile = preheaderdata[25];
4686 prefextrasize = int(preheaderdata[26], 16);
4687 prefextrafields = int(preheaderdata[27], 16);
4688 extrafieldslist = [];
4689 extrastart = 28;
4690 extraend = extrastart + prefextrafields;
4691 extrafieldslist = [];
4692 if(extrastart<extraend):
4693 extrafieldslist.append(preheaderdata[extrastart]);
4694 extrastart = extrastart + 1;
4695 prefcs = preheaderdata[-2].lower();
4696 prenewfcs = preheaderdata[-1].lower();
4697 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4698 if(prefcs!=prenewfcs and not skipchecksum):
4699 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4700 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4701 return False;
4702 valid_archive = False;
4703 invalid_archive = True;
4704 prefhend = catfp.tell() - 1;
4705 prefcontentstart = catfp.tell();
4706 prefcontents = "";
4707 pyhascontents = False;
4708 if(prefsize>0):
4709 if(prefcompression):
4710 prefcontents = catfp.read(prefsize);
4711 else:
4712 prefcontents = catfp.read(prefcsize);
4713 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4714 pyhascontents = True;
4715 if(prefccs!=prenewfccs and not skipchecksum):
4716 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4717 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4718 return False;
4719 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4720 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4721 if(abs(fseeknextasnum)==0):
4722 pass;
4723 catfp.seek(fseeknextasnum, 1);
4724 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4725 fseeknextasnum = int(prefseeknextfile);
4726 if(abs(fseeknextasnum)==0):
4727 pass;
4728 catfp.seek(fseeknextasnum, 1);
4729 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4730 fseeknextasnum = int(prefseeknextfile);
4731 if(abs(fseeknextasnum)==0):
4732 pass;
4733 catfp.seek(fseeknextasnum, 0);
4734 else:
4735 return False;
4736 il = il + 1;
4737 catfp.seek(seekstart, 0);
4738 fileidnum = il;
4739 catfheadsize = int(preheaderdata[0], 16);
4740 catfnumfields = int(preheaderdata[1], 16);
4741 catftype = int(preheaderdata[2], 16);
4742 if(re.findall(r"^[.|/]", preheaderdata[3])):
4743 catfname = preheaderdata[3];
4744 else:
4745 catfname = "./"+preheaderdata[3];
4746 catflinkname = preheaderdata[4];
4747 catfsize = int(preheaderdata[5], 16);
4748 catfbasedir = os.path.dirname(catfname);
4749 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4750 if(returnfp):
4751 catlist.update({'catfp': catfp});
4752 else:
4753 catfp.close();
4754 return catlist;
4755
4756 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4757
4758 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4759 formatspecs = FormatSpecsListToDict(formatspecs);
4760 if(hasattr(infile, "read") or hasattr(infile, "write")):
4761 catfp = infile;
4762 catfp.seek(0, 0);
4763 catfp = UncompressArchiveFile(catfp, formatspecs);
4764 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4765 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4766 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4767 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4768 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4769 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4770 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4771 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4772 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4773 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4774 return False;
4775 if(not catfp):
4776 return False;
4777 catfp.seek(0, 0);
4778 elif(infile=="-"):
4779 catfp = BytesIO();
4780 if(hasattr(sys.stdin, "buffer")):
4781 shutil.copyfileobj(sys.stdin.buffer, catfp);
4782 else:
4783 shutil.copyfileobj(sys.stdin, catfp);
4784 catfp.seek(0, 0);
4785 catfp = UncompressArchiveFile(catfp, formatspecs);
4786 if(not catfp):
4787 return False;
4788 catfp.seek(0, 0);
4789 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4790 catfp = download_file_from_internet_file(infile);
4791 catfp = UncompressArchiveFile(catfp, formatspecs);
4792 catfp.seek(0, 0);
4793 if(not catfp):
4794 return False;
4795 catfp.seek(0, 0);
4796 else:
4797 infile = RemoveWindowsPath(infile);
4798 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4799 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4800 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4801 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4802 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4803 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4804 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4805 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4806 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4807 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4808 return False;
4809 compresscheck = CheckCompressionType(infile, formatspecs, True);
4810 if(not compresscheck):
4811 fextname = os.path.splitext(infile)[1];
4812 if(fextname==".gz"):
4813 compresscheck = "gzip";
4814 elif(fextname==".bz2"):
4815 compresscheck = "bzip2";
4816 elif(fextname==".zst"):
4817 compresscheck = "zstd";
4818 elif(fextname==".lz4" or fextname==".clz4"):
4819 compresscheck = "lz4";
4820 elif(fextname==".lzo" or fextname==".lzop"):
4821 compresscheck = "lzo";
4822 elif(fextname==".lzma"):
4823 compresscheck = "lzma";
4824 elif(fextname==".xz"):
4825 compresscheck = "xz";
4826 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
4827 compresscheck = "zlib";
4828 else:
4829 return False;
4830 if(not compresscheck):
4831 return False;
4832 catfp = UncompressFile(infile, formatspecs, "rb");
4833 '''
4834 try:
4835 catfp.seek(0, 2);
4836 except OSError:
4837 SeekToEndOfFile(catfp);
4838 except ValueError:
4839 SeekToEndOfFile(catfp);
4840 CatSize = catfp.tell();
4841 CatSizeEnd = CatSize;
4842 '''
4843 try:
4844 catfp.seek(0, 0);
4845 except OSError:
4846 return False;
4847 except ValueError:
4848 return False;
4849 curloc = catfp.tell();
4850 if(curloc>0):
4851 catfp.seek(0, 0);
4852 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4853 if(curloc>0):
4854 catfp.seek(curloc, 0);
4855 catstring = catheader[0];
4856 catversion = re.findall(r"([\d]+)", catstring);
4857 fprenumfiles = catheader[1];
4858 fnumfiles = int(fprenumfiles, 16);
4859 fprechecksumtype = catheader[2];
4860 fprechecksum = catheader[3];
4861 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4862 fnumfileshex = format(int(fnumfiles), 'x').lower();
4863 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4864 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4865 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4866 fheadtell = len(fileheader);
4867 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4868 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4869 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4870 return False;
4871 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4872 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4873 seekto = fnumfiles - 1
4874 filefound = False;
4875 if(seekto>=0):
4876 il = -1;
4877 while(il < seekto):
4878 prefhstart = catfp.tell();
4879 if(formatspecs['new_style']):
4880 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4881 else:
4882 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4883 if(len(preheaderdata)==0):
4884 break;
4885 prefheadsize = int(preheaderdata[0], 16);
4886 prefnumfields = int(preheaderdata[1], 16);
4887 preftype = int(preheaderdata[2], 16);
4888 if(re.findall(r"^[.|/]", preheaderdata[3])):
4889 prefname = preheaderdata[3];
4890 else:
4891 prefname = "./"+preheaderdata[3];
4892 prefbasedir = os.path.dirname(prefname);
4893 preflinkname = preheaderdata[4];
4894 prefsize = int(preheaderdata[5], 16);
4895 prefatime = int(preheaderdata[6], 16);
4896 prefmtime = int(preheaderdata[7], 16);
4897 prefctime = int(preheaderdata[8], 16);
4898 prefbtime = int(preheaderdata[9], 16);
4899 prefmode = int(preheaderdata[10], 16);
4900 prefchmode = stat.S_IMODE(prefmode);
4901 preftypemod = stat.S_IFMT(prefmode);
4902 prefwinattributes = int(preheaderdata[11], 16);
4903 prefcompression = preheaderdata[12];
4904 prefcsize = int(preheaderdata[13], 16);
4905 prefuid = int(preheaderdata[14], 16);
4906 prefuname = preheaderdata[15];
4907 prefgid = int(preheaderdata[16], 16);
4908 prefgname = preheaderdata[17];
4909 fid = int(preheaderdata[18], 16);
4910 finode = int(preheaderdata[19], 16);
4911 flinkcount = int(preheaderdata[20], 16);
4912 prefdev_minor = int(preheaderdata[21], 16);
4913 prefdev_major = int(preheaderdata[22], 16);
4914 prefrdev_minor = int(preheaderdata[23], 16);
4915 prefrdev_major = int(preheaderdata[24], 16);
4916 prefseeknextfile = preheaderdata[25];
4917 prefextrasize = int(preheaderdata[26], 16);
4918 prefextrafields = int(preheaderdata[27], 16);
4919 extrafieldslist = [];
4920 extrastart = 28;
4921 extraend = extrastart + prefextrafields;
4922 extrafieldslist = [];
4923 if(extrastart<extraend):
4924 extrafieldslist.append(preheaderdata[extrastart]);
4925 extrastart = extrastart + 1;
4926 prefcs = preheaderdata[-2].lower();
4927 prenewfcs = preheaderdata[-1].lower();
4928 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4929 if(prefcs!=prenewfcs and not skipchecksum):
4930 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4931 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4932 return False;
4933 valid_archive = False;
4934 invalid_archive = True;
4935 prefhend = catfp.tell() - 1;
4936 prefcontentstart = catfp.tell();
4937 prefcontents = "";
4938 pyhascontents = False;
4939 if(prefsize>0):
4940 if(prefcompression):
4941 prefcontents = catfp.read(prefsize);
4942 else:
4943 prefcontents = catfp.read(prefcsize);
4944 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4945 pyhascontents = True;
4946 if(prefccs!=prenewfccs and not skipchecksum):
4947 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4948 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4949 return False;
4950 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4951 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4952 if(abs(fseeknextasnum)==0):
4953 pass;
4954 catfp.seek(fseeknextasnum, 1);
4955 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4956 fseeknextasnum = int(prefseeknextfile);
4957 if(abs(fseeknextasnum)==0):
4958 pass;
4959 catfp.seek(fseeknextasnum, 1);
4960 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4961 fseeknextasnum = int(prefseeknextfile);
4962 if(abs(fseeknextasnum)==0):
4963 pass;
4964 catfp.seek(fseeknextasnum, 0);
4965 else:
4966 return False;
4967 il = il + 1;
4968 filefound = False;
4969 prefname = preheaderdata[2];
4970 if(re.findall(r"^[.|/]", preheaderdata[2])):
4971 prefname = preheaderdata[2];
4972 else:
4973 prefname = "./"+preheaderdata[2];
4974 if(prefname==seekfile):
4975 filefound = True;
4976 break;
4977 catfp.seek(seekstart, 0);
4978 fileidnum = il;
4979 catfheadsize = int(preheaderdata[0], 16);
4980 catfnumfields = int(preheaderdata[1], 16);
4981 catftype = int(preheaderdata[2], 16);
4982 if(re.findall(r"^[.|/]", preheaderdata[3])):
4983 catfname = preheaderdata[3];
4984 else:
4985 catfname = "./"+preheaderdata[3];
4986 catflinkname = preheaderdata[4];
4987 catfsize = int(preheaderdata[5], 16);
4988 catfbasedir = os.path.dirname(catfname);
4989 if(filefound):
4990 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4991 else:
4992 return False;
4993 if(returnfp):
4994 catlist.update({'catfp': catfp});
4995 else:
4996 catfp.close();
4997 return catlist;
4998
4999 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
5000
5001 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
5002 formatspecs = FormatSpecsListToDict(formatspecs);
5003 if(verbose):
5004 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5005 if(hasattr(infile, "read") or hasattr(infile, "write")):
5006 catfp = infile;
5007 catfp.seek(0, 0);
5008 catfp = UncompressArchiveFile(catfp, formatspecs);
5009 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
5010 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5011 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5012 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5013 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5014 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5015 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5016 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5017 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5018 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5019 return False;
5020 if(not catfp):
5021 return False;
5022 catfp.seek(0, 0);
5023 elif(infile=="-"):
5024 catfp = BytesIO();
5025 if(hasattr(sys.stdin, "buffer")):
5026 shutil.copyfileobj(sys.stdin.buffer, catfp);
5027 else:
5028 shutil.copyfileobj(sys.stdin, catfp);
5029 catfp.seek(0, 0);
5030 catfp = UncompressArchiveFile(catfp, formatspecs);
5031 if(not catfp):
5032 return False;
5033 catfp.seek(0, 0);
5034 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5035 catfp = download_file_from_internet_file(infile);
5036 catfp = UncompressArchiveFile(catfp, formatspecs);
5037 catfp.seek(0, 0);
5038 if(not catfp):
5039 return False;
5040 catfp.seek(0, 0);
5041 else:
5042 infile = RemoveWindowsPath(infile);
5043 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5044 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5045 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5046 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5047 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5048 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5049 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5050 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5051 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
5052 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5053 return False;
5054 compresscheck = CheckCompressionType(infile, formatspecs, True);
5055 if(not compresscheck):
5056 fextname = os.path.splitext(infile)[1];
5057 if(fextname==".gz"):
5058 compresscheck = "gzip";
5059 elif(fextname==".bz2"):
5060 compresscheck = "bzip2";
5061 elif(fextname==".zst"):
5062 compresscheck = "zstd";
5063 elif(fextname==".lz4" or fextname==".clz4"):
5064 compresscheck = "lz4";
5065 elif(fextname==".lzo" or fextname==".lzop"):
5066 compresscheck = "lzo";
5067 elif(fextname==".lzma"):
5068 compresscheck = "lzma";
5069 elif(fextname==".xz"):
5070 compresscheck = "xz";
5071 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
5072 compresscheck = "zlib";
5073 else:
5074 return False;
5075 if(not compresscheck):
5076 return False;
5077 catfp = UncompressFile(infile, formatspecs, "rb");
5078 '''
5079 try:
5080 catfp.seek(0, 2);
5081 except OSError:
5082 SeekToEndOfFile(catfp);
5083 except ValueError:
5084 SeekToEndOfFile(catfp);
5085 CatSize = catfp.tell();
5086 CatSizeEnd = CatSize;
5087 '''
5088 try:
5089 catfp.seek(0, 0);
5090 except OSError:
5091 return False;
5092 except ValueError:
5093 return False;
5094 curloc = catfp.tell();
5095 if(curloc>0):
5096 catfp.seek(0, 0);
5097 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
5098 if(curloc>0):
5099 catfp.seek(curloc, 0);
5100 catstring = catheader[0];
5101 catversion = re.findall(r"([\d]+)", catstring);
5102 fprenumfiles = catheader[1];
5103 fnumfiles = int(fprenumfiles, 16);
5104 fprechecksumtype = catheader[2];
5105 fprechecksum = catheader[3];
5106 il = 0;
5107 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
5108 fnumfileshex = format(int(fnumfiles), 'x').lower();
5109 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
5110 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5111 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5112 valid_archive = True;
5113 invalid_archive = False;
5114 if(verbose):
5115 if(hasattr(infile, "read") or hasattr(infile, "write")):
5116 try:
5117 VerbosePrintOut(infile.name);
5118 except AttributeError:
5119 VerbosePrintOut(infile);
5120 else:
5121 VerbosePrintOut(infile);
5122 VerbosePrintOut("Number of Records " + str(fnumfiles));
5123 if(fprechecksum==catfileheadercshex):
5124 if(verbose):
5125 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
5126 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
5127 else:
5128 if(verbose):
5129 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
5130 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
5131 valid_archive = False;
5132 invalid_archive = True;
5133 if(verbose):
5134 VerbosePrintOut("");
5135 while(il<fnumfiles):
5136 catfhstart = catfp.tell();
5137 if(formatspecs['new_style']):
5138 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5139 else:
5140 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5141 if(len(catheaderdata)==0):
5142 break;
5143 catfheadsize = int(catheaderdata[0], 16);
5144 catfnumfields = int(catheaderdata[1], 16);
5145 catftype = int(catheaderdata[2], 16);
5146 if(re.findall(r"^[.|/]", catheaderdata[3])):
5147 catfname = catheaderdata[3];
5148 else:
5149 catfname = "./"+catheaderdata[3];
5150 catfbasedir = os.path.dirname(catfname);
5151 catflinkname = catheaderdata[4];
5152 catfsize = int(catheaderdata[5], 16);
5153 catfatime = int(catheaderdata[6], 16);
5154 catfmtime = int(catheaderdata[7], 16);
5155 catfctime = int(catheaderdata[8], 16);
5156 catfbtime = int(catheaderdata[9], 16);
5157 catfmode = int(catheaderdata[10], 16);
5158 catfchmode = stat.S_IMODE(catfmode);
5159 catftypemod = stat.S_IFMT(catfmode);
5160 catfwinattributes = int(catheaderdata[11], 16);
5161 catfcompression = catheaderdata[12];
5162 catfcsize = int(catheaderdata[13], 16);
5163 catfuid = int(catheaderdata[14], 16);
5164 catfuname = catheaderdata[15];
5165 catfgid = int(catheaderdata[16], 16);
5166 catfgname = catheaderdata[17];
5167 fid = int(catheaderdata[18], 16);
5168 finode = int(catheaderdata[19], 16);
5169 flinkcount = int(catheaderdata[20], 16);
5170 catfdev_minor = int(catheaderdata[21], 16);
5171 catfdev_major = int(catheaderdata[22], 16);
5172 catfrdev_minor = int(catheaderdata[23], 16);
5173 catfrdev_major = int(catheaderdata[24], 16);
5174 catfseeknextfile = catheaderdata[25];
5175 catfextrasize = int(catheaderdata[26], 16);
5176 catfextrafields = int(catheaderdata[27], 16);
5177 extrafieldslist = [];
5178 extrastart = 28;
5179 extraend = extrastart + catfextrafields;
5180 extrafieldslist = [];
5181 if(extrastart<extraend):
5182 extrafieldslist.append(catheaderdata[extrastart]);
5183 extrastart = extrastart + 1;
5184 catfcs = catheaderdata[-2].lower();
5185 catfccs = catheaderdata[-1].lower();
5186 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5187 if(verbose):
5188 VerbosePrintOut(catfname);
5189 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
5190 if(catfcs==catnewfcs):
5191 if(verbose):
5192 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
5193 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
5194 else:
5195 if(verbose):
5196 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
5197 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5198 valid_archive = False;
5199 invalid_archive = True;
5200 catfhend = catfp.tell() - 1;
5201 catfcontentstart = catfp.tell();
5202 catfcontents = "";
5203 pyhascontents = False;
5204 if(catfsize>0):
5205 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5206 catfcontents = catfp.read(catfsize);
5207 else:
5208 catfcontents = catfp.read(catfcsize);
5209 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
5210 pyhascontents = True;
5211 if(catfccs==catnewfccs):
5212 if(verbose):
5213 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
5214 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
5215 else:
5216 if(verbose):
5217 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
5218 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5219 valid_archive = False;
5220 invalid_archive = True;
5221 if(verbose):
5222 VerbosePrintOut("");
5223 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5224 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5225 if(abs(fseeknextasnum)==0):
5226 pass;
5227 catfp.seek(fseeknextasnum, 1);
5228 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5229 fseeknextasnum = int(catfseeknextfile);
5230 if(abs(fseeknextasnum)==0):
5231 pass;
5232 catfp.seek(fseeknextasnum, 1);
5233 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5234 fseeknextasnum = int(catfseeknextfile);
5235 if(abs(fseeknextasnum)==0):
5236 pass;
5237 catfp.seek(fseeknextasnum, 0);
5238 else:
5239 return False;
5240 il = il + 1;
5241 if(valid_archive):
5242 if(returnfp):
5243 return catfp;
5244 else:
5245 catfp.close();
5246 return True;
5247 else:
5248 catfp.close();
5249 return False;
5250
5251 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
5252
5253 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5254 formatspecs = FormatSpecsListToDict(formatspecs);
5255 if(hasattr(infile, "read") or hasattr(infile, "write")):
5256 catfp = infile;
5257 catfp.seek(0, 0);
5258 catfp = UncompressArchiveFile(catfp, formatspecs);
5259 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
5260 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5261 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5262 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5263 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5264 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5265 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5266 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5267 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5268 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5269 return False;
5270 if(not catfp):
5271 return False;
5272 catfp.seek(0, 0);
5273 elif(infile=="-"):
5274 catfp = BytesIO();
5275 if(hasattr(sys.stdin, "buffer")):
5276 shutil.copyfileobj(sys.stdin.buffer, catfp);
5277 else:
5278 shutil.copyfileobj(sys.stdin, catfp);
5279 catfp.seek(0, 0);
5280 catfp = UncompressArchiveFile(catfp, formatspecs);
5281 if(not catfp):
5282 return False;
5283 catfp.seek(0, 0);
5284 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5285 catfp = download_file_from_internet_file(infile);
5286 catfp = UncompressArchiveFile(catfp, formatspecs);
5287 catfp.seek(0, 0);
5288 if(not catfp):
5289 return False;
5290 catfp.seek(0, 0);
5291 else:
5292 infile = RemoveWindowsPath(infile);
5293 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5294 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5295 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5296 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5297 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5298 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5299 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5300 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5301 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5302 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5303 return False;
5304 compresscheck = CheckCompressionType(infile, formatspecs, True);
5305 if(not compresscheck):
5306 fextname = os.path.splitext(infile)[1];
5307 if(fextname==".gz"):
5308 compresscheck = "gzip";
5309 elif(fextname==".bz2"):
5310 compresscheck = "bzip2";
5311 elif(fextname==".zst"):
5312 compresscheck = "zstd";
5313 elif(fextname==".lz4" or fextname==".clz4"):
5314 compresscheck = "lz4";
5315 elif(fextname==".lzo" or fextname==".lzop"):
5316 compresscheck = "lzo";
5317 elif(fextname==".lzma"):
5318 compresscheck = "lzma";
5319 elif(fextname==".xz"):
5320 compresscheck = "xz";
5321 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
5322 compresscheck = "zlib";
5323 else:
5324 return False;
5325 if(not compresscheck):
5326 return False;
5327 catfp = UncompressFile(infile, formatspecs, "rb");
5328 '''
5329 try:
5330 catfp.seek(0, 2);
5331 except OSError:
5332 SeekToEndOfFile(catfp);
5333 except ValueError:
5334 SeekToEndOfFile(catfp);
5335 CatSize = catfp.tell();
5336 CatSizeEnd = CatSize;
5337 '''
5338 try:
5339 catfp.seek(0, 0);
5340 except OSError:
5341 return False;
5342 except ValueError:
5343 return False;
5344 curloc = catfp.tell();
5345 if(curloc>0):
5346 catfp.seek(0, 0);
5347 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
5348 if(curloc>0):
5349 catfp.seek(curloc, 0);
5350 catstring = catheader[0];
5351 catversion = re.findall(r"([\d]+)", catstring);
5352 fprenumfiles = catheader[1];
5353 fnumfiles = int(fprenumfiles, 16);
5354 fprechecksumtype = catheader[2];
5355 fprechecksum = catheader[3];
5356 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
5357 fnumfileshex = format(int(fnumfiles), 'x').lower();
5358 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
5359 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5360 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5361 fheadtell = len(fileheader);
5362 if(fprechecksum!=catfileheadercshex and not skipchecksum):
5363 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
5364 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
5365 return False;
5366 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
5367 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
5368 if(seekstart<0 and seekstart>fnumfiles):
5369 seekstart = 0;
5370 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
5371 seekend = fnumfiles;
5372 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
5373 seekend = fnumfiles - abs(seekend);
5374 if(seekstart>0):
5375 il = 0;
5376 while(il < seekstart):
5377 prefhstart = catfp.tell();
5378 if(formatspecs['new_style']):
5379 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5380 else:
5381 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5382 if(len(preheaderdata)==0):
5383 break;
5384 prefheadsize = int(preheaderdata[0], 16);
5385 prefnumfields = int(preheaderdata[1], 16);
5386 if(re.findall(r"^[.|/]", preheaderdata[3])):
5387 prefname = preheaderdata[3];
5388 else:
5389 prefname = "./"+preheaderdata[3];
5390 prefsize = int(preheaderdata[5], 16);
5391 prefcompression = preheaderdata[12];
5392 prefcsize = int(preheaderdata[13], 16);
5393 prefseeknextfile = preheaderdata[25];
5394 prefextrasize = int(preheaderdata[26], 16);
5395 prefextrafields = int(preheaderdata[27], 16);
5396 extrafieldslist = [];
5397 extrastart = 28;
5398 extraend = extrastart + prefextrafields;
5399 extrafieldslist = [];
5400 if(extrastart<extraend):
5401 extrafieldslist.append(preheaderdata[extrastart]);
5402 extrastart = extrastart + 1;
5403 prefcs = preheaderdata[-2].lower();
5404 prenewfcs = preheaderdata[-1].lower();
5405 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
5406 if(prefcs!=prenewfcs and not skipchecksum):
5407 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
5408 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
5409 return False;
5410 valid_archive = False;
5411 invalid_archive = True;
5412 prefhend = catfp.tell() - 1;
5413 prefcontentstart = catfp.tell();
5414 prefcontents = "";
5415 pyhascontents = False;
5416 if(prefsize>0):
5417 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
5418 prefcontents = catfp.read(prefsize);
5419 else:
5420 prefcontents = catfp.read(prefcsize);
5421 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
5422 pyhascontents = True;
5423 if(prefccs!=prenewfccs and not skipchecksum):
5424 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
5425 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
5426 return False;
5427 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
5428 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
5429 if(abs(fseeknextasnum)==0):
5430 pass;
5431 catfp.seek(fseeknextasnum, 1);
5432 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
5433 fseeknextasnum = int(prefseeknextfile);
5434 if(abs(fseeknextasnum)==0):
5435 pass;
5436 catfp.seek(fseeknextasnum, 1);
5437 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
5438 fseeknextasnum = int(prefseeknextfile);
5439 if(abs(fseeknextasnum)==0):
5440 pass;
5441 catfp.seek(fseeknextasnum, 0);
5442 else:
5443 return False;
5444 il = il + 1;
5445 fileidnum = seekstart;
5446 realidnum = 0;
5447 while(fileidnum<seekend):
5448 catfhstart = catfp.tell();
5449 if(formatspecs['new_style']):
5450 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5451 else:
5452 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5453 if(len(catheaderdata)==0):
5454 break;
5455 catfheadsize = int(catheaderdata[0], 16);
5456 catfnumfields = int(catheaderdata[1], 16);
5457 catftype = int(catheaderdata[2], 16);
5458 if(re.findall(r"^[.|/]", catheaderdata[3])):
5459 catfname = catheaderdata[3];
5460 else:
5461 catfname = "./"+catheaderdata[3];
5462 catfbasedir = os.path.dirname(catfname);
5463 catflinkname = catheaderdata[4];
5464 catfsize = int(catheaderdata[5], 16);
5465 catfatime = int(catheaderdata[6], 16);
5466 catfmtime = int(catheaderdata[7], 16);
5467 catfctime = int(catheaderdata[8], 16);
5468 catfbtime = int(catheaderdata[9], 16);
5469 catfmode = int(catheaderdata[10], 16);
5470 catfchmode = stat.S_IMODE(catfmode);
5471 catftypemod = stat.S_IFMT(catfmode);
5472 catfwinattributes = int(catheaderdata[11], 16);
5473 catfcompression = catheaderdata[12];
5474 catfcsize = int(catheaderdata[13], 16);
5475 catfuid = int(catheaderdata[14], 16);
5476 catfuname = catheaderdata[15];
5477 catfgid = int(catheaderdata[16], 16);
5478 catfgname = catheaderdata[17];
5479 catfid = int(catheaderdata[18], 16);
5480 catfinode = int(catheaderdata[19], 16);
5481 catflinkcount = int(catheaderdata[20], 16);
5482 catfdev_minor = int(catheaderdata[21], 16);
5483 catfdev_major = int(catheaderdata[22], 16);
5484 catfrdev_minor = int(catheaderdata[23], 16);
5485 catfrdev_major = int(catheaderdata[24], 16);
5486 catfseeknextfile = catheaderdata[25];
5487 catfextrasize = int(catheaderdata[26], 16);
5488 catfextrafields = int(catheaderdata[27], 16);
5489 extrafieldslist = [];
5490 extrastart = 28;
5491 extraend = extrastart + catfextrafields;
5492 extrafieldslist = [];
5493 if(extrastart<extraend):
5494 extrafieldslist.append(catheaderdata[extrastart]);
5495 extrastart = extrastart + 1;
5496 catfcs = catheaderdata[-2].lower();
5497 catfccs = catheaderdata[-1].lower();
5498 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5499 if(catfcs!=catnewfcs and not skipchecksum):
5500 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5501 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5502 return False;
5503 catfhend = catfp.tell() - 1;
5504 catfcontentstart = catfp.tell();
5505 catfcontents = BytesIO();
5506 pyhascontents = False;
5507 if(catfsize>0 and not listonly):
5508 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5509 catfcontents.write(catfp.read(catfsize));
5510 else:
5511 catfcontents.write(catfp.read(catfcsize));
5512 catfcontents.seek(0, 0);
5513 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5514 pyhascontents = True;
5515 if(catfccs!=catnewfccs and skipchecksum):
5516 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5517 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5518 return False;
5519 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5520 pass;
5521 else:
5522 catfcontents.seek(0, 0);
5523 if(uncompress):
5524 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5525 catfcontents.seek(0, 0);
5526 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5527 if(catfsize>0 and listonly):
5528 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5529 catfp.seek(catfsize, 1);
5530 else:
5531 catfp.seek(catfcsize, 1);
5532 pyhascontents = False;
5533 catfcontentend = catfp.tell();
5534 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5535 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5536 if(abs(fseeknextasnum)==0):
5537 pass;
5538 catfp.seek(fseeknextasnum, 1);
5539 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5540 fseeknextasnum = int(catfseeknextfile);
5541 if(abs(fseeknextasnum)==0):
5542 pass;
5543 catfp.seek(fseeknextasnum, 1);
5544 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5545 fseeknextasnum = int(catfseeknextfile);
5546 if(abs(fseeknextasnum)==0):
5547 pass;
5548 catfp.seek(fseeknextasnum, 0);
5549 else:
5550 return False;
5551 catfcontents.seek(0, 0);
5552 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5553 fileidnum = fileidnum + 1;
5554 realidnum = realidnum + 1;
5555 if(returnfp):
5556 catlist.update({'catfp': catfp});
5557 else:
5558 catfp.close();
5559 return catlist;
5560
5561 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5562
5563 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5564 formatspecs = FormatSpecsListToDict(formatspecs);
5565 catfp = BytesIO(catstr);
5566 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5567 return listcatfiles;
5568
5569 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5570
5571 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5572 formatspecs = FormatSpecsListToDict(formatspecs);
5573 catfp = BytesIO();
5574 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5575 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5576 return listcatfiles;
5577
5578 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5579 formatspecs = FormatSpecsListToDict(formatspecs);
5580 catfp = BytesIO();
5581 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5582 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5583 return listcatfiles;
5584
5585 if(not rarfile_support):
5586 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5587 return False;
5588
5589 if(rarfile_support):
5590 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5591 formatspecs = FormatSpecsListToDict(formatspecs);
5592 catfp = BytesIO();
5593 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5594 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5595 return listcatfiles;
5596
5597 if(not py7zr_support):
5598 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5599 return False;
5600
5601 if(py7zr_support):
5602 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5603 formatspecs = FormatSpecsListToDict(formatspecs);
5604 catfp = BytesIO();
5605 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5606 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5607 return listcatfiles;
5608
5609 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5610 formatspecs = FormatSpecsListToDict(formatspecs);
5611 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5612 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5613 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5614 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5615 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5616 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5617 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5618 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5619 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5620 elif(checkcompressfile=="catfile"):
5621 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5622 else:
5623 return False;
5624 return False;
5625
5626 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5627 formatspecs = FormatSpecsListToDict(formatspecs);
5628 catver = formatspecs['format_ver'];
5629 fileheaderver = str(int(catver.replace(".", "")));
5630 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5631 advancedlist = formatspecs['use_advanced_list'];
5632 altinode = formatspecs['use_alt_inode'];
5633 infilelist = [];
5634 if(infiles=="-"):
5635 for line in sys.stdin:
5636 infilelist.append(line.strip());
5637 infilelist = list(filter(None, infilelist));
5638 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5639 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5640 return False;
5641 with UncompressFile(infiles, formatspecs, "r") as finfile:
5642 for line in finfile:
5643 infilelist.append(line.strip());
5644 infilelist = list(filter(None, infilelist));
5645 else:
5646 if(isinstance(infiles, (list, tuple, ))):
5647 infilelist = list(filter(None, infiles));
5648 elif(isinstance(infiles, (str, ))):
5649 infilelist = list(filter(None, [infiles]));
5650 if(advancedlist):
5651 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5652 else:
5653 GetDirList = ListDir(infilelist, followlink, False);
5654 if(not GetDirList):
5655 return False;
5656 curinode = 0;
5657 curfid = 0;
5658 inodelist = [];
5659 inodetofile = {};
5660 filetoinode = {};
5661 inodetocatinode = {};
5662 fileidnum = 0;
5663 fnumfiles = int(len(GetDirList));
5664 catver = formatspecs['format_ver'];
5665 fileheaderver = str(int(catver.replace(".", "")));
5666 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5667 fnumfileshex = format(int(fnumfiles), 'x').lower();
5668 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5669 catversion = re.findall(r"([\d]+)", fileheader);
5670 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5671 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5672 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5673 fheadtell = len(fileheader);
5674 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5675 for curfname in GetDirList:
5676 catfhstart = fheadtell;
5677 if(re.findall(r"^[.|/]", curfname)):
5678 fname = curfname;
5679 else:
5680 fname = "./"+curfname;
5681 if(verbose):
5682 VerbosePrintOut(fname);
5683 if(not followlink or followlink is None):
5684 fstatinfo = os.lstat(fname);
5685 else:
5686 fstatinfo = os.stat(fname);
5687 fpremode = fstatinfo.st_mode;
5688 finode = fstatinfo.st_ino;
5689 flinkcount = fstatinfo.st_nlink;
5690 ftype = 0;
5691 if(stat.S_ISREG(fpremode)):
5692 ftype = 0;
5693 elif(stat.S_ISLNK(fpremode)):
5694 ftype = 2;
5695 elif(stat.S_ISCHR(fpremode)):
5696 ftype = 3;
5697 elif(stat.S_ISBLK(fpremode)):
5698 ftype = 4;
5699 elif(stat.S_ISDIR(fpremode)):
5700 ftype = 5;
5701 elif(stat.S_ISFIFO(fpremode)):
5702 ftype = 6;
5703 elif(stat.S_ISSOCK(fpremode)):
5704 ftype = 8;
5705 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5706 ftype = 9;
5707 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5708 ftype = 10;
5709 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5710 ftype = 11;
5711 else:
5712 ftype = 0;
5713 flinkname = "";
5714 fbasedir = os.path.dirname(fname);
5715 fcurfid = curfid;
5716 if(not followlink and finode!=0):
5717 if(ftype!=1):
5718 if(finode in inodelist):
5719 ftype = 1;
5720 flinkname = inodetofile[finode];
5721 if(altinode):
5722 fcurinode = finode;
5723 else:
5724 fcurinode = inodetocatinode[finode];
5725 if(finode not in inodelist):
5726 inodelist.append(finode);
5727 inodetofile.update({finode: fname});
5728 inodetocatinode.update({finode: curinode});
5729 if(altinode):
5730 fcurinode = finode;
5731 else:
5732 fcurinode = curinode;
5733 curinode = curinode + 1;
5734 else:
5735 fcurinode = curinode;
5736 curinode = curinode + 1;
5737 curfid = curfid + 1;
5738 if(ftype==2):
5739 flinkname = os.readlink(fname);
5740 fdev = fstatinfo.st_dev;
5741 getfdev = GetDevMajorMinor(fdev);
5742 fdev_minor = getfdev[0];
5743 fdev_major = getfdev[1];
5744 frdev = fstatinfo.st_dev;
5745 if(hasattr(fstatinfo, "st_rdev")):
5746 frdev = fstatinfo.st_rdev;
5747 else:
5748 frdev = fstatinfo.st_dev;
5749 getfrdev = GetDevMajorMinor(frdev);
5750 frdev_minor = getfrdev[0];
5751 frdev_major = getfrdev[1];
5752 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5753 fsize = "0";
5754 if(ftype==0 or ftype==7):
5755 fsize = fstatinfo.st_size;
5756 fatime = fstatinfo.st_atime;
5757 fmtime = fstatinfo.st_mtime;
5758 fctime = fstatinfo.st_ctime;
5759 if(hasattr(fstatinfo, "st_birthtime")):
5760 fbtime = fstatinfo.st_birthtime;
5761 else:
5762 fbtime = fstatinfo.st_ctime;
5763 fmode = fstatinfo.st_mode;
5764 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5765 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5766 fuid = fstatinfo.st_uid;
5767 fgid = fstatinfo.st_gid;
5768 funame = "";
5769 try:
5770 import pwd;
5771 try:
5772 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5773 funame = userinfo.pw_name;
5774 except KeyError:
5775 funame = "";
5776 except ImportError:
5777 funame = "";
5778 fgname = "";
5779 try:
5780 import grp;
5781 try:
5782 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5783 fgname = groupinfo.gr_name;
5784 except KeyError:
5785 fgname = "";
5786 except ImportError:
5787 fgname = "";
5788 fdev_minor = fdev_minor;
5789 fdev_major = fdev_major;
5790 frdev_minor = frdev_minor;
5791 frdev_major = frdev_major;
5792 flinkcount = flinkcount;
5793 if(hasattr(fstatinfo, "st_file_attributes")):
5794 fwinattributes = fstatinfo.st_file_attributes;
5795 else:
5796 fwinattributes = 0;
5797 fcompression = "";
5798 fcsize = 0;
5799 fcontents = BytesIO();
5800 if(ftype==0 or ftype==7):
5801 with open(fname, "rb") as fpc:
5802 shutil.copyfileobj(fpc, fcontents);
5803 if(followlink and (ftype==1 or ftype==2)):
5804 flstatinfo = os.stat(flinkname);
5805 with open(flinkname, "rb") as fpc:
5806 shutil.copyfileobj(fpc, fcontents);
5807 fcontents.seek(0, 0);
5808 ftypehex = format(ftype, 'x').lower();
5809 extrafields = len(extradata);
5810 extrafieldslist = extradata;
5811 catfextrafields = extrafields;
5812 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5813 if(len(extradata)>0):
5814 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5815 extrasizelen = len(extrasizestr);
5816 extrasizelenhex = format(extrasizelen, 'x').lower();
5817 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5818 catoutlen = len(catoutlist) + len(extradata) + 3;
5819 catoutlenhex = format(catoutlen, 'x').lower();
5820 catoutlist.insert(0, catoutlenhex);
5821 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5822 catheaderdata = catoutlist;
5823 if(len(extradata)>0):
5824 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5825 if(fsize==0):
5826 checksumlist = [checksumtype, "none"];
5827 else:
5828 checksumlist = [checksumtype, checksumtype];
5829 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5830 catfnumfields = catoutlen;
5831 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5832 fcontents.seek(0, 0);
5833 if(fsize==0):
5834 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5835 else:
5836 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5837 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5838 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5839 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5840 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5841 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5842 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5843 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5844 catfcontentstart = fheadtell;
5845 fheadtell += len(catfileoutstr) + 1;
5846 catfcontentend = fheadtell - 1;
5847 catfhend = catfcontentend;
5848 fcontents.seek(0, 0);
5849 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5850 pyhascontents = False;
5851 if(int(fsize)>0 and not listonly):
5852 pyhascontents = True;
5853 if(int(fsize)>0 and listonly):
5854 fcontents = BytesIO();
5855 pyhascontents = False;
5856 fcontents.seek(0, 0);
5857 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5858 fileidnum = fileidnum + 1;
5859 return catlist;
5860
5861 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5862 formatspecs = FormatSpecsListToDict(formatspecs);
5863 curinode = 0;
5864 curfid = 0;
5865 inodelist = [];
5866 inodetofile = {};
5867 filetoinode = {};
5868 inodetocatinode = {};
5869 fileidnum = 0;
5870 if(infile=="-"):
5871 infile = BytesIO();
5872 if(hasattr(sys.stdin, "buffer")):
5873 shutil.copyfileobj(sys.stdin.buffer, infile);
5874 else:
5875 shutil.copyfileobj(sys.stdin, infile);
5876 infile.seek(0, 0);
5877 if(not infile):
5878 return False;
5879 infile.seek(0, 0);
5880 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5881 infile = download_file_from_internet_file(infile);
5882 infile.seek(0, 0);
5883 if(not infile):
5884 return False;
5885 infile.seek(0, 0);
5886 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5887 return False;
5888 elif(os.path.exists(infile) and os.path.isfile(infile)):
5889 try:
5890 if(not tarfile.TarFileCheck(infile)):
5891 return False;
5892 except AttributeError:
5893 if(not TarFileCheck(infile)):
5894 return False;
5895 try:
5896 if(hasattr(infile, "read") or hasattr(infile, "write")):
5897 tarfp = tarfile.open(fileobj=infile, mode="r");
5898 else:
5899 tarfp = tarfile.open(infile, "r");
5900 except FileNotFoundError:
5901 return False;
5902 fnumfiles = int(len(tarfp.getmembers()));
5903 catver = formatspecs['format_ver'];
5904 fileheaderver = str(int(catver.replace(".", "")));
5905 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5906 fnumfileshex = format(int(fnumfiles), 'x').lower();
5907 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5908 catversion = re.findall(r"([\d]+)", fileheader);
5909 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5910 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5911 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5912 fheadtell = len(fileheader);
5913 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5914 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5915 catfhstart = fheadtell;
5916 if(re.findall(r"^[.|/]", member.name)):
5917 fname = member.name;
5918 else:
5919 fname = "./"+member.name;
5920 if(verbose):
5921 VerbosePrintOut(fname);
5922 fpremode = member.mode;
5923 ffullmode = member.mode;
5924 flinkcount = 0;
5925 ftype = 0;
5926 if(member.isreg()):
5927 ffullmode = member.mode + stat.S_IFREG;
5928 ftype = 0;
5929 elif(member.isdev()):
5930 ffullmode = member.mode;
5931 ftype = 7;
5932 elif(member.islnk()):
5933 ffullmode = member.mode + stat.S_IFREG;
5934 ftype = 1;
5935 elif(member.issym()):
5936 ffullmode = member.mode + stat.S_IFLNK;
5937 ftype = 2;
5938 elif(member.ischr()):
5939 ffullmode = member.mode + stat.S_IFCHR;
5940 ftype = 3;
5941 elif(member.isblk()):
5942 ffullmode = member.mode + stat.S_IFBLK;
5943 ftype = 4;
5944 elif(member.isdir()):
5945 ffullmode = member.mode + stat.S_IFDIR;
5946 ftype = 5;
5947 elif(member.isfifo()):
5948 ffullmode = member.mode + stat.S_IFIFO;
5949 ftype = 6;
5950 elif(member.issparse()):
5951 ffullmode = member.mode;
5952 ftype = 12;
5953 else:
5954 ffullmode = member.mode;
5955 ftype = 0;
5956 flinkname = "";
5957 fbasedir = os.path.dirname(fname);
5958 fcurfid = curfid;
5959 fcurinode = curfid;
5960 finode = fcurinode;
5961 curfid = curfid + 1;
5962 if(ftype==2):
5963 flinkname = member.linkname;
5964 fdev_minor = member.devminor;
5965 fdev_major = member.devmajor;
5966 frdev_minor = member.devminor;
5967 frdev_major = member.devmajor;
5968 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5969 fsize = "0";
5970 elif(ftype==0 or ftype==7):
5971 fsize = member.size;
5972 else:
5973 fsize = member.size;
5974 fatime = member.mtime;
5975 fmtime = member.mtime;
5976 fctime = member.mtime;
5977 fbtime = member.mtime;
5978 fmode = ffullmode;
5979 fchmode = stat.S_IMODE(ffullmode);
5980 ftypemod = stat.S_IFMT(ffullmode);
5981 fuid = member.uid;
5982 fgid = member.gid;
5983 funame = member.uname;
5984 fgname = member.gname;
5985 flinkcount = flinkcount;
5986 fwinattributes = int(0);
5987 fcompression = "";
5988 fcsize = 0;
5989 fcontents = BytesIO();
5990 if(ftype==0 or ftype==7):
5991 with tarfp.extractfile(member) as fpc:
5992 shutil.copyfileobj(fpc, fcontents);
5993 fcontents.seek(0, 0);
5994 ftypehex = format(ftype, 'x').lower();
5995 extrafields = len(extradata);
5996 extrafieldslist = extradata;
5997 catfextrafields = extrafields;
5998 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5999 if(len(extradata)>0):
6000 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6001 extrasizelen = len(extrasizestr);
6002 extrasizelenhex = format(extrasizelen, 'x').lower();
6003 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6004 catoutlen = len(catoutlist) + len(extradata) + 3;
6005 catoutlenhex = format(catoutlen, 'x').lower();
6006 catoutlist.insert(0, catoutlenhex);
6007 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6008 catheaderdata = catoutlist;
6009 if(len(extradata)>0):
6010 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6011 if(fsize==0):
6012 checksumlist = [checksumtype, "none"];
6013 else:
6014 checksumlist = [checksumtype, checksumtype];
6015 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6016 catfnumfields = catoutlen;
6017 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6018 fcontents.seek(0, 0);
6019 if(fsize==0):
6020 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6021 else:
6022 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6023 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6024 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6025 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6026 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6027 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6028 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6029 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6030 catfcontentstart = fheadtell;
6031 fheadtell += len(catfileoutstr) + 1;
6032 catfcontentend = fheadtell - 1;
6033 catfhend = catfcontentend;
6034 fcontents.seek(0, 0);
6035 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6036 pyhascontents = False;
6037 if(int(fsize)>0 and not listonly):
6038 pyhascontents = True;
6039 if(int(fsize)>0 and listonly):
6040 fcontents = BytesIO();
6041 pyhascontents = False;
6042 fcontents.seek(0, 0);
6043 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6044 fileidnum = fileidnum + 1;
6045 return catlist;
6046
6047 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6048 formatspecs = FormatSpecsListToDict(formatspecs);
6049 curinode = 0;
6050 curfid = 0;
6051 inodelist = [];
6052 inodetofile = {};
6053 filetoinode = {};
6054 inodetocatinode = {};
6055 fileidnum = 0;
6056 if(infile=="-"):
6057 infile = BytesIO();
6058 if(hasattr(sys.stdin, "buffer")):
6059 shutil.copyfileobj(sys.stdin.buffer, infile);
6060 else:
6061 shutil.copyfileobj(sys.stdin, infile);
6062 infile.seek(0, 0);
6063 if(not infile):
6064 return False;
6065 infile.seek(0, 0);
6066 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6067 infile = download_file_from_internet_file(infile);
6068 infile.seek(0, 0);
6069 if(not infile):
6070 return False;
6071 infile.seek(0, 0);
6072 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6073 return False;
6074 if(not zipfile.is_zipfile(infile)):
6075 return False;
6076 try:
6077 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6078 except FileNotFoundError:
6079 return False;
6080 ziptest = zipfp.testzip();
6081 if(ziptest):
6082 VerbosePrintOut("Bad file found!");
6083 fnumfiles = int(len(zipfp.infolist()));
6084 catver = formatspecs['format_ver'];
6085 fileheaderver = str(int(catver.replace(".", "")));
6086 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6087 catversion = re.findall(r"([\d]+)", fileheader);
6088 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6089 fnumfileshex = format(int(fnumfiles), 'x').lower();
6090 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6091 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6092 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6093 fheadtell = len(fileheader);
6094 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6095 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6096 catfhstart = fheadtell;
6097 if(re.findall(r"^[.|/]", member.filename)):
6098 fname = member.filename;
6099 else:
6100 fname = "./"+member.filename;
6101 zipinfo = zipfp.getinfo(member.filename);
6102 if(verbose):
6103 VerbosePrintOut(fname);
6104 if(not member.is_dir()):
6105 fpremode = stat.S_IFREG + 438;
6106 elif(member.is_dir()):
6107 fpremode = stat.S_IFDIR + 511;
6108 flinkcount = 0;
6109 ftype = 0;
6110 if(not member.is_dir()):
6111 ftype = 0;
6112 elif(member.is_dir()):
6113 ftype = 5;
6114 flinkname = "";
6115 fbasedir = os.path.dirname(fname);
6116 fcurfid = curfid;
6117 fcurinode = curfid;
6118 finode = fcurinode;
6119 curfid = curfid + 1;
6120 fdev_minor = 0;
6121 fdev_major = 0;
6122 frdev_minor = 0;
6123 frdev_major = 0;
6124 if(ftype==5):
6125 fsize = "0";
6126 elif(ftype==0):
6127 fsize = member.file_size;
6128 else:
6129 fsize = member.file_size;
6130 fatime = time.mktime(member.date_time + (0, 0, -1));
6131 fmtime = time.mktime(member.date_time + (0, 0, -1));
6132 fctime = time.mktime(member.date_time + (0, 0, -1));
6133 fbtime = time.mktime(member.date_time + (0, 0, -1));
6134 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6135 fwinattributes = int(zipinfo.external_attr);
6136 if(not member.is_dir()):
6137 fmode = int(stat.S_IFREG + 438);
6138 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6139 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6140 elif(member.is_dir()):
6141 fmode = int(stat.S_IFDIR + 511);
6142 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6143 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6144 elif(zipinfo.create_system==3):
6145 fwinattributes = int(0);
6146 try:
6147 fmode = int(zipinfo.external_attr);
6148 fchmode = stat.S_IMODE(fmode);
6149 ftypemod = stat.S_IFMT(fmode);
6150 except OverflowError:
6151 fmode = int(zipinfo.external_attr >> 16);
6152 fchmode = stat.S_IMODE(fmode);
6153 ftypemod = stat.S_IFMT(fmode);
6154 else:
6155 fwinattributes = int(0);
6156 if(not member.is_dir()):
6157 fmode = int(stat.S_IFREG + 438);
6158 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6159 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6160 elif(member.is_dir()):
6161 fmode = int(stat.S_IFDIR + 511);
6162 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6163 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6164 fcompression = "";
6165 fcsize = 0;
6166 try:
6167 fuid = os.getuid();
6168 except AttributeError:
6169 fuid = 0;
6170 except KeyError:
6171 fuid = 0;
6172 try:
6173 fgid = os.getgid();
6174 except AttributeError:
6175 fgid = 0;
6176 except KeyError:
6177 fgid = 0;
6178 try:
6179 import pwd;
6180 try:
6181 userinfo = pwd.getpwuid(os.getuid());
6182 funame = userinfo.pw_name;
6183 except KeyError:
6184 funame = "";
6185 except AttributeError:
6186 funame = "";
6187 except ImportError:
6188 funame = "";
6189 fgname = "";
6190 try:
6191 import grp;
6192 try:
6193 groupinfo = grp.getgrgid(os.getgid());
6194 fgname = groupinfo.gr_name;
6195 except KeyError:
6196 fgname = "";
6197 except AttributeError:
6198 fgname = "";
6199 except ImportError:
6200 fgname = "";
6201 fcontents = BytesIO();
6202 if(ftype==0):
6203 fcontents.write(zipfp.read(member.filename));
6204 fcontents.seek(0, 0);
6205 ftypehex = format(ftype, 'x').lower();
6206 extrafields = len(extradata);
6207 extrafieldslist = extradata;
6208 catfextrafields = extrafields;
6209 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6210 if(len(extradata)>0):
6211 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6212 extrasizelen = len(extrasizestr);
6213 extrasizelenhex = format(extrasizelen, 'x').lower();
6214 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6215 catoutlen = len(catoutlist) + len(extradata) + 3;
6216 catoutlenhex = format(catoutlen, 'x').lower();
6217 catoutlist.insert(0, catoutlenhex);
6218 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6219 catheaderdata = catoutlist;
6220 if(len(extradata)>0):
6221 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6222 if(fsize==0):
6223 checksumlist = [checksumtype, "none"];
6224 else:
6225 checksumlist = [checksumtype, checksumtype];
6226 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6227 catfnumfields = catoutlen;
6228 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6229 fcontents.seek(0, 0);
6230 if(fsize==0):
6231 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6232 else:
6233 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6234 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6235 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6236 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6237 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6238 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6239 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6240 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6241 catfcontentstart = fheadtell;
6242 fheadtell += len(catfileoutstr) + 1;
6243 catfcontentend = fheadtell - 1;
6244 catfhend = catfcontentend;
6245 fcontents.seek(0, 0);
6246 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6247 pyhascontents = False;
6248 if(int(fsize)>0 and not listonly):
6249 pyhascontents = True;
6250 if(int(fsize)>0 and listonly):
6251 fcontents = BytesIO();
6252 pyhascontents = False;
6253 fcontents.seek(0, 0);
6254 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6255 fileidnum = fileidnum + 1;
6256 return catlist;
6257
6258 if(not rarfile_support):
6259 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6260 return False;
6261
6262 if(rarfile_support):
6263 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6264 formatspecs = FormatSpecsListToDict(formatspecs);
6265 curinode = 0;
6266 curfid = 0;
6267 inodelist = [];
6268 inodetofile = {};
6269 filetoinode = {};
6270 inodetocatinode = {};
6271 fileidnum = 0;
6272 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
6273 return False;
6274 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6275 return False;
6276 rarfp = rarfile.RarFile(infile, "r");
6277 rartest = rarfp.testrar();
6278 if(rartest):
6279 VerbosePrintOut("Bad file found!");
6280 fnumfiles = int(len(rarfp.infolist()));
6281 catver = formatspecs['format_ver'];
6282 fileheaderver = str(int(catver.replace(".", "")));
6283 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6284 catversion = re.findall(r"([\d]+)", fileheader);
6285 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6286 fnumfileshex = format(int(fnumfiles), 'x').lower();
6287 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6288 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6289 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6290 fheadtell = len(fileheader);
6291 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6292 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6293 catfhstart = fheadtell;
6294 is_unix = False;
6295 is_windows = False;
6296 if(member.host_os==rarfile.RAR_OS_UNIX):
6297 is_windows = False;
6298 try:
6299 member.external_attr
6300 is_unix = True;
6301 except AttributeError:
6302 is_unix = False;
6303 elif(member.host_os==rarfile.RAR_OS_WIN32):
6304 is_unix = False;
6305 try:
6306 member.external_attr
6307 is_windows = True;
6308 except AttributeError:
6309 is_windows = False;
6310 else:
6311 is_unix = False;
6312 is_windows = False;
6313 if(re.findall(r"^[.|/]", member.filename)):
6314 fname = member.filename;
6315 else:
6316 fname = "./"+member.filename;
6317 rarinfo = rarfp.getinfo(member.filename);
6318 if(verbose):
6319 VerbosePrintOut(fname);
6320 if(is_unix and member.external_attr !=0):
6321 fpremode = int(member.external_attr);
6322 elif(member.is_file()):
6323 fpremode = stat.S_IFREG + 438;
6324 elif(member.is_symlink()):
6325 fpremode = stat.S_IFLNK + 438;
6326 elif(member.is_dir()):
6327 fpremode = stat.S_IFDIR + 511;
6328 if(is_windows and member.external_attr !=0):
6329 fwinattributes = int(member.external_attr);
6330 else:
6331 fwinattributes = int(0);
6332 fcompression = "";
6333 fcsize = 0;
6334 flinkcount = 0;
6335 ftype = 0;
6336 if(member.is_file()):
6337 ftype = 0;
6338 elif(member.is_symlink()):
6339 ftype = 2;
6340 elif(member.is_dir()):
6341 ftype = 5;
6342 flinkname = "";
6343 if(ftype==2):
6344 flinkname = rarfp.read(member.filename).decode("UTF-8");
6345 fbasedir = os.path.dirname(fname);
6346 fcurfid = curfid;
6347 fcurinode = curfid;
6348 finode = fcurinode;
6349 curfid = curfid + 1;
6350 fdev_minor = 0;
6351 fdev_major = 0;
6352 frdev_minor = 0;
6353 frdev_major = 0;
6354 if(ftype==5):
6355 fsize = "0";
6356 if(ftype==0):
6357 fsize = member.file_size;
6358 try:
6359 if(member.atime):
6360 fatime = int(member.atime.timestamp());
6361 else:
6362 fatime = int(member.mtime.timestamp());
6363 except AttributeError:
6364 fatime = int(member.mtime.timestamp());
6365 fmtime = int(member.mtime.timestamp());
6366 try:
6367 if(member.ctime):
6368 fctime = int(member.ctime.timestamp());
6369 else:
6370 fctime = int(member.mtime.timestamp());
6371 except AttributeError:
6372 fctime = int(member.mtime.timestamp());
6373 fbtime = int(member.mtime.timestamp());
6374 if(is_unix and member.external_attr !=0):
6375 fmode = int(member.external_attr);
6376 fchmode = int(stat.S_IMODE(member.external_attr));
6377 ftypemod = int(stat.S_IFMT(member.external_attr));
6378 elif(member.is_file()):
6379 fmode = int(stat.S_IFREG + 438)
6380 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6381 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6382 elif(member.is_symlink()):
6383 fmode = int(stat.S_IFLNK + 438)
6384 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6385 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6386 elif(member.is_dir()):
6387 fmode = int(stat.S_IFDIR + 511)
6388 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6389 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6390 try:
6391 fuid = os.getuid();
6392 except AttributeError:
6393 fuid = 0;
6394 except KeyError:
6395 fuid = 0;
6396 try:
6397 fgid = os.getgid();
6398 except AttributeError:
6399 fgid = 0;
6400 except KeyError:
6401 fgid = 0;
6402 try:
6403 import pwd;
6404 try:
6405 userinfo = pwd.getpwuid(os.getuid());
6406 funame = userinfo.pw_name;
6407 except KeyError:
6408 funame = "";
6409 except AttributeError:
6410 funame = "";
6411 except ImportError:
6412 funame = "";
6413 fgname = "";
6414 try:
6415 import grp;
6416 try:
6417 groupinfo = grp.getgrgid(os.getgid());
6418 fgname = groupinfo.gr_name;
6419 except KeyError:
6420 fgname = "";
6421 except AttributeError:
6422 fgname = "";
6423 except ImportError:
6424 fgname = "";
6425 fcontents = BytesIO();
6426 if(ftype==0):
6427 fcontents.write(rarfp.read(member.filename));
6428 fcontents.seek(0, 0);
6429 ftypehex = format(ftype, 'x').lower();
6430 extrafields = len(extradata);
6431 extrafieldslist = extradata;
6432 catfextrafields = extrafields;
6433 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6434 if(len(extradata)>0):
6435 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6436 extrasizelen = len(extrasizestr);
6437 extrasizelenhex = format(extrasizelen, 'x').lower();
6438 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6439 catoutlen = len(catoutlist) + len(extradata) + 3;
6440 catoutlenhex = format(catoutlen, 'x').lower();
6441 catoutlist.insert(0, catoutlenhex);
6442 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6443 if(len(extradata)>0):
6444 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6445 if(fsize==0):
6446 checksumlist = [checksumtype, "none"];
6447 else:
6448 checksumlist = [checksumtype, checksumtype];
6449 ccatfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6450 catfnumfields = 24 + catfextrafields;
6451 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6452 fcontents.seek(0, 0);
6453 if(fsize==0):
6454 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6455 else:
6456 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6457 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6458 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6459 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6460 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6461 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6462 catheaderdata = catoutlist;
6463 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6464 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6465 catfcontentstart = fheadtell;
6466 fheadtell += len(catfileoutstr) + 1;
6467 catfcontentend = fheadtell - 1;
6468 catfhend = catfcontentend;
6469 fcontents.seek(0, 0);
6470 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6471 pyhascontents = False;
6472 if(int(fsize)>0 and not listonly):
6473 pyhascontents = True;
6474 if(int(fsize)>0 and listonly):
6475 fcontents = BytesIO();
6476 pyhascontents = False;
6477 fcontents.seek(0, 0);
6478 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6479 fileidnum = fileidnum + 1;
6480 return catlist;
6481
6482 if(not py7zr_support):
6483 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6484 return False;
6485
6486 if(py7zr_support):
6487 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6488 formatspecs = FormatSpecsListToDict(formatspecs);
6489 curinode = 0;
6490 curfid = 0;
6491 inodelist = [];
6492 inodetofile = {};
6493 filetoinode = {};
6494 inodetocatinode = {};
6495 fileidnum = 0;
6496 szpfp = py7zr.SevenZipFile(infile, mode="r");
6497 file_content = szpfp.readall();
6498 #sztest = szpfp.testzip();
6499 sztestalt = szpfp.test();
6500 if(sztestalt):
6501 VerbosePrintOut("Bad file found!");
6502 numfiles = int(len(szpfp.list()));
6503 catver = formatspecs['format_ver'];
6504 fileheaderver = str(int(catver.replace(".", "")));
6505 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6506 catversion = re.findall(r"([\d]+)", fileheader);
6507 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6508 fnumfileshex = format(int(fnumfiles), 'x').lower();
6509 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6510 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6511 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6512 fheadtell = len(fileheader);
6513 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6514 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6515 catfhstart = fheadtell;
6516 if(re.findall(r"^[.|/]", member.filename)):
6517 fname = member.filename;
6518 else:
6519 fname = "./"+member.filename;
6520 if(not member.is_directory):
6521 fpremode = int(stat.S_IFREG + 438);
6522 elif(member.is_directory):
6523 fpremode = int(stat.S_IFDIR + 511);
6524 fwinattributes = int(0);
6525 fcompression = "";
6526 fcsize = 0;
6527 flinkcount = 0;
6528 ftype = 0;
6529 if(member.is_directory):
6530 ftype = 5;
6531 else:
6532 ftype = 0;
6533 flinkname = "";
6534 fbasedir = os.path.dirname(fname);
6535 fcurfid = curfid;
6536 fcurinode = curfid;
6537 finode = fcurinode;
6538 curfid = curfid + 1;
6539 fdev_minor = 0;
6540 fdev_major = 0;
6541 frdev_minor = 0;
6542 frdev_major = 0;
6543 if(ftype==5):
6544 fsize = "0";
6545 fatime = int(member.creationtime.timestamp());
6546 fmtime = int(member.creationtime.timestamp());
6547 fctime = int(member.creationtime.timestamp());
6548 fbtime = int(member.creationtime.timestamp());
6549 if(member.is_directory):
6550 fmode = int(stat.S_IFDIR + 511)
6551 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6552 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6553 else:
6554 fmode = int(stat.S_IFLNK + 438)
6555 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6556 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6557 try:
6558 fuid = os.getuid();
6559 except AttributeError:
6560 fuid = 0;
6561 except KeyError:
6562 fuid = 0;
6563 try:
6564 fgid = os.getgid();
6565 except AttributeError:
6566 fgid = 0;
6567 except KeyError:
6568 fgid = 0;
6569 try:
6570 import pwd;
6571 try:
6572 userinfo = pwd.getpwuid(os.getuid());
6573 funame = userinfo.pw_name;
6574 except KeyError:
6575 funame = "";
6576 except AttributeError:
6577 funame = "";
6578 except ImportError:
6579 funame = "";
6580 fgname = "";
6581 try:
6582 import grp;
6583 try:
6584 groupinfo = grp.getgrgid(os.getgid());
6585 fgname = groupinfo.gr_name;
6586 except KeyError:
6587 fgname = "";
6588 except AttributeError:
6589 fgname = "";
6590 except ImportError:
6591 fgname = "";
6592 fcontents = BytesIO();
6593 if(ftype==0):
6594 fcontents.write(file_content[member.filename].read());
6595 fsize = format(fcontents.tell(), 'x').lower();
6596 fileop.close();
6597 fcontents.seek(0, 0);
6598 ftypehex = format(ftype, 'x').lower();
6599 extrafields = len(extradata);
6600 extrafieldslist = extradata;
6601 catfextrafields = extrafields;
6602 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6603 if(len(extradata)>0):
6604 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6605 extrasizelen = len(extrasizestr);
6606 extrasizelenhex = format(extrasizelen, 'x').lower();
6607 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6608 catoutlen = len(catoutlist) + len(extradata) + 3;
6609 catoutlenhex = format(catoutlen, 'x').lower();
6610 catoutlist.insert(0, catoutlenhex);
6611 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6612 catheaderdata = catoutlist;
6613 if(len(extradata)>0):
6614 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6615 if(fsize==0):
6616 checksumlist = [checksumtype, "none"];
6617 else:
6618 checksumlist = [checksumtype, checksumtype];
6619 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6620 catfnumfields = 24 + catfextrafields;
6621 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6622 fcontents.seek(0, 0);
6623 if(fsize==0):
6624 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6625 else:
6626 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6627 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6628 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6629 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6630 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6631 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6632 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6633 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6634 catfcontentstart = fheadtell;
6635 fheadtell += len(catfileoutstr) + 1;
6636 catfcontentend = fheadtell - 1;
6637 catfhend = catfcontentend;
6638 fcontents.seek(0, 0);
6639 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6640 pyhascontents = False;
6641 if(int(fsize)>0 and not listonly):
6642 pyhascontents = True;
6643 if(int(fsize)>0 and listonly):
6644 fcontents = BytesIO();
6645 pyhascontents = False;
6646 fcontents.seek(0, 0);
6647 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6648 fileidnum = fileidnum + 1;
6649 return catlist;
6650
6651 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6652 formatspecs = FormatSpecsListToDict(formatspecs);
6653 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6654 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
6655 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6656 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
6657 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6658 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
6659 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6660 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
6661 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6662 elif(checkcompressfile=="catfile"):
6663 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6664 else:
6665 return False;
6666 return False;
6667
6668 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6669 formatspecs = FormatSpecsListToDict(formatspecs);
6670 outarray = BytesIO();
6671 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6672 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6673 return listcatfiles;
6674
6675 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6676 formatspecs = FormatSpecsListToDict(formatspecs);
6677 if(isinstance(inarray, dict)):
6678 listcatfiles = inarray;
6679 else:
6680 return False;
6681 if(not listcatfiles):
6682 return False;
6683 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6684 if(returnfp):
6685 catarray.update({'catfp': listcatfiles['catfp']});
6686 lenlist = len(listcatfiles['ffilelist']);
6687 lcfi = 0;
6688 lcfx = int(listcatfiles['fnumfiles']);
6689 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6690 lcfx = int(lenlist);
6691 else:
6692 lcfx = int(listcatfiles['fnumfiles']);
6693 while(lcfi < lcfx):
6694 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6695 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6696 catarray['filetoid'].update(filetoidarray);
6697 catarray['idtofile'].update(idtofilearray);
6698 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6699 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6700 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6701 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6702 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6703 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6704 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6705 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6706 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6707 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6708 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6709 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6710 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6711 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6712 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6713 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6714 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6715 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6716 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6717 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6718 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6719 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6720 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6721 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6722 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6723 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6724 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6725 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6726 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6727 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6728 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6729 lcfi = lcfi + 1;
6730 return catarray;
6731
6732 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6733
6734 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6735 formatspecs = FormatSpecsListToDict(formatspecs);
6736 if(isinstance(infile, dict)):
6737 listcatfiles = infile;
6738 else:
6739 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6740 infile = RemoveWindowsPath(infile);
6741 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6742 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6743 outfile = RemoveWindowsPath(outfile);
6744 checksumtype = checksumtype.lower();
6745 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6746 checksumtype="crc32";
6747 if(checksumtype=="none"):
6748 checksumtype = "";
6749 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6750 compression = "auto";
6751 if(compression not in compressionlist and compression is None):
6752 compression = "auto";
6753 if(verbose):
6754 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6755 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6756 if(os.path.exists(outfile)):
6757 try:
6758 os.unlink(outfile);
6759 except OSError as e:
6760 pass;
6761 if(not listcatfiles):
6762 return False;
6763 if(outfile=="-"):
6764 verbose = False;
6765 catfp = BytesIO();
6766 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6767 catfp = outfile;
6768 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6769 catfp = BytesIO();
6770 else:
6771 fbasename = os.path.splitext(outfile)[0];
6772 fextname = os.path.splitext(outfile)[1];
6773 if(not compresswholefile and fextname in outextlistwd):
6774 compresswholefile = True;
6775 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6776 catver = formatspecs['format_ver'];
6777 fileheaderver = str(int(catver.replace(".", "")));
6778 lenlist = len(listcatfiles['ffilelist']);
6779 fnumfiles = int(listcatfiles['fnumfiles']);
6780 if(lenlist>fnumfiles or lenlist<fnumfiles):
6781 fnumfiles = lenlist;
6782 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6783 lenlist = len(listcatfiles['ffilelist']);
6784 fnumfiles = int(listcatfiles['fnumfiles']);
6785 lcfi = 0;
6786 lcfx = int(listcatfiles['fnumfiles']);
6787 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6788 lcfx = int(lenlist);
6789 else:
6790 lcfx = int(listcatfiles['fnumfiles']);
6791 curinode = 0;
6792 curfid = 0;
6793 inodelist = [];
6794 inodetofile = {};
6795 filetoinode = {};
6796 reallcfi = 0;
6797 while(lcfi < lcfx):
6798 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6799 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6800 else:
6801 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6802 if(verbose):
6803 VerbosePrintOut(fname);
6804 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6805 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6806 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6807 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6808 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6809 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6810 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6811 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6812 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6813 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6814 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6815 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6816 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6817 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6818 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6819 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6820 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6821 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6822 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6823 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6824 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6825 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6826 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6827 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6828 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6829 if(not followlink and len(extradata)<0):
6830 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6831 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6832 fcompression = "";
6833 fcsize = format(int(0), 'x').lower();
6834 if(not compresswholefile):
6835 fcontents.seek(0, 2);
6836 ucfsize = fcontents.tell();
6837 fcontents.seek(0, 0);
6838 if(compression=="auto"):
6839 ilsize = len(compressionlistalt);
6840 ilmin = 0;
6841 ilcsize = [];
6842 while(ilmin < ilsize):
6843 cfcontents = BytesIO();
6844 shutil.copyfileobj(fcontents, cfcontents);
6845 fcontents.seek(0, 0);
6846 cfcontents.seek(0, 0);
6847 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6848 if(cfcontents):
6849 cfcontents.seek(0, 2);
6850 ilcsize.append(cfcontents.tell());
6851 cfcontents.close();
6852 else:
6853 try:
6854 ilcsize.append(sys.maxint);
6855 except AttributeError:
6856 ilcsize.append(sys.maxsize);
6857 ilmin = ilmin + 1;
6858 ilcmin = ilcsize.index(min(ilcsize));
6859 compression = compressionlistalt[ilcmin];
6860 fcontents.seek(0, 0);
6861 cfcontents = BytesIO();
6862 shutil.copyfileobj(fcontents, cfcontents);
6863 cfcontents.seek(0, 0);
6864 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6865 cfcontents.seek(0, 2);
6866 cfsize = cfcontents.tell();
6867 if(ucfsize > cfsize):
6868 fcsize = format(int(cfsize), 'x').lower();
6869 fcompression = compression;
6870 fcontents.close();
6871 fcontents = cfcontents;
6872 if(followlink):
6873 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6874 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6875 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6876 flinkinfo = listcatfiles['ffilelist'][flinkid];
6877 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6878 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6879 flinkname = flinkinfo['flinkname'];
6880 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6881 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6882 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6883 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6884 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6885 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6886 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6887 funame = flinkinfo['funame'];
6888 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6889 fgname = flinkinfo['fgname'];
6890 finode = format(int(flinkinfo['finode']), 'x').lower();
6891 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6892 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6893 fcompression = flinkinfo['fcompression'];
6894 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6895 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6896 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6897 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6898 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6899 fseeknextfile = flinkinfo['fseeknextfile'];
6900 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6901 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6902 if(len(extradata)<0):
6903 extradata = flinkinfo['fextralist'];
6904 fcontents = flinkinfo['fcontents'];
6905 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6906 else:
6907 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6908 fcurfid = format(curfid, 'x').lower();
6909 if(not followlink and finode!=0):
6910 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6911 fcurinode = format(int(curinode), 'x').lower();
6912 inodetofile.update({curinode: fname});
6913 filetoinode.update({fname: curinode});
6914 curinode = curinode + 1;
6915 else:
6916 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6917 else:
6918 fcurinode = format(int(curinode), 'x').lower();
6919 curinode = curinode + 1;
6920 curfid = curfid + 1;
6921 if(fcompression=="none"):
6922 fcompression = "";
6923 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6924 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6925 fcontents.close();
6926 lcfi = lcfi + 1;
6927 reallcfi = reallcfi + 1;
6928 if(lcfx>0):
6929 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6930 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6931 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6932 try:
6933 catfp.flush();
6934 os.fsync(catfp.fileno());
6935 except io.UnsupportedOperation:
6936 pass;
6937 except AttributeError:
6938 pass;
6939 except OSError as e:
6940 pass;
6941 if(outfile=="-"):
6942 catfp.seek(0, 0);
6943 if(hasattr(sys.stdout, "buffer")):
6944 shutil.copyfileobj(catfp, sys.stdout.buffer);
6945 else:
6946 shutil.copyfileobj(catfp, sys.stdout);
6947 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6948 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6949 catfp.seek(0, 0);
6950 upload_file_to_internet_file(catfp, outfile);
6951 if(returnfp):
6952 catfp.seek(0, 0);
6953 return catfp;
6954 else:
6955 catfp.close();
6956 return True;
6957
6958 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6959
6960 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6961 formatspecs = FormatSpecsListToDict(formatspecs);
6962 catfp = BytesIO(catstr);
6963 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6964 return listcatfiles;
6965
6966 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6967
6968 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6969 formatspecs = FormatSpecsListToDict(formatspecs);
6970 outarray = BytesIO();
6971 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6972 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6973 return listcatfiles;
6974
6975 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6976
6977 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6978 formatspecs = FormatSpecsListToDict(formatspecs);
6979 if(outdir is not None):
6980 outdir = RemoveWindowsPath(outdir);
6981 if(verbose):
6982 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6983 if(isinstance(infile, dict)):
6984 listcatfiles = infile;
6985 else:
6986 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6987 infile = RemoveWindowsPath(infile);
6988 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6989 if(not listcatfiles):
6990 return False;
6991 lenlist = len(listcatfiles['ffilelist']);
6992 fnumfiles = int(listcatfiles['fnumfiles']);
6993 lcfi = 0;
6994 lcfx = int(listcatfiles['fnumfiles']);
6995 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6996 lcfx = int(lenlist);
6997 else:
6998 lcfx = int(listcatfiles['fnumfiles']);
6999 while(lcfi<lcfx):
7000 funame = "";
7001 try:
7002 import pwd;
7003 try:
7004 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
7005 funame = userinfo.pw_name;
7006 except KeyError:
7007 funame = "";
7008 except ImportError:
7009 funame = "";
7010 fgname = "";
7011 try:
7012 import grp;
7013 try:
7014 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
7015 fgname = groupinfo.gr_name;
7016 except KeyError:
7017 fgname = "";
7018 except ImportError:
7019 fgname = "";
7020 if(verbose):
7021 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7022 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
7023 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7024 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
7025 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
7026 try:
7027 fpc.flush();
7028 os.fsync(fpc.fileno());
7029 except io.UnsupportedOperation:
7030 pass;
7031 except AttributeError:
7032 pass;
7033 except OSError as e:
7034 pass;
7035 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
7036 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
7037 if(preservepermissions):
7038 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7039 if(preservetime):
7040 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
7041 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
7042 if(followlink):
7043 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
7044 flinkid = prelistcatfiles['filetoid'][getflinkpath];
7045 flinkinfo = listcatfiles['ffilelist'][flinkid];
7046 funame = "";
7047 try:
7048 import pwd;
7049 try:
7050 userinfo = pwd.getpwuid(flinkinfo['fuid']);
7051 funame = userinfo.pw_name;
7052 except KeyError:
7053 funame = "";
7054 except ImportError:
7055 funame = "";
7056 fgname = "";
7057 try:
7058 import grp;
7059 try:
7060 groupinfo = grp.getgrgid(flinkinfo['fgid']);
7061 fgname = groupinfo.gr_name;
7062 except KeyError:
7063 fgname = "";
7064 except ImportError:
7065 fgname = "";
7066 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
7067 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7068 flinkinfo['fcontents'].seek(0, 0);
7069 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
7070 try:
7071 fpc.flush();
7072 os.fsync(fpc.fileno());
7073 except io.UnsupportedOperation:
7074 pass;
7075 except AttributeError:
7076 pass;
7077 except OSError as e:
7078 pass;
7079 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7080 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7081 if(preservepermissions):
7082 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7083 if(preservetime):
7084 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7085 if(flinkinfo['ftype']==1):
7086 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7087 if(flinkinfo['ftype']==2):
7088 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7089 if(flinkinfo['ftype']==5):
7090 if(preservepermissions):
7091 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7092 else:
7093 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7094 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7095 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7096 if(preservepermissions):
7097 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7098 if(preservetime):
7099 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7100 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
7101 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7102 else:
7103 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7104 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
7105 if(followlink):
7106 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
7107 flinkid = prelistcatfiles['filetoid'][getflinkpath];
7108 flinkinfo = listcatfiles['ffilelist'][flinkid];
7109 funame = "";
7110 try:
7111 import pwd;
7112 try:
7113 userinfo = pwd.getpwuid(flinkinfo['fuid']);
7114 funame = userinfo.pw_name;
7115 except KeyError:
7116 funame = "";
7117 except ImportError:
7118 funame = "";
7119 fgname = "";
7120 try:
7121 import grp;
7122 try:
7123 groupinfo = grp.getgrgid(flinkinfo['fgid']);
7124 fgname = groupinfo.gr_name;
7125 except KeyError:
7126 fgname = "";
7127 except ImportError:
7128 fgname = "";
7129 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
7130 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7131 flinkinfo['fcontents'].seek(0, 0);
7132 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
7133 try:
7134 fpc.flush();
7135 os.fsync(fpc.fileno());
7136 except io.UnsupportedOperation:
7137 pass;
7138 except AttributeError:
7139 pass;
7140 except OSError as e:
7141 pass;
7142 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7143 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7144 if(preservepermissions):
7145 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7146 if(preservetime):
7147 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7148 if(flinkinfo['ftype']==1):
7149 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7150 if(flinkinfo['ftype']==2):
7151 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7152 if(flinkinfo['ftype']==5):
7153 if(preservepermissions):
7154 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7155 else:
7156 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7157 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7158 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7159 if(preservepermissions):
7160 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7161 if(preservetime):
7162 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7163 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
7164 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7165 else:
7166 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7167 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
7168 if(preservepermissions):
7169 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7170 else:
7171 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7172 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
7173 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
7174 if(preservepermissions):
7175 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7176 if(preservetime):
7177 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
7178 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
7179 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7180 lcfi = lcfi + 1;
7181 if(returnfp):
7182 return listcatfiles['ffilelist']['catfp'];
7183 else:
7184 return True;
7185
7186 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
7187
7188 if(hasattr(shutil, "register_unpack_format")):
7189 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
7190 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
7191 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
7192
7193 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7194 formatspecs = FormatSpecsListToDict(formatspecs);
7195 catfp = BytesIO(catstr);
7196 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7197 return listcatfiles;
7198
7199 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
7200
7201 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7202 formatspecs = FormatSpecsListToDict(formatspecs);
7203 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7204 if(isinstance(infile, dict)):
7205 listcatfiles = infile;
7206 else:
7207 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
7208 infile = RemoveWindowsPath(infile);
7209 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
7210 if(not listcatfiles):
7211 return False;
7212 lenlist = len(listcatfiles['ffilelist']);
7213 fnumfiles = int(listcatfiles['fnumfiles']);
7214 lcfi = 0;
7215 lcfx = int(listcatfiles['fnumfiles']);
7216 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
7217 lcfx = int(lenlist);
7218 else:
7219 lcfx = int(listcatfiles['fnumfiles']);
7220 returnval = {};
7221 while(lcfi<lcfx):
7222 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
7223 if(not verbose):
7224 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
7225 if(verbose):
7226 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7227 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
7228 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
7229 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
7230 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
7231 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
7232 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
7233 if(len(fuprint)<=0):
7234 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
7235 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
7236 if(len(fgprint)<=0):
7237 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
7238 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7239 lcfi = lcfi + 1;
7240 if(returnfp):
7241 return listcatfiles['catfp'];
7242 else:
7243 return True;
7244
7245 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
7246
7247 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7248 formatspecs = FormatSpecsListToDict(formatspecs);
7249 catfp = BytesIO(catstr);
7250 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7251 return listcatfiles;
7252
7253 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
7254
7255 def TarFileListFiles(infile, verbose=False, returnfp=False):
7256 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7257 if(infile=="-"):
7258 infile = BytesIO();
7259 if(hasattr(sys.stdin, "buffer")):
7260 shutil.copyfileobj(sys.stdin.buffer, infile);
7261 else:
7262 shutil.copyfileobj(sys.stdin, infile);
7263 infile.seek(0, 0);
7264 if(not infile):
7265 return False;
7266 infile.seek(0, 0);
7267 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7268 infile = download_file_from_internet_file(infile);
7269 infile.seek(0, 0);
7270 if(not infile):
7271 return False;
7272 infile.seek(0, 0);
7273 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7274 return False;
7275 elif(os.path.exists(infile) and os.path.isfile(infile)):
7276 try:
7277 if(not tarfile.TarFileCheck(infile)):
7278 return False;
7279 except AttributeError:
7280 if(not TarFileCheck(infile)):
7281 return False;
7282 try:
7283 if(hasattr(infile, "read") or hasattr(infile, "write")):
7284 tarfp = tarfile.open(fileobj=infile, mode="r");
7285 else:
7286 tarfp = tarfile.open(infile, "r");
7287 except FileNotFoundError:
7288 return False;
7289 lcfi = 0
7290 returnval = {};
7291 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
7292 returnval.update({lcfi: member.name});
7293 fpremode = member.mode;
7294 ffullmode = member.mode;
7295 flinkcount = 0;
7296 ftype = 0;
7297 if(member.isreg()):
7298 ffullmode = member.mode + stat.S_IFREG;
7299 ftype = 0;
7300 elif(member.isdev()):
7301 ffullmode = member.mode;
7302 ftype = 7;
7303 elif(member.islnk()):
7304 ffullmode = member.mode + stat.S_IFREG;
7305 ftype = 1;
7306 elif(member.issym()):
7307 ffullmode = member.mode + stat.S_IFLNK;
7308 ftype = 2;
7309 elif(member.ischr()):
7310 ffullmode = member.mode + stat.S_IFCHR;
7311 ftype = 3;
7312 elif(member.isblk()):
7313 ffullmode = member.mode + stat.S_IFBLK;
7314 ftype = 4;
7315 elif(member.isdir()):
7316 ffullmode = member.mode + stat.S_IFDIR;
7317 ftype = 5;
7318 elif(member.isfifo()):
7319 ffullmode = member.mode + stat.S_IFIFO;
7320 ftype = 6;
7321 elif(member.issparse()):
7322 ffullmode = member.mode;
7323 ftype = 12;
7324 if(not verbose):
7325 VerbosePrintOut(member.name);
7326 elif(verbose):
7327 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7328 printfname = member.name;
7329 if(member.islnk()):
7330 printfname = member.name + " link to " + member.linkname;
7331 elif(member.issym()):
7332 printfname = member.name + " -> " + member.linkname;
7333 fuprint = member.uname;
7334 if(len(fuprint)<=0):
7335 fuprint = member.uid;
7336 fgprint = member.gname;
7337 if(len(fgprint)<=0):
7338 fgprint = member.gid;
7339 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7340 lcfi = lcfi + 1;
7341 if(returnfp):
7342 return listcatfiles['catfp'];
7343 else:
7344 return True;
7345
7346 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7347 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7348 if(infile=="-"):
7349 infile = BytesIO();
7350 if(hasattr(sys.stdin, "buffer")):
7351 shutil.copyfileobj(sys.stdin.buffer, infile);
7352 else:
7353 shutil.copyfileobj(sys.stdin, infile);
7354 infile.seek(0, 0);
7355 if(not infile):
7356 return False;
7357 infile.seek(0, 0);
7358 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7359 infile = download_file_from_internet_file(infile);
7360 infile.seek(0, 0);
7361 if(not infile):
7362 return False;
7363 infile.seek(0, 0);
7364 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7365 return False;
7366 if(not zipfile.is_zipfile(infile)):
7367 return False;
7368 try:
7369 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7370 except FileNotFoundError:
7371 print(6);return False;
7372 lcfi = 0;
7373 returnval = {};
7374 ziptest = zipfp.testzip();
7375 if(ziptest):
7376 VerbosePrintOut("Bad file found!");
7377 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7378 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7379 fwinattributes = int(zipinfo.external_attr);
7380 if(not member.is_dir()):
7381 fmode = int(stat.S_IFREG + 438);
7382 fchmode = int(stat.S_IMODE(fmode));
7383 ftypemod = int(stat.S_IFMT(fmode));
7384 elif(member.is_dir()):
7385 fmode = int(stat.S_IFDIR + 511);
7386 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7387 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7388 elif(zipinfo.create_system==3):
7389 fwinattributes =int(0);
7390 try:
7391 fmode = int(zipinfo.external_attr);
7392 fchmode = stat.S_IMODE(fmode);
7393 ftypemod = stat.S_IFMT(fmode);
7394 except OverflowError:
7395 fmode = int(zipinfo.external_attr >> 16);
7396 fchmode = stat.S_IMODE(fmode);
7397 ftypemod = stat.S_IFMT(fmode);
7398 else:
7399 fwinattributes = int(0);
7400 if(not member.is_dir()):
7401 fmode = int(stat.S_IFREG + 438);
7402 fchmode = int(stat.S_IMODE(fmode));
7403 ftypemod = int(stat.S_IFMT(fmode));
7404 elif(member.is_dir()):
7405 fmode = int(stat.S_IFDIR + 511);
7406 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7407 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7408 returnval.update({lcfi: member.filename});
7409 if(not verbose):
7410 VerbosePrintOut(member.filename);
7411 if(verbose):
7412 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7413 permissionstr = "";
7414 for fmodval in str(oct(fmode))[-3:]:
7415 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7416 if(not member.is_dir()):
7417 ftype = 0;
7418 permissionstr = "-" + permissionstr;
7419 elif(member.is_dir()):
7420 ftype = 5;
7421 permissionstr = "d" + permissionstr;
7422 printfname = member.filename;
7423 try:
7424 fuid = int(os.getuid());
7425 except AttributeError:
7426 fuid = int(0);
7427 except KeyError:
7428 fuid = int(0);
7429 try:
7430 fgid = int(os.getgid());
7431 except AttributeError:
7432 fgid = int(0);
7433 except KeyError:
7434 fgid = int(0);
7435 try:
7436 import pwd;
7437 try:
7438 userinfo = pwd.getpwuid(os.getuid());
7439 funame = userinfo.pw_name;
7440 except KeyError:
7441 funame = "";
7442 except AttributeError:
7443 funame = "";
7444 except ImportError:
7445 funame = "";
7446 fgname = "";
7447 try:
7448 import grp;
7449 try:
7450 groupinfo = grp.getgrgid(os.getgid());
7451 fgname = groupinfo.gr_name;
7452 except KeyError:
7453 fgname = "";
7454 except AttributeError:
7455 fgname = "";
7456 except ImportError:
7457 fgname = "";
7458 fuprint = funame;
7459 if(len(fuprint)<=0):
7460 fuprint = str(fuid);
7461 fgprint = fgname;
7462 if(len(fgprint)<=0):
7463 fgprint = str(fgid);
7464 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7465 lcfi = lcfi + 1;
7466 if(returnfp):
7467 return listcatfiles['catfp'];
7468 else:
7469 return True;
7470
7471 if(not rarfile_support):
7472 def RarFileListFiles(infile, verbose=False, returnfp=False):
7473 return False;
7474
7475 if(rarfile_support):
7476 def RarFileListFiles(infile, verbose=False, returnfp=False):
7477 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7478 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7479 return False;
7480 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7481 return False;
7482 lcfi = 0;
7483 returnval = {};
7484 rarfp = rarfile.RarFile(infile, "r");
7485 rartest = rarfp.testrar();
7486 if(rartest):
7487 VerbosePrintOut("Bad file found!");
7488 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7489 is_unix = False;
7490 is_windows = False;
7491 if(member.host_os==rarfile.RAR_OS_UNIX):
7492 is_windows = False;
7493 try:
7494 member.external_attr
7495 is_unix = True;
7496 except AttributeError:
7497 is_unix = False;
7498 elif(member.host_os==rarfile.RAR_OS_WIN32):
7499 is_unix = False;
7500 try:
7501 member.external_attr
7502 is_windows = True;
7503 except AttributeError:
7504 is_windows = False;
7505 else:
7506 is_unix = False;
7507 is_windows = False;
7508 if(is_unix and member.external_attr !=0):
7509 fpremode = int(member.external_attr);
7510 elif(member.is_file()):
7511 fpremode = int(stat.S_IFREG + 438);
7512 elif(member.is_symlink()):
7513 fpremode = int(stat.S_IFLNK + 438);
7514 elif(member.is_dir()):
7515 fpremode = int(stat.S_IFDIR + 511);
7516 if(is_windows and member.external_attr !=0):
7517 fwinattributes = int(member.external_attr);
7518 else:
7519 fwinattributes = int(0);
7520 if(is_unix and member.external_attr !=0):
7521 fmode = int(member.external_attr);
7522 fchmode = int(stat.S_IMODE(member.external_attr));
7523 ftypemod = int(stat.S_IFMT(member.external_attr));
7524 elif(member.is_file()):
7525 fmode = int(stat.S_IFREG + 438);
7526 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7527 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7528 elif(member.is_symlink()):
7529 fmode = int(stat.S_IFLNK + 438);
7530 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7531 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7532 elif(member.is_dir()):
7533 fmode = int(stat.S_IFDIR + 511);
7534 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7535 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7536 returnval.update({lcfi: member.filename});
7537 if(not verbose):
7538 VerbosePrintOut(member.filename);
7539 if(verbose):
7540 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7541 permissionstr = "";
7542 for fmodval in str(oct(fmode))[-3:]:
7543 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7544 if(member.is_file()):
7545 ftype = 0;
7546 permissionstr = "-" + permissionstr;
7547 printfname = member.filename;
7548 elif(member.is_symlink()):
7549 ftype = 2;
7550 permissionstr = "l" + permissionstr;
7551 printfname = member.name + " -> " + member.read().decode("UTF-8");
7552 elif(member.is_dir()):
7553 ftype = 5;
7554 permissionstr = "d" + permissionstr;
7555 printfname = member.filename;
7556 try:
7557 fuid = int(os.getuid());
7558 except AttributeError:
7559 fuid = int(0);
7560 except KeyError:
7561 fuid = int(0);
7562 try:
7563 fgid = int(os.getgid());
7564 except AttributeError:
7565 fgid = int(0);
7566 except KeyError:
7567 fgid = int(0);
7568 try:
7569 import pwd;
7570 try:
7571 userinfo = pwd.getpwuid(os.getuid());
7572 funame = userinfo.pw_name;
7573 except KeyError:
7574 funame = "";
7575 except AttributeError:
7576 funame = "";
7577 except ImportError:
7578 funame = "";
7579 fgname = "";
7580 try:
7581 import grp;
7582 try:
7583 groupinfo = grp.getgrgid(os.getgid());
7584 fgname = groupinfo.gr_name;
7585 except KeyError:
7586 fgname = "";
7587 except AttributeError:
7588 fgname = "";
7589 except ImportError:
7590 fgname = "";
7591 fuprint = funame;
7592 if(len(fuprint)<=0):
7593 fuprint = str(fuid);
7594 fgprint = fgname;
7595 if(len(fgprint)<=0):
7596 fgprint = str(fgid);
7597 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7598 lcfi = lcfi + 1;
7599 if(returnfp):
7600 return listcatfiles['catfp'];
7601 else:
7602 return True;
7603
7604 if(not py7zr_support):
7605 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7606 return False;
7607
7608 if(py7zr_support):
7609 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7610 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7611 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7612 return False;
7613 lcfi = 0;
7614 returnval = {};
7615 szpfp = py7zr.SevenZipFile(infile, mode="r");
7616 file_content = szpfp.readall();
7617 #sztest = szpfp.testzip();
7618 sztestalt = szpfp.test();
7619 if(sztestalt):
7620 VerbosePrintOut("Bad file found!");
7621 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7622 if(re.findall(r"^[.|/]", member.filename)):
7623 fname = member.filename;
7624 else:
7625 fname = "./"+member.filename;
7626 if(not member.is_directory):
7627 fpremode = int(stat.S_IFREG + 438);
7628 elif(member.is_directory):
7629 fpremode = int(stat.S_IFDIR + 511);
7630 fwinattributes = int(0);
7631 if(member.is_directory):
7632 fmode = int(stat.S_IFDIR + 511);
7633 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7634 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7635 else:
7636 fmode = int(stat.S_IFLNK + 438);
7637 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7638 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7639 returnval.update({lcfi: member.filename});
7640 if(not verbose):
7641 VerbosePrintOut(member.filename);
7642 if(verbose):
7643 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7644 permissionstr = "";
7645 for fmodval in str(oct(fmode))[-3:]:
7646 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7647 fsize = int("0");
7648 if(not member.is_directory):
7649 ftype = 0;
7650 permissionstr = "-" + permissionstr;
7651 printfname = member.filename;
7652 elif(member.is_directory):
7653 ftype = 5;
7654 permissionstr = "d" + permissionstr;
7655 printfname = member.filename;
7656 if(ftype==0):
7657 fsize = len(file_content[member.filename].read());
7658 file_content[member.filename].close();
7659 try:
7660 fuid = int(os.getuid());
7661 except AttributeError:
7662 fuid = int(0);
7663 except KeyError:
7664 fuid = int(0);
7665 try:
7666 fgid = int(os.getgid());
7667 except AttributeError:
7668 fgid = int(0);
7669 except KeyError:
7670 fgid = int(0);
7671 try:
7672 import pwd;
7673 try:
7674 userinfo = pwd.getpwuid(os.getuid());
7675 funame = userinfo.pw_name;
7676 except KeyError:
7677 funame = "";
7678 except AttributeError:
7679 funame = "";
7680 except ImportError:
7681 funame = "";
7682 fgname = "";
7683 try:
7684 import grp;
7685 try:
7686 groupinfo = grp.getgrgid(os.getgid());
7687 fgname = groupinfo.gr_name;
7688 except KeyError:
7689 fgname = "";
7690 except AttributeError:
7691 fgname = "";
7692 except ImportError:
7693 fgname = "";
7694 fuprint = funame;
7695 if(len(fuprint)<=0):
7696 fuprint = str(fuid);
7697 fgprint = fgname;
7698 if(len(fgprint)<=0):
7699 fgprint = str(fgid);
7700 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7701 lcfi = lcfi + 1;
7702 if(returnfp):
7703 return listcatfiles['catfp'];
7704 else:
7705 return True;
7706
7707 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7708 formatspecs = FormatSpecsListToDict(formatspecs);
7709 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7710 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7711 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
7712 return TarFileListFiles(infile, verbose, returnfp);
7713 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
7714 return ZipFileListFiles(infile, verbose, returnfp);
7715 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
7716 return RarFileListFiles(infile, verbose, returnfp);
7717 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
7718 return SevenZipFileListFiles(infile, verbose, returnfp);
7719 elif(checkcompressfile=="catfile"):
7720 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7721 else:
7722 return False;
7723 return False;
7724
7725 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7726 formatspecs = FormatSpecsListToDict(formatspecs);
7727 outarray = BytesIO();
7728 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7729 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7730 return listcatfiles;
7731
7732 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7733 formatspecs = FormatSpecsListToDict(formatspecs);
7734 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7735 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7736 return listcatfiles;
7737
7738 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7739 formatspecs = FormatSpecsListToDict(formatspecs);
7740 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7741 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7742 return listcatfiles;
7743
7744 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7745
7746 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7747 formatspecs = FormatSpecsListToDict(formatspecs);
7748 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7749 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7750 return listcatfiles;
7751
7752 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7753
7754 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7755 formatspecs = FormatSpecsListToDict(formatspecs);
7756 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7757 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7758 return listcatfiles;
7759
7760 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7761
7762 if(not rarfile_support):
7763 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7764 return False;
7765
7766 if(rarfile_support):
7767 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7768 formatspecs = FormatSpecsListToDict(formatspecs);
7769 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7770 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7771 return listcatfiles;
7772
7773 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7774
7775 if(not py7zr_support):
7776 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7777 return False;
7778
7779 if(py7zr_support):
7780 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7781 formatspecs = FormatSpecsListToDict(formatspecs);
7782 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7783 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7784 return listcatfiles;
7785
7786 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7787
7788 def download_file_from_ftp_file(url):
7789 urlparts = urlparse(url);
7790 file_name = os.path.basename(urlparts.path);
7791 file_dir = os.path.dirname(urlparts.path);
7792 if(urlparts.username is not None):
7793 ftp_username = urlparts.username;
7794 else:
7795 ftp_username = "anonymous";
7796 if(urlparts.password is not None):
7797 ftp_password = urlparts.password;
7798 elif(urlparts.password is None and urlparts.username=="anonymous"):
7799 ftp_password = "anonymous";
7800 else:
7801 ftp_password = "";
7802 if(urlparts.scheme=="ftp"):
7803 ftp = FTP();
7804 elif(urlparts.scheme=="ftps" and ftpssl):
7805 ftp = FTP_TLS();
7806 else:
7807 return False;
7808 if(urlparts.scheme=="sftp"):
7809 if(__use_pysftp__):
7810 return download_file_from_pysftp_file(url);
7811 else:
7812 return download_file_from_sftp_file(url);
7813 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7814 return download_file_from_http_file(url);
7815 ftp_port = urlparts.port;
7816 if(urlparts.port is None):
7817 ftp_port = 21;
7818 try:
7819 ftp.connect(urlparts.hostname, ftp_port);
7820 except socket.gaierror:
7821 log.info("Error With URL "+url);
7822 return False;
7823 except socket.timeout:
7824 log.info("Error With URL "+url);
7825 return False;
7826 ftp.login(urlparts.username, urlparts.password);
7827 if(urlparts.scheme=="ftps"):
7828 ftp.prot_p();
7829 ftpfile = BytesIO();
7830 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7831 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7832 ftp.close();
7833 ftpfile.seek(0, 0);
7834 return ftpfile;
7835
7836 def download_file_from_ftp_string(url):
7837 ftpfile = download_file_from_ftp_file(url);
7838 return ftpfile.read();
7839
7840 def upload_file_to_ftp_file(ftpfile, url):
7841 urlparts = urlparse(url);
7842 file_name = os.path.basename(urlparts.path);
7843 file_dir = os.path.dirname(urlparts.path);
7844 if(urlparts.username is not None):
7845 ftp_username = urlparts.username;
7846 else:
7847 ftp_username = "anonymous";
7848 if(urlparts.password is not None):
7849 ftp_password = urlparts.password;
7850 elif(urlparts.password is None and urlparts.username=="anonymous"):
7851 ftp_password = "anonymous";
7852 else:
7853 ftp_password = "";
7854 if(urlparts.scheme=="ftp"):
7855 ftp = FTP();
7856 elif(urlparts.scheme=="ftps" and ftpssl):
7857 ftp = FTP_TLS();
7858 else:
7859 return False;
7860 if(urlparts.scheme=="sftp"):
7861 if(__use_pysftp__):
7862 return upload_file_to_pysftp_file(url);
7863 else:
7864 return upload_file_to_sftp_file(url);
7865 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7866 return False;
7867 ftp_port = urlparts.port;
7868 if(urlparts.port is None):
7869 ftp_port = 21;
7870 try:
7871 ftp.connect(urlparts.hostname, ftp_port);
7872 except socket.gaierror:
7873 log.info("Error With URL "+url);
7874 return False;
7875 except socket.timeout:
7876 log.info("Error With URL "+url);
7877 return False;
7878 ftp.login(urlparts.username, urlparts.password);
7879 if(urlparts.scheme=="ftps"):
7880 ftp.prot_p();
7881 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7882 ftp.close();
7883 ftpfile.seek(0, 0);
7884 return ftpfile;
7885
7886 def upload_file_to_ftp_string(ftpstring, url):
7887 ftpfileo = BytesIO(ftpstring);
7888 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7889 ftpfileo.close();
7890 return ftpfile;
7891
7892 class RawIteratorWrapper:
7893 def __init__(self, iterator):
7894 self.iterator = iterator;
7895 self.buffer = b"";
7896 self._iterator_exhausted = False;
7897 def read(self, size=-1):
7898 if self._iterator_exhausted:
7899 return b'';
7900 while size < 0 or len(self.buffer) < size:
7901 try:
7902 chunk = next(self.iterator);
7903 self.buffer += chunk;
7904 except StopIteration:
7905 self._iterator_exhausted = True;
7906 break;
7907 if size < 0:
7908 size = len(self.buffer);
7909 result, self.buffer = self.buffer[:size], self.buffer[size:];
7910 return result;
7911
7912 def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
7913 if headers is None:
7914 headers = {};
7915 # Parse the URL to extract username and password if present
7916 urlparts = urlparse(url);
7917 username = urlparts.username;
7918 password = urlparts.password;
7919 # Rebuild the URL without the username and password
7920 netloc = urlparts.hostname;
7921 if urlparts.scheme == "sftp":
7922 if __use_pysftp__:
7923 return download_file_from_pysftp_file(url);
7924 else:
7925 return download_file_from_sftp_file(url);
7926 elif urlparts.scheme == "ftp" or urlparts.scheme == "ftps":
7927 return download_file_from_ftp_file(url);
7928 if urlparts.port:
7929 netloc += ':' + str(urlparts.port);
7930 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7931 # Create a temporary file object
7932 httpfile = BytesIO();
7933 if usehttp == 'requests' and haverequests:
7934 # Use the requests library if selected and available
7935 if username and password:
7936 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7937 else:
7938 response = requests.get(rebuilt_url, headers=headers, stream=True);
7939 response.raw.decode_content = True
7940 shutil.copyfileobj(response.raw, httpfile);
7941 elif usehttp == 'httpx' and havehttpx:
7942 # Use httpx if selected and available
7943 with httpx.Client(follow_redirects=True) as client:
7944 if username and password:
7945 response = client.get(rebuilt_url, headers=headers, auth=(username, password));
7946 else:
7947 response = client.get(rebuilt_url, headers=headers);
7948 raw_wrapper = RawIteratorWrapper(response.iter_bytes());
7949 shutil.copyfileobj(raw_wrapper, httpfile);
7950 else:
7951 # Use urllib as a fallback
7952 # Build a Request object for urllib
7953 request = Request(rebuilt_url, headers=headers);
7954 # Create an opener object for handling URLs
7955 if username and password:
7956 # Create a password manager
7957 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7958 # Add the username and password
7959 password_mgr.add_password(None, rebuilt_url, username, password);
7960 # Create an authentication handler using the password manager
7961 auth_handler = HTTPBasicAuthHandler(password_mgr);
7962 # Build the opener with the authentication handler
7963 opener = build_opener(auth_handler);
7964 else:
7965 opener = build_opener();
7966 response = opener.open(request);
7967 shutil.copyfileobj(response, httpfile);
7968 # Reset file pointer to the start
7969 httpfile.seek(0, 0);
7970 # Return the temporary file object
7971 return httpfile;
7972
7973 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt, usehttp=__use_http_lib__):
7974 httpfile = download_file_from_http_file(url, headers, usehttp);
7975 return ftpfile.read();
7976
7977 if(haveparamiko):
7978 def download_file_from_sftp_file(url):
7979 urlparts = urlparse(url);
7980 file_name = os.path.basename(urlparts.path);
7981 file_dir = os.path.dirname(urlparts.path);
7982 sftp_port = urlparts.port;
7983 if(urlparts.port is None):
7984 sftp_port = 22;
7985 else:
7986 sftp_port = urlparts.port;
7987 if(urlparts.username is not None):
7988 sftp_username = urlparts.username;
7989 else:
7990 sftp_username = "anonymous";
7991 if(urlparts.password is not None):
7992 sftp_password = urlparts.password;
7993 elif(urlparts.password is None and urlparts.username=="anonymous"):
7994 sftp_password = "anonymous";
7995 else:
7996 sftp_password = "";
7997 if(urlparts.scheme=="ftp"):
7998 return download_file_from_ftp_file(url);
7999 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8000 return download_file_from_http_file(url);
8001 if(urlparts.scheme!="sftp"):
8002 return False;
8003 ssh = paramiko.SSHClient();
8004 ssh.load_system_host_keys();
8005 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
8006 try:
8007 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8008 except paramiko.ssh_exception.SSHException:
8009 return False;
8010 except socket.gaierror:
8011 log.info("Error With URL "+url);
8012 return False;
8013 except socket.timeout:
8014 log.info("Error With URL "+url);
8015 return False;
8016 sftp = ssh.open_sftp();
8017 sftpfile = BytesIO();
8018 sftp.getfo(urlparts.path, sftpfile);
8019 sftp.close();
8020 ssh.close();
8021 sftpfile.seek(0, 0);
8022 return sftpfile;
8023 else:
8024 def download_file_from_sftp_file(url):
8025 return False;
8026
8027 if(haveparamiko):
8028 def download_file_from_sftp_string(url):
8029 sftpfile = download_file_from_sftp_file(url);
8030 return sftpfile.read();
8031 else:
8032 def download_file_from_ftp_string(url):
8033 return False;
8034
8035 if(haveparamiko):
8036 def upload_file_to_sftp_file(sftpfile, url):
8037 urlparts = urlparse(url);
8038 file_name = os.path.basename(urlparts.path);
8039 file_dir = os.path.dirname(urlparts.path);
8040 sftp_port = urlparts.port;
8041 if(urlparts.port is None):
8042 sftp_port = 22;
8043 else:
8044 sftp_port = urlparts.port;
8045 if(urlparts.username is not None):
8046 sftp_username = urlparts.username;
8047 else:
8048 sftp_username = "anonymous";
8049 if(urlparts.password is not None):
8050 sftp_password = urlparts.password;
8051 elif(urlparts.password is None and urlparts.username=="anonymous"):
8052 sftp_password = "anonymous";
8053 else:
8054 sftp_password = "";
8055 if(urlparts.scheme=="ftp"):
8056 return upload_file_to_ftp_file(url);
8057 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8058 return False;
8059 if(urlparts.scheme!="sftp"):
8060 return False;
8061 ssh = paramiko.SSHClient();
8062 ssh.load_system_host_keys();
8063 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
8064 try:
8065 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8066 except paramiko.ssh_exception.SSHException:
8067 return False;
8068 except socket.gaierror:
8069 log.info("Error With URL "+url);
8070 return False;
8071 except socket.timeout:
8072 log.info("Error With URL "+url);
8073 return False;
8074 sftp = ssh.open_sftp();
8075 sftp.putfo(sftpfile, urlparts.path);
8076 sftp.close();
8077 ssh.close();
8078 sftpfile.seek(0, 0);
8079 return sftpfile;
8080 else:
8081 def upload_file_to_sftp_file(sftpfile, url):
8082 return False;
8083
8084 if(haveparamiko):
8085 def upload_file_to_sftp_string(sftpstring, url):
8086 sftpfileo = BytesIO(sftpstring);
8087 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
8088 sftpfileo.close();
8089 return sftpfile;
8090 else:
8091 def upload_file_to_sftp_string(url):
8092 return False;
8093
8094 if(havepysftp):
8095 def download_file_from_pysftp_file(url):
8096 urlparts = urlparse(url);
8097 file_name = os.path.basename(urlparts.path);
8098 file_dir = os.path.dirname(urlparts.path);
8099 sftp_port = urlparts.port;
8100 if(urlparts.port is None):
8101 sftp_port = 22;
8102 else:
8103 sftp_port = urlparts.port;
8104 if(urlparts.username is not None):
8105 sftp_username = urlparts.username;
8106 else:
8107 sftp_username = "anonymous";
8108 if(urlparts.password is not None):
8109 sftp_password = urlparts.password;
8110 elif(urlparts.password is None and urlparts.username=="anonymous"):
8111 sftp_password = "anonymous";
8112 else:
8113 sftp_password = "";
8114 if(urlparts.scheme=="ftp"):
8115 return download_file_from_ftp_file(url);
8116 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8117 return download_file_from_http_file(url);
8118 if(urlparts.scheme!="sftp"):
8119 return False;
8120 try:
8121 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8122 except paramiko.ssh_exception.SSHException:
8123 return False;
8124 except socket.gaierror:
8125 log.info("Error With URL "+url);
8126 return False;
8127 except socket.timeout:
8128 log.info("Error With URL "+url);
8129 return False;
8130 sftp = ssh.open_sftp();
8131 sftpfile = BytesIO();
8132 sftp.getfo(urlparts.path, sftpfile);
8133 sftp.close();
8134 ssh.close();
8135 sftpfile.seek(0, 0);
8136 return sftpfile;
8137 else:
8138 def download_file_from_pysftp_file(url):
8139 return False;
8140
8141 if(havepysftp):
8142 def download_file_from_pysftp_string(url):
8143 sftpfile = download_file_from_pysftp_file(url);
8144 return sftpfile.read();
8145 else:
8146 def download_file_from_ftp_string(url):
8147 return False;
8148
8149 if(havepysftp):
8150 def upload_file_to_pysftp_file(sftpfile, url):
8151 urlparts = urlparse(url);
8152 file_name = os.path.basename(urlparts.path);
8153 file_dir = os.path.dirname(urlparts.path);
8154 sftp_port = urlparts.port;
8155 if(urlparts.port is None):
8156 sftp_port = 22;
8157 else:
8158 sftp_port = urlparts.port;
8159 if(urlparts.username is not None):
8160 sftp_username = urlparts.username;
8161 else:
8162 sftp_username = "anonymous";
8163 if(urlparts.password is not None):
8164 sftp_password = urlparts.password;
8165 elif(urlparts.password is None and urlparts.username=="anonymous"):
8166 sftp_password = "anonymous";
8167 else:
8168 sftp_password = "";
8169 if(urlparts.scheme=="ftp"):
8170 return upload_file_to_ftp_file(url);
8171 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8172 return False;
8173 if(urlparts.scheme!="sftp"):
8174 return False;
8175 try:
8176 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8177 except paramiko.ssh_exception.SSHException:
8178 return False;
8179 except socket.gaierror:
8180 log.info("Error With URL "+url);
8181 return False;
8182 except socket.timeout:
8183 log.info("Error With URL "+url);
8184 return False;
8185 sftp = ssh.open_sftp();
8186 sftp.putfo(sftpfile, urlparts.path);
8187 sftp.close();
8188 ssh.close();
8189 sftpfile.seek(0, 0);
8190 return sftpfile;
8191 else:
8192 def upload_file_to_pysftp_file(sftpfile, url):
8193 return False;
8194
8195 if(havepysftp):
8196 def upload_file_to_pysftp_string(sftpstring, url):
8197 sftpfileo = BytesIO(sftpstring);
8198 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
8199 sftpfileo.close();
8200 return sftpfile;
8201 else:
8202 def upload_file_to_pysftp_string(url):
8203 return False;
8204
8205 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt, usehttp=__use_http_lib__):
8206 urlparts = urlparse(url);
8207 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8208 return download_file_from_http_file(url, headers, usehttp);
8209 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8210 return download_file_from_ftp_file(url);
8211 elif(urlparts.scheme=="sftp"):
8212 if(__use_pysftp__ and havepysftp):
8213 return download_file_from_pysftp_file(url);
8214 else:
8215 return download_file_from_sftp_file(url);
8216 else:
8217 return False;
8218 return False;
8219
8220 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
8221 formatspecs = FormatSpecsListToDict(formatspecs);
8222 fp = download_file_from_internet_file(url);
8223 fp = UncompressArchiveFile(fp, formatspecs);
8224 fp.seek(0, 0);
8225 if(not fp):
8226 return False;
8227 return fp;
8228
8229 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
8230 urlparts = urlparse(url);
8231 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8232 return download_file_from_http_string(url, headers);
8233 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8234 return download_file_from_ftp_string(url);
8235 elif(urlparts.scheme=="sftp"):
8236 if(__use_pysftp__ and havepysftp):
8237 return download_file_from_pysftp_string(url);
8238 else:
8239 return download_file_from_sftp_string(url);
8240 else:
8241 return False;
8242 return False;
8243
8244 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
8245 formatspecs = FormatSpecsListToDict(formatspecs);
8246 fp = download_file_from_internet_string(url);
8247 fp = UncompressArchiveFile(fp, formatspecs);
8248 fp.seek(0, 0);
8249 if(not fp):
8250 return False;
8251 return fp;
8252
8253 def upload_file_to_internet_file(ifp, url):
8254 urlparts = urlparse(url);
8255 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8256 return False;
8257 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8258 return upload_file_to_ftp_file(ifp, url);
8259 elif(urlparts.scheme=="sftp"):
8260 if(__use_pysftp__ and havepysftp):
8261 return upload_file_to_pysftp_file(ifp, url);
8262 else:
8263 return upload_file_to_sftp_file(ifp, url);
8264 else:
8265 return False;
8266 return False;
8267
8268 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
8269 formatspecs = FormatSpecsListToDict(formatspecs);
8270 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
8271 if(not catfileout):
8272 return False;
8273 catfp.seek(0, 0);
8274 upload_file_to_internet_file(catfp, outfile);
8275 return True;
8276
8277 def upload_file_to_internet_string(ifp, url):
8278 urlparts = urlparse(url);
8279 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8280 return False;
8281 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8282 return upload_file_to_ftp_string(ifp, url);
8283 elif(urlparts.scheme=="sftp"):
8284 if(__use_pysftp__ and havepysftp):
8285 return upload_file_to_pysftp_string(ifp, url);
8286 else:
8287 return upload_file_to_sftp_string(ifp, url);
8288 else:
8289 return False;
8290 return False;
8291
8292 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
8293 formatspecs = FormatSpecsListToDict(formatspecs);
8294 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
8295 if(not catfileout):
8296 return False;
8297 catfp.seek(0, 0);
8298 upload_file_to_internet_file(catfp, outfile);
8299 return True;
8300
8301 try:
8302 if(hasattr(shutil, "register_archive_format")):
8303 # Register the packing format
8304 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
8305 except shutil.RegistryError:
8306 pass;
8307
8308 try:
8309 if(hasattr(shutil, "register_unpack_format")):
8310 # Register the unpacking format
8311 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
8312 except shutil.RegistryError:
8313 pass;
A tar like file format name catfile after unix cat command (concatenate files).