2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/12/2024 Ver. 0.11.0 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, tempfile
, zipfile
, platform
;
25 from ftplib
import FTP
, FTP_TLS
;
28 from ftplib
import FTP
;
29 if(sys
.version
[0]=="2"):
30 from urlparse
import urlparse
, urlunparse
;
31 elif(sys
.version
[0]>="3"):
32 from urllib
.parse
import urlunparse
;
33 from urllib
.parse
import urlparse
;
35 if os
.name
== 'nt': # Only modify if on Windows
36 if sys
.version
[0] == "2":
38 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
39 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
42 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
43 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
45 hashlib_guaranteed
= False;
46 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
47 os
.environ
["LC_CTYPE"] = "UTF-8";
51 sys
.setdefaultencoding('UTF-8');
54 except AttributeError:
58 except AttributeError:
62 from zlib
import crc32
;
64 from binascii
import crc32
;
66 if(sys
.version_info
[0]==2):
67 FileNotFoundError
= IOError;
69 rarfile_support
= False;
72 rarfile_support
= True;
74 rarfile_support
= False;
76 py7zr_support
= False;
81 py7zr_support
= False;
84 from xtarfile
import is_tarfile
;
87 from safetar
import is_tarfile
;
89 from tarfile
import is_tarfile
;
92 import xtarfile
as tarfile
;
95 import safetar
as tarfile
;
104 haveparamiko
= False;
113 haverequests
= False;
118 haverequests
= False;
122 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
126 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
127 from urllib
.parse
import urlparse
;
130 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
131 from urlparse
import urlparse
;
133 if(sys
.version
[0]=="2"):
135 from io
import StringIO
, BytesIO
;
138 from cStringIO
import StringIO
;
139 from cStringIO
import StringIO
as BytesIO
;
141 from StringIO
import StringIO
;
142 from StringIO
import StringIO
as BytesIO
;
143 elif(sys
.version
[0]>="3"):
144 from io
import StringIO
, BytesIO
;
149 from cStringIO
import StringIO
as BytesIO
;
155 from StringIO
import StringIO
as BytesIO
;
161 from io
import BytesIO
;
166 __use_pysftp__
= False;
168 __use_pysftp__
= False;
169 __file_format_name__
= "CatFile";
170 __program_name__
= "Py"+__file_format_name__
;
171 __file_format_lower__
= __file_format_name__
.lower();
172 __file_format_magic__
= __file_format_name__
;
173 __file_format_len__
= len(__file_format_magic__
);
174 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__
= "\x00";
176 __file_format_ver__
= "001";
177 __use_new_style__
= True;
178 __use_advanced_list__
= True;
179 __use_alt_inode__
= False;
180 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
181 __file_format_dict__
= {'format_name': __file_format_name__
, 'format_magic': __file_format_magic__
, 'format_lower': __file_format_lower__
, 'format_len': __file_format_len__
, 'format_hex': __file_format_hex__
, 'format_delimiter': __file_format_delimiter__
, 'format_ver': __file_format_ver__
, 'new_style': __use_new_style__
, 'use_advanced_list': __use_advanced_list__
, 'use_alt_inode': __use_alt_inode__
};
182 __project__
= __program_name__
;
183 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
184 __version_info__
= (0, 11, 0, "RC 1", 1);
185 __version_date_info__
= (2024, 5, 12, "RC 1", 1);
186 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
187 __revision__
= __version_info__
[3];
188 __revision_id__
= "$Id$";
189 if(__version_info__
[4] is not None):
190 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
191 if(__version_info__
[4] is None):
192 __version_date_plusrc__
= __version_date__
;
193 if(__version_info__
[3] is not None):
194 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
195 if(__version_info__
[3] is None):
196 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
198 PyBitness
= platform
.architecture();
199 if(PyBitness
=="32bit" or PyBitness
=="32"):
201 elif(PyBitness
=="64bit" or PyBitness
=="64"):
206 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
207 if(platform
.python_implementation()!=""):
208 py_implementation
= platform
.python_implementation();
209 if(platform
.python_implementation()==""):
210 py_implementation
= "Python";
211 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
212 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
213 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
214 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
215 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
216 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
217 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
219 compressionlist
= ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
220 compressionlistalt
= ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
221 outextlist
= ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
222 outextlistwd
= ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
224 tarfile_mimetype
= "application/tar";
225 tarfile_tar_mimetype
= tarfile_mimetype
;
226 zipfile_mimetype
= "application/zip";
227 zipfile_zip_mimetype
= zipfile_mimetype
;
228 rarfile_mimetype
= "application/rar";
229 rarfile_rar_mimetype
= rarfile_mimetype
;
230 archivefile_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"";
231 archivefile_cat_mimetype
= archivefile_mimetype
;
232 archivefile_gzip_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+gzip";
233 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
234 archivefile_bzip2_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+bzip2";
235 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
236 archivefile_lz4_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lz4";
237 archivefile_lzop_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzop";
238 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
239 archivefile_zstandard_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+zstandard";
240 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
241 archivefile_lzma_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzma";
242 archivefile_xz_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+xz";
243 archivefile_extensions
= ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
245 if __name__
== "__main__":
247 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
251 curscrpath
= curscrpath
.replace(os
.sep
, "/");
252 curscrpath
= curscrpath
+ "/";
253 scrfile
= curscrpath
+ "catfile.py";
254 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
255 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
258 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
264 "warning": logging
.warning
,
265 "error": logging
.error
,
266 "critical": logging
.critical
,
267 "exception": logging
.exception
,
268 "logalt": lambda x
: logging
.log(dgblevel
, x
),
269 "debug": logging
.debug
271 log_function
= log_functions
.get(outtype
);
273 log_function(dbgtxt
);
277 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
278 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
281 def RemoveWindowsPath(dpath
):
285 dpath
= dpath
.replace(os
.path
.sep
, "/");
286 dpath
= dpath
.rstrip("/");
287 if(dpath
=="." or dpath
==".."):
291 def NormalizeRelativePath(inpath
):
292 inpath
= RemoveWindowsPath(inpath
);
293 if(os
.path
.isabs(inpath
)):
296 if(inpath
.startswith("./") or inpath
.startswith("../")):
299 outpath
= "./" + inpath
;
302 def PrependPath(base_dir
, child_path
):
303 # Check if base_dir is None or empty, if so, return child_path as is
306 # Ensure base_dir ends with exactly one slash
307 if not base_dir
.endswith('/'):
309 # Check if child_path starts with ./ or ../ (indicating a relative path)
310 if child_path
.startswith('./') or child_path
.startswith('../'):
311 # For relative paths, we don't alter the child_path
312 return base_dir
+ child_path
;
314 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
315 return base_dir
+ child_path
.lstrip('/');
317 def ListDir(dirpath
, followlink
=False, duplicates
=False):
318 if(isinstance(dirpath
, (list, tuple, ))):
319 dirpath
= list(filter(None, dirpath
));
320 elif(isinstance(dirpath
, (str, ))):
321 dirpath
= list(filter(None, [dirpath
]));
323 for mydirfile
in dirpath
:
324 if(not os
.path
.exists(mydirfile
)):
326 mydirfile
= NormalizeRelativePath(mydirfile
);
327 if(os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
)):
328 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
329 if(os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
)):
330 for root
, dirs
, filenames
in os
.walk(mydirfile
):
332 dpath
= RemoveWindowsPath(dpath
);
333 if(dpath
not in retlist
and not duplicates
):
334 retlist
.append(dpath
);
336 retlist
.append(dpath
);
337 for file in filenames
:
338 fpath
= os
.path
.join(root
, file);
339 fpath
= RemoveWindowsPath(fpath
);
340 if(fpath
not in retlist
and not duplicates
):
341 retlist
.append(fpath
);
343 retlist
.append(fpath
);
345 retlist
.append(RemoveWindowsPath(mydirfile
));
348 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
349 if isinstance(dirpath
, (list, tuple)):
350 dirpath
= list(filter(None, dirpath
));
351 elif isinstance(dirpath
, str):
352 dirpath
= list(filter(None, [dirpath
]));
354 for mydirfile
in dirpath
:
355 if not os
.path
.exists(mydirfile
):
357 mydirfile
= NormalizeRelativePath(mydirfile
);
358 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
359 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
))
360 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
361 for root
, dirs
, filenames
in os
.walk(mydirfile
):
362 # Sort dirs and filenames alphabetically in place
363 dirs
.sort(key
=lambda x
: x
.lower());
364 filenames
.sort(key
=lambda x
: x
.lower());
365 dpath
= RemoveWindowsPath(root
);
366 if not duplicates
and dpath
not in retlist
:
367 retlist
.append(dpath
);
369 retlist
.append(dpath
);
370 for file in filenames
:
371 fpath
= os
.path
.join(root
, file);
372 fpath
= RemoveWindowsPath(fpath
);
373 if not duplicates
and fpath
not in retlist
:
374 retlist
.append(fpath
);
376 retlist
.append(fpath
);
378 retlist
.append(RemoveWindowsPath(mydirfile
));
381 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
382 # Define a new function that wraps the target function
383 def alias_function(*args
, **kwargs
):
384 return target_function(*args
, **kwargs
);
386 # Create the function name by combining the prefix, base name, and the suffix
387 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
389 # Add the new function to the global namespace
390 globals()[function_name
] = alias_function
;
392 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
393 # Create the function name by combining the prefix, base name, and the suffix
394 # Use the format method for string formatting, compatible with Python 2 and 3
395 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
396 # Add the new function (alias of the target_function) to the global namespace
397 # This line is compatible as-is with both Python 2 and 3
398 globals()[function_name
] = target_function
400 # initial_value can be 0xFFFF or 0x0000
401 def crc16_ansi(msg
, initial_value
=0xFFFF):
402 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
403 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
404 crc
= initial_value
; # Initial value
406 crc ^
= b
<< 8; # XOR byte into CRC top byte
407 for _
in range(8): # Process each bit
408 if crc
& 0x8000: # If the top bit is set
409 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
411 crc
= crc
<< 1; # Just shift left
412 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
415 # initial_value can be 0xFFFF or 0x0000
416 def crc16_ibm(msg
, initial_value
=0xFFFF):
417 return crc16_ansi(msg
, initial_value
);
419 # initial_value is 0xFFFF
421 return crc16_ansi(msg
, 0xFFFF);
423 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
424 def crc16_ccitt(msg
, initial_value
=0xFFFF):
425 # CRC-16-CCITT polynomial
426 poly
= 0x1021; # Polynomial for CRC-16-CCITT
427 # Use the specified initial value
430 crc ^
= b
<< 8; # XOR byte into CRC top byte
431 for _
in range(8): # Process each bit
432 if crc
& 0x8000: # If the top bit is set
433 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
435 crc
= crc
<< 1; # Just shift left
436 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
439 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
440 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
441 # CRC-64-ECMA polynomial and initial value
442 poly
= 0x42F0E1EBA9EA3693;
443 crc
= initial_value
; # Initial value for CRC-64-ECMA
445 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
446 for _
in range(8): # Process each bit
447 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
448 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
450 crc
<<= 1; # Just shift left if the MSB is 0
451 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
454 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
455 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
456 # CRC-64-ISO polynomial and initial value
457 poly
= 0x000000000000001B;
458 crc
= initial_value
; # Common initial value for CRC-64-ISO
460 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
461 for _
in range(8): # Process each bit
462 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
463 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
465 crc
<<= 1; # Just shift left if the MSB is 0
466 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
469 def GetDataFromArray(data
, path
, default
=None):
473 element
= element
[key
];
475 except (KeyError, TypeError, IndexError):
478 def GetDataFromArrayAlt(structure
, path
, default
=None):
481 if isinstance(element
, dict) and key
in element
:
482 element
= element
[key
];
483 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
484 element
= element
[key
];
489 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
490 if isinstance(inlist
, list):
491 fileheader
= AppendNullBytes(inlist
, formatspecs
['format_delimiter']);
493 fileheader
= AppendNullByte(inlist
, formatspecs
['format_delimiter']);
495 fileheader
= fileheader
.encode('UTF-8');
496 if(checksumtype
=="none" or checksumtype
==""):
497 catfileheadercshex
= format(0, 'x').lower();
498 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
499 catfileheadercshex
= format(crc16(fileheader
) & 0xffff, '04x').lower();
500 elif(checksumtype
=="crc16_ccitt"):
501 catfileheadercshex
= format(crc16_ccitt(fileheader
) & 0xffff, '04x').lower();
502 elif(checksumtype
=="adler32"):
503 catfileheadercshex
= format(zlib
.adler32(fileheader
) & 0xffffffff, '08x').lower();
504 elif(checksumtype
=="crc32"):
505 catfileheadercshex
= format(crc32(fileheader
) & 0xffffffff, '08x').lower();
506 elif(checksumtype
=="crc64_ecma"):
507 catfileheadercshex
= format(crc64_ecma(fileheader
) & 0xffffffffffffffff, '016x').lower();
508 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
509 catfileheadercshex
= format(crc64_iso(fileheader
) & 0xffffffffffffffff, '016x').lower();
510 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
511 checksumoutstr
= hashlib
.new(checksumtype
);
512 checksumoutstr
.update(fileheader
);
513 catfileheadercshex
= checksumoutstr
.hexdigest().lower();
515 catfileheadercshex
= format(0, 'x').lower();
516 return catfileheadercshex
;
518 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
520 instr
= instr
.encode('UTF-8');
521 if(checksumtype
=="none" or checksumtype
==""):
522 catinstrcshex
= format(0, 'x').lower();
523 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
524 catinstrcshex
= format(crc16(instr
) & 0xffff, '04x').lower();
525 elif(checksumtype
=="crc16_ccitt"):
526 catinstrcshex
= format(crc16_ccitt(instr
) & 0xffff, '04x').lower();
527 elif(checksumtype
=="adler32"):
528 catinstrcshex
= format(zlib
.adler32(instr
) & 0xffffffff, '08x').lower();
529 elif(checksumtype
=="crc32"):
530 catinstrcshex
= format(crc32(instr
) & 0xffffffff, '08x').lower();
531 elif(checksumtype
=="crc64_ecma"):
532 catinstrcshex
= format(crc64_ecma(instr
) & 0xffffffffffffffff, '016x').lower();
533 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
534 catinstrcshex
= format(crc64_iso(instr
) & 0xffffffffffffffff, '016x').lower();
535 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
536 checksumoutstr
= hashlib
.new(checksumtype
);
537 checksumoutstr
.update(instr
);
538 catinstrcshex
= checksumoutstr
.hexdigest().lower();
540 catinstrcshex
= format(0, 'x').lower();
541 return catinstrcshex
;
543 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
544 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
);
545 inchecksum
= inchecksum
.lower();
546 catfileheadercshex
= catfileheadercshex
.lower();
547 if(inchecksum
==catfileheadercshex
):
552 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
553 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
);
554 inchecksum
= inchecksum
.lower();
555 catinfilecshex
= catinfilecshex
.lower();
556 if(inchecksum
==catinfilecshex
):
561 def ReadTillNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
564 nullbyte
= delimiter
.encode("UTF-8");
566 curbyte
= fp
.read(1);
567 if(curbyte
==nullbyte
or not curbyte
):
569 curfullbyte
= curfullbyte
+ curbyte
;
570 return curfullbyte
.decode('UTF-8');
572 def ReadUntilNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
573 return ReadTillNullByteOld(fp
, delimiter
);
575 def ReadTillNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
576 delimiter
= delimiter
.encode('UTF-8') # Ensure the delimiter is in bytes
577 buffer = bytearray();
579 delimiter_length
= len(delimiter
);
581 chunk
= fp
.read(chunk_size
)
583 # End of file reached without finding the delimiter
585 buffer.extend(chunk
);
586 total_read
+= len(chunk
);
587 if delimiter
in buffer:
588 # Delimiter found, calculate where to reset the file pointer
589 index
= buffer.find(delimiter
);
590 # Calculate how many extra bytes were read after the delimiter
591 extra_bytes_read
= len(buffer) - (index
+ delimiter_length
);
592 # Move the file pointer back to just after the delimiter
593 fp
.seek(-extra_bytes_read
, 1);
594 buffer = buffer[:index
];
596 if total_read
>= max_read
:
597 # Stop reading if max limit is reached to prevent excessive memory usage
598 raise MemoryError("Maximum read limit reached without finding the delimiter.");
599 # Check for incomplete UTF-8 sequences at the end of the buffer
600 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
601 # This suggests that the last byte might be the start of a multi-byte character
602 # Try to read one more byte to complete the character
603 extra_byte
= fp
.read(1);
605 buffer.extend(extra_byte
);
607 # No more data available
610 return buffer.decode('UTF-8', errors
='replace');
611 except UnicodeDecodeError:
612 return buffer.decode('UTF-8', errors
='replace');
614 def ReadUntilNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
615 return ReadTillNullByteAlt(fp
, delimiter
, chunk_size
, max_read
);
617 def ReadTillNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
618 curfullbyte
= bytearray();
619 nullbyte
= delimiter
.encode("UTF-8");
620 total_read
= 0; # Track the total number of bytes read
622 curbyte
= fp
.read(1);
623 if curbyte
== nullbyte
or not curbyte
:
625 curfullbyte
.extend(curbyte
);
627 if total_read
>= max_read
:
628 raise MemoryError("Maximum read limit reached without finding the delimiter.");
629 # Decode the full byte array to string once out of the loop
631 return curfullbyte
.decode('UTF-8');
632 except UnicodeDecodeError:
633 # Handle potential partial UTF-8 characters
634 for i
in range(1, 4):
636 return curfullbyte
[:-i
].decode('UTF-8');
637 except UnicodeDecodeError:
639 raise; # Re-raise if decoding fails even after trimming
641 def ReadUntilNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
642 return ReadTillNullByte(fp
, delimiter
, max_read
);
644 def ReadTillNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
645 delimiter
= delimiter
.encode('UTF-8'); # Ensure the delimiter is in bytes
646 buffer = bytearray();
648 delimiter_length
= len(delimiter
);
650 while len(results
) < num_delimiters
:
651 chunk
= fp
.read(chunk_size
);
653 # End of file reached; decode whatever is collected if it's the last needed part
655 results
.append(buffer.decode('UTF-8', errors
='replace'));
658 total_read
+= len(chunk
);
659 # Check if we have found the delimiter
660 while delimiter
in buffer:
661 index
= buffer.find(delimiter
);
662 # Decode the section before the delimiter
663 results
.append(buffer[:index
].decode('UTF-8', errors
='replace'));
664 # Remove the processed part from the buffer
665 buffer = buffer[index
+ delimiter_length
:];
666 if len(results
) == num_delimiters
:
667 # If reached the required number of delimiters, adjust the file pointer and stop
668 fp
.seek(-len(buffer), 1);
670 if total_read
>= max_read
:
671 # Stop reading if max limit is reached to prevent excessive memory usage
672 raise MemoryError("Maximum read limit reached without finding the delimiter.");
673 # Check for incomplete UTF-8 sequences at the end of the buffer
674 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
675 # This suggests that the last byte might be the start of a multi-byte character
676 # Try to read one more byte to complete the character
677 extra_byte
= fp
.read(1);
679 buffer.extend(extra_byte
);
681 # No more data available
683 # Process remaining buffer if less than the required number of delimiters were found
684 if len(buffer) > 0 and len(results
) < num_delimiters
:
685 results
.append(buffer.decode('UTF-8', errors
='replace'));
688 def ReadUntilNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
689 return ReadTillNullByteByNum(fp
, delimiter
, num_delimiters
, chunk_size
, max_read
);
691 def SeekToEndOfFile(fp
):
695 if(lasttell
==fp
.tell()):
697 lasttell
= fp
.tell();
700 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_dict__
['format_delimiter']):
704 while(rocount
<roend
):
705 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
706 rocount
= rocount
+ 1;
709 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
710 headerpresize
= ReadTillNullByte(fp
, delimiter
);
711 headersize
= int(headerpresize
, 16);
714 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
717 roend
= int(len(headercontent
));
718 HeaderOut
= [headerpresize
];
719 while(rocount
<roend
):
720 HeaderOut
.append(headercontent
[rocount
]);
721 rocount
= rocount
+ 1;
724 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
725 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
726 headersize
= int(preheaderdata
[0], 16);
727 headernumfields
= int(preheaderdata
[1], 16);
728 if(headersize
<=0 or headernumfields
<=0):
730 headerdata
= ReadTillNullByteByNum(fp
, delimiter
, headernumfields
);
731 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
732 HeaderOut
= preheaderdata
+ headerdata
;
735 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
736 delimiter
= formatspecs
['format_delimiter'];
737 fheaderstart
= fp
.tell();
738 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
739 if(len(HeaderOut
)==0):
741 if(re
.findall("^[.|/]", HeaderOut
[3])):
742 fname
= HeaderOut
[3];
744 fname
= "./"+HeaderOut
[3];
745 fchecksumtype
= HeaderOut
[-3].lower();
746 fcs
= HeaderOut
[-2].lower();
747 fccs
= HeaderOut
[-1].lower();
748 fsize
= int(HeaderOut
[5], 16);
749 fcompression
= HeaderOut
[12];
750 fcsize
= int(HeaderOut
[13], 16);
751 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
752 if(fcs
!=newfcs
and not skipchecksum
):
753 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
754 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
756 fhend
= fp
.tell() - 1;
757 fcontentstart
= fp
.tell();
758 fcontents
= BytesIO();
759 if(fsize
>0 and not listonly
):
760 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
761 fcontents
.write(fp
.read(fsize
));
763 fcontents
.write(fp
.read(fcsize
));
764 elif(fsize
>0 and listonly
):
765 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
769 fcontents
.seek(0, 0);
770 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
771 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
772 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
773 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
775 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
778 fcontents
.seek(0, 0);
779 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
781 fcontentend
= fp
.tell() - 1;
782 HeaderOut
.append(fcontents
);
785 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
786 delimiter
= formatspecs
['format_delimiter'];
787 fheaderstart
= fp
.tell();
788 if(formatspecs
['new_style']):
789 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
791 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
792 if(len(HeaderOut
)==0):
794 fheadsize
= int(HeaderOut
[0], 16);
795 fnumfields
= int(HeaderOut
[1], 16);
796 ftype
= int(HeaderOut
[2], 16);
797 if(re
.findall("^[.|/]", HeaderOut
[3])):
798 fname
= HeaderOut
[3];
800 fname
= "./"+HeaderOut
[3];
801 fbasedir
= os
.path
.dirname(fname
);
802 flinkname
= HeaderOut
[4];
803 fsize
= int(HeaderOut
[5], 16);
804 fatime
= int(HeaderOut
[6], 16);
805 fmtime
= int(HeaderOut
[7], 16);
806 fctime
= int(HeaderOut
[8], 16);
807 fbtime
= int(HeaderOut
[9], 16);
808 fmode
= int(HeaderOut
[10], 16);
809 fchmode
= stat
.S_IMODE(fmode
);
810 ftypemod
= stat
.S_IFMT(fmode
);
811 fwinattributes
= int(HeaderOut
[11], 16);
812 fcompression
= HeaderOut
[12];
813 fcsize
= int(HeaderOut
[13], 16);
814 fuid
= int(HeaderOut
[14], 16);
815 funame
= HeaderOut
[15];
816 fgid
= int(HeaderOut
[16], 16);
817 fgname
= HeaderOut
[17];
818 fid
= int(HeaderOut
[18], 16);
819 finode
= int(HeaderOut
[19], 16);
820 flinkcount
= int(HeaderOut
[20], 16);
821 fdev_minor
= int(HeaderOut
[21], 16);
822 fdev_major
= int(HeaderOut
[22], 16);
823 frdev_minor
= int(HeaderOut
[23], 16);
824 frdev_major
= int(HeaderOut
[24], 16);
825 fextrasize
= int(HeaderOut
[25], 16);
826 fextrafields
= int(HeaderOut
[26], 16);
827 extrafieldslist
= [];
829 extraend
= extrastart
+ fextrafields
;
830 extrafieldslist
= [];
831 if(extrastart
<extraend
):
832 extrafieldslist
.append(HeaderOut
[extrastart
]);
833 extrastart
= extrastart
+ 1;
834 fchecksumtype
= HeaderOut
[extrastart
].lower();
835 fcs
= HeaderOut
[extrastart
+ 1].lower();
836 fccs
= HeaderOut
[extrastart
+ 2].lower();
837 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
838 if(fcs
!=newfcs
and not skipchecksum
):
839 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
840 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
842 fhend
= fp
.tell() - 1;
843 fcontentstart
= fp
.tell();
844 fcontents
= BytesIO();
845 pyhascontents
= False;
846 if(fsize
>0 and not listonly
):
847 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
848 fcontents
.write(fp
.read(fsize
));
850 fcontents
.write(fp
.read(fcsize
));
851 pyhascontents
= True;
852 elif(fsize
>0 and listonly
):
853 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
857 pyhascontents
= False;
858 fcontents
.seek(0, 0);
859 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
860 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
861 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
862 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
864 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
867 fcontents
.seek(0, 0);
868 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
870 fcontentend
= fp
.tell() - 1;
871 fcontents
.seek(0, 0);
872 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': fchecksumtype
, 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
875 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
876 delimiter
= formatspecs
['format_delimiter'];
877 fheaderstart
= fp
.tell();
878 if(formatspecs
['new_style']):
879 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
881 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
882 if(len(HeaderOut
)==0):
884 fheadsize
= int(HeaderOut
[0], 16);
885 fnumfields
= int(HeaderOut
[1], 16);
886 ftype
= int(HeaderOut
[2], 16);
887 if(re
.findall("^[.|/]", HeaderOut
[3])):
888 fname
= HeaderOut
[3];
890 fname
= "./"+HeaderOut
[3];
891 fbasedir
= os
.path
.dirname(fname
);
892 flinkname
= HeaderOut
[4];
893 fsize
= int(HeaderOut
[5], 16);
894 fatime
= int(HeaderOut
[6], 16);
895 fmtime
= int(HeaderOut
[7], 16);
896 fctime
= int(HeaderOut
[8], 16);
897 fbtime
= int(HeaderOut
[9], 16);
898 fmode
= int(HeaderOut
[10], 16);
899 fchmode
= stat
.S_IMODE(fmode
);
900 ftypemod
= stat
.S_IFMT(fmode
);
901 fwinattributes
= int(HeaderOut
[11], 16);
902 fcompression
= HeaderOut
[12];
903 fcsize
= int(HeaderOut
[13], 16);
904 fuid
= int(HeaderOut
[14], 16);
905 funame
= HeaderOut
[15];
906 fgid
= int(HeaderOut
[16], 16);
907 fgname
= HeaderOut
[17];
908 fid
= int(HeaderOut
[18], 16);
909 finode
= int(HeaderOut
[19], 16);
910 flinkcount
= int(HeaderOut
[20], 16);
911 fdev_minor
= int(HeaderOut
[21], 16);
912 fdev_major
= int(HeaderOut
[22], 16);
913 frdev_minor
= int(HeaderOut
[23], 16);
914 frdev_major
= int(HeaderOut
[24], 16);
915 fextrasize
= int(HeaderOut
[25], 16);
916 fextrafields
= int(HeaderOut
[26], 16);
917 extrafieldslist
= [];
919 extraend
= extrastart
+ fextrafields
;
920 extrafieldslist
= [];
921 if(extrastart
<extraend
):
922 extrafieldslist
.append(HeaderOut
[extrastart
]);
923 extrastart
= extrastart
+ 1;
924 fchecksumtype
= HeaderOut
[extrastart
].lower();
925 fcs
= HeaderOut
[extrastart
+ 1].lower();
926 fccs
= HeaderOut
[extrastart
+ 2].lower();
927 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
928 if(fcs
!=newfcs
and not skipchecksum
):
929 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
930 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
932 fhend
= fp
.tell() - 1;
933 fcontentstart
= fp
.tell();
934 fcontents
= BytesIO();
935 pyhascontents
= False;
936 if(fsize
>0 and not listonly
):
937 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
938 fcontents
.write(fp
.read(fsize
));
940 fcontents
.write(fp
.read(fcsize
));
941 pyhascontents
= True;
942 elif(fsize
>0 and listonly
):
943 if(fcompression
=="none" or fcompression
=="" or fcompression
=="atuo"):
947 pyhascontents
= False;
948 fcontents
.seek(0, 0);
949 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
950 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
951 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
952 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
954 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
957 fcontents
.seek(0, 0);
958 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
960 fcontentend
= fp
.tell() - 1;
961 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrafieldslist
, fchecksumtype
, fcontents
];
964 def ReadFileDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
965 delimiter
= formatspecs
['format_delimiter'];
969 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
972 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
973 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
974 if(not headercheck
and not skipchecksum
):
975 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
976 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
978 fnumfiles
= int(catheader
[1], 16);
981 while(countnum
< fnumfiles
):
982 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, skipchecksum
, formatspecs
)
983 if(len(HeaderOut
)==0):
985 flist
.append(HeaderOut
);
986 countnum
= countnum
+ 1;
989 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
990 delimiter
= formatspecs
['format_delimiter'];
994 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
997 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
998 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
999 if(not headercheck
and not skipchecksum
):
1000 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1001 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1003 catstring
= catheader
[0];
1004 catversion
= re
.findall(r
"([\d]+)$", catstring
);
1005 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
1006 fprenumfiles
= catheader
[1];
1007 fnumfiles
= int(fprenumfiles
, 16);
1008 fprechecksumtype
= catheader
[2];
1009 fprechecksum
= catheader
[3];
1010 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
1011 if(seekstart
<0 and seekstart
>fnumfiles
):
1013 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1014 seekend
= fnumfiles
;
1015 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1016 seekend
= fnumfiles
- abs(seekend
);
1019 while(il
< seekstart
):
1020 prefhstart
= fp
.tell();
1021 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1022 if(len(preheaderdata
)==0):
1024 prefsize
= int(preheaderdata
[5], 16);
1025 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
1026 prefcs
= preheaderdata
[-2];
1027 if(prefcs
!=prenewfcs
and not skipchecksum
):
1028 VVerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1029 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1031 valid_archive
= False;
1032 invalid_archive
= True;
1033 prefhend
= fp
.tell() - 1;
1034 prefcontentstart
= fp
.tell();
1035 prefcontents
= BytesIO();
1036 pyhascontents
= False;
1038 prefcontents
.write(fp
.read(prefsize
));
1039 prefcontents
.seek(0, 0);
1040 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
1041 prefccs
= preheaderdata
[-1];
1042 pyhascontents
= True;
1043 if(prefccs
!=prenewfccs
and not skipchecksum
):
1044 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1045 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1050 countnum
= seekstart
;
1051 while(countnum
< seekend
):
1052 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': realidnum
}});
1053 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
1054 if(len(HeaderOut
)==0):
1056 catlist
['ffilelist'][realidnum
].update(HeaderOut
);
1057 countnum
= countnum
+ 1;
1058 realidnum
= realidnum
+ 1;
1061 def ReadFileDataBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1062 if(isinstance(infile
, dict)):
1063 listcatfiles
= infile
;
1065 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1067 listcatfiles
= ReadFileDataBySizeWithContentToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
1068 if(not listcatfiles
):
1070 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1071 lenlist
= len(listcatfiles
['ffilelist']);
1073 lcfx
= int(listcatfiles
['fnumfiles']);
1074 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1075 lcfx
= int(lenlist
);
1077 lcfx
= int(listcatfiles
['fnumfiles']);
1079 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1080 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1081 catarray
['filetoid'].update(filetoidarray
);
1082 catarray
['idtofile'].update(idtofilearray
);
1083 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1084 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1085 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1086 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1087 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1088 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1089 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1090 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1091 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1092 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1093 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1094 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1095 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1096 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1097 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1098 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1099 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1100 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1101 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1102 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1103 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1104 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1105 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1106 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1107 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1108 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1109 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1110 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1111 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1112 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1113 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1117 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1118 delimiter
= formatspecs
['format_delimiter'];
1122 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1125 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1126 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1127 if(not headercheck
and not skipchecksum
):
1128 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1129 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1131 catstring
= catheader
[0];
1132 catversion
= re
.findall(r
"([\d]+)$", catstring
);
1133 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
1134 fprenumfiles
= catheader
[1];
1135 fnumfiles
= int(fprenumfiles
, 16);
1136 fprechecksumtype
= catheader
[2];
1137 fprechecksum
= catheader
[3];
1139 if(seekstart
<0 and seekstart
>fnumfiles
):
1141 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1142 seekend
= fnumfiles
;
1143 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1144 seekend
= fnumfiles
- abs(seekend
);
1147 while(il
< seekstart
):
1148 prefhstart
= fp
.tell();
1149 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1150 if(len(preheaderdata
)==0):
1152 prefsize
= int(preheaderdata
[5], 16);
1153 prefcompression
= preheaderdata
[12];
1154 prefcsize
= int(preheaderdata
[13], 16);
1155 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
1156 prefcs
= preheaderdata
[-2];
1157 if(prefcs
!=prenewfcs
and not skipchecksum
):
1158 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1159 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1161 valid_archive
= False;
1162 invalid_archive
= True;
1163 prefhend
= fp
.tell() - 1;
1164 prefcontentstart
= fp
.tell();
1166 pyhascontents
= False;
1168 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
1169 prefcontents
= catfp
.read(prefsize
);
1171 prefcontents
= catfp
.read(prefcsize
);
1172 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
1173 prefccs
= preheaderdata
[-1];
1174 pyhascontents
= True;
1175 if(prefccs
!=prenewfccs
and not skipchecksum
):
1176 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1177 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1182 countnum
= seekstart
;
1183 while(countnum
< seekend
):
1184 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, skipchecksum
, formatspecs
);
1185 if(len(HeaderOut
)==0):
1187 catlist
.append(HeaderOut
);
1188 countnum
= countnum
+ 1;
1189 realidnum
= realidnum
+ 1;
1192 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1193 delimiter
= formatspecs
['format_delimiter'];
1194 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1197 fp
= UncompressArchiveFile(fp
, formatspecs
);
1198 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1199 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1206 if(hasattr(sys
.stdin
, "buffer")):
1207 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1209 shutil
.copyfileobj(sys
.stdin
, fp
);
1211 fp
= UncompressArchiveFile(fp
, formatspecs
);
1215 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1216 fp
= download_file_from_internet_file(infile
);
1217 fp
= UncompressArchiveFile(fp
, formatspecs
);
1223 infile
= RemoveWindowsPath(infile
);
1224 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1225 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1227 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1228 if(not compresscheck
):
1229 fextname
= os
.path
.splitext(infile
)[1];
1230 if(fextname
==".gz"):
1231 compresscheck
= "gzip";
1232 elif(fextname
==".bz2"):
1233 compresscheck
= "bzip2";
1234 elif(fextname
==".zst"):
1235 compresscheck
= "zstd";
1236 elif(fextname
==".lz4" or fextname
==".clz4"):
1237 compresscheck
= "lz4";
1238 elif(fextname
==".lzo" or fextname
==".lzop"):
1239 compresscheck
= "lzo";
1240 elif(fextname
==".lzma" or fextname
==".xz"):
1241 compresscheck
= "lzma";
1244 if(not compresscheck
):
1246 fp
= UncompressFile(infile
, formatspecs
, "rb");
1247 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
1249 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1250 delimiter
= formatspecs
['format_delimiter'];
1251 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1254 fp
= UncompressArchiveFile(fp
, formatspecs
);
1255 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1256 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1263 if(hasattr(sys
.stdin
, "buffer")):
1264 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1266 shutil
.copyfileobj(sys
.stdin
, fp
);
1268 fp
= UncompressArchiveFile(fp
, formatspecs
);
1272 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1273 fp
= download_file_from_internet_file(infile
);
1274 fp
= UncompressArchiveFile(fp
, formatspecs
);
1280 infile
= RemoveWindowsPath(infile
);
1281 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1282 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1284 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1285 if(not compresscheck
):
1286 fextname
= os
.path
.splitext(infile
)[1];
1287 if(fextname
==".gz"):
1288 compresscheck
= "gzip";
1289 elif(fextname
==".bz2"):
1290 compresscheck
= "bzip2";
1291 elif(fextname
==".zst"):
1292 compresscheck
= "zstd";
1293 elif(fextname
==".lz4" or fextname
==".clz4"):
1294 compresscheck
= "lz4";
1295 elif(fextname
==".lzo" or fextname
==".lzop"):
1296 compresscheck
= "lzo";
1297 elif(fextname
==".lzma" or fextname
==".xz"):
1298 compresscheck
= "lzma";
1301 if(not compresscheck
):
1303 fp
= UncompressFile(infile
, formatspecs
, "rb");
1304 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
1306 def ReadInFileBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1307 if(isinstance(infile
, dict)):
1308 listcatfiles
= infile
;
1310 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1311 infile
= RemoveWindowsPath(infile
);
1312 listcatfiles
= ReadInFileBySizeWithContentToArray(infile
, listonly
, skipchecksum
, formatspecs
);
1313 if(not listcatfiles
):
1315 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1316 lenlist
= len(listcatfiles
['ffilelist']);
1318 lcfx
= int(listcatfiles
['fnumfiles']);
1319 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1320 lcfx
= int(lenlist
);
1322 lcfx
= int(listcatfiles
['fnumfiles']);
1324 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1325 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1326 catarray
['filetoid'].update(filetoidarray
);
1327 catarray
['idtofile'].update(idtofilearray
);
1328 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1329 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1330 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1331 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1332 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1333 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1334 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1335 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1336 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1337 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1338 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1339 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1340 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1341 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1342 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1343 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1344 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1345 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1346 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1347 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1348 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1349 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1350 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1351 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1352 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1353 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1354 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1355 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1356 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1357 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1358 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1362 def AppendNullByte(indata
, delimiter
=__file_format_dict__
['format_delimiter']):
1363 outdata
= str(indata
) + delimiter
;
1366 def AppendNullBytes(indata
=[], delimiter
=__file_format_dict__
['format_delimiter']):
1371 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1375 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1376 delimiter
= formatspecs
['format_delimiter'];
1377 catver
= formatspecs
['format_ver'];
1378 fileheaderver
= str(int(catver
.replace(".", "")));
1379 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
1380 fp
.write(fileheader
.encode('UTF-8'));
1381 fnumfiles
= format(int(numfiles
), 'x').lower();
1382 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
['format_delimiter']);
1383 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1384 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
1385 fp
.write(fnumfilesa
.encode('UTF-8'));
1388 os
.fsync(fp
.fileno());
1389 except io
.UnsupportedOperation
:
1391 except AttributeError:
1393 except OSError as e
:
1397 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1398 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1401 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_dict__
, returnfp
=False):
1402 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1403 if(os
.path
.exists(outfile
)):
1406 except OSError as e
:
1410 catfpfp
= BytesIO();
1411 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1413 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1416 fbasename
= os
.path
.splitext(outfile
)[0];
1417 fextname
= os
.path
.splitext(outfile
)[1];
1418 catfp
= CompressOpenFile(outfile
, True, compressionlevel
);
1419 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1420 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1421 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1424 os
.fsync(catfp
.fileno());
1425 except io
.UnsupportedOperation
:
1427 except AttributeError:
1429 except OSError as e
:
1433 if(hasattr(sys
.stdout
, "buffer")):
1434 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1436 shutil
.copyfileobj(catfp
, sys
.stdout
);
1437 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1438 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1440 upload_file_to_internet_file(catfp
, outfile
);
1448 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1449 extrafields
= format(len(extradata
), 'x').lower();
1450 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
1451 if(len(extradata
)>0):
1452 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1453 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1454 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1455 catoutlenhex
= format(catoutlen
, 'x').lower();
1456 catoutlist
= filevalues
;
1457 catoutlist
.insert(0, catoutlenhex
);
1458 catoutlist
.append(extrasizelen
);
1459 catoutlist
.append(extrafields
);
1460 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
1461 if(len(extradata
)>0):
1462 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1463 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
1464 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1465 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1466 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1467 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1468 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
1469 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1470 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1471 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1472 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
1473 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1474 fp
.write(catfileout
);
1477 os
.fsync(fp
.fileno());
1478 except io
.UnsupportedOperation
:
1480 except AttributeError:
1482 except OSError as e
:
1486 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1487 advancedlist
= formatspecs
['use_advanced_list'];
1488 altinode
= formatspecs
['use_alt_inode'];
1490 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1492 for line
in sys
.stdin
:
1493 infilelist
.append(line
.strip());
1494 infilelist
= list(filter(None, infilelist
));
1495 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1496 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1498 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1499 for line
in finfile
:
1500 infilelist
.append(line
.strip());
1501 infilelist
= list(filter(None, infilelist
));
1503 if(isinstance(infiles
, (list, tuple, ))):
1504 infilelist
= list(filter(None, infiles
));
1505 elif(isinstance(infiles
, (str, ))):
1506 infilelist
= list(filter(None, [infiles
]));
1508 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1510 GetDirList
= ListDir(infilelist
, followlink
, False);
1518 inodetocatinode
= {};
1519 numfiles
= int(len(GetDirList
));
1520 fnumfiles
= format(numfiles
, 'x').lower();
1521 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1522 for curfname
in GetDirList
:
1523 if(re
.findall("^[.|/]", curfname
)):
1526 fname
= "./"+curfname
;
1528 VerbosePrintOut(fname
);
1529 if(not followlink
or followlink
is None):
1530 fstatinfo
= os
.lstat(fname
);
1532 fstatinfo
= os
.stat(fname
);
1533 fpremode
= fstatinfo
.st_mode
;
1534 finode
= fstatinfo
.st_ino
;
1535 flinkcount
= fstatinfo
.st_nlink
;
1537 if(stat
.S_ISREG(fpremode
)):
1539 elif(stat
.S_ISLNK(fpremode
)):
1541 elif(stat
.S_ISCHR(fpremode
)):
1543 elif(stat
.S_ISBLK(fpremode
)):
1545 elif(stat
.S_ISDIR(fpremode
)):
1547 elif(stat
.S_ISFIFO(fpremode
)):
1549 elif(stat
.S_ISSOCK(fpremode
)):
1551 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1553 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1555 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1560 fcurfid
= format(int(curfid
), 'x').lower();
1561 if(not followlink
and finode
!=0):
1563 if(finode
in inodelist
):
1565 flinkname
= inodetofile
[finode
];
1567 fcurinode
= format(int(finode
), 'x').lower();
1569 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1570 if(finode
not in inodelist
):
1571 inodelist
.append(finode
);
1572 inodetofile
.update({finode
: fname
});
1573 inodetocatinode
.update({finode
: curinode
});
1575 fcurinode
= format(int(finode
), 'x').lower();
1577 fcurinode
= format(int(curinode
), 'x').lower();
1578 curinode
= curinode
+ 1;
1580 fcurinode
= format(int(curinode
), 'x').lower();
1581 curinode
= curinode
+ 1;
1582 curfid
= curfid
+ 1;
1584 flinkname
= os
.readlink(fname
);
1585 fdev
= fstatinfo
.st_dev
;
1586 getfdev
= GetDevMajorMinor(fdev
);
1587 fdev_minor
= getfdev
[0];
1588 fdev_major
= getfdev
[1];
1589 frdev
= fstatinfo
.st_dev
;
1590 if(hasattr(fstatinfo
, "st_rdev")):
1591 frdev
= fstatinfo
.st_rdev
;
1593 frdev
= fstatinfo
.st_dev
;
1594 getfrdev
= GetDevMajorMinor(frdev
);
1595 frdev_minor
= getfrdev
[0];
1596 frdev_major
= getfrdev
[1];
1597 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1598 fsize
= format(int("0"), 'x').lower();
1599 elif(ftype
==0 or ftype
==7):
1600 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1602 fsize
= format(int(fstatinfo
.st_size
)).lower();
1603 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1604 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1605 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1606 if(hasattr(fstatinfo
, "st_birthtime")):
1607 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1609 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1610 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1611 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1612 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1613 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1614 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1619 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1620 funame
= userinfo
.pw_name
;
1629 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1630 fgname
= groupinfo
.gr_name
;
1635 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1636 fdev_major
= format(int(fdev_major
), 'x').lower();
1637 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1638 frdev_major
= format(int(frdev_major
), 'x').lower();
1639 finode
= format(int(finode
), 'x').lower();
1640 flinkcount
= format(int(flinkcount
), 'x').lower();
1641 if(hasattr(fstatinfo
, "st_file_attributes")):
1642 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1644 fwinattributes
= format(int(0), 'x').lower();
1646 fcsize
= format(int(0), 'x').lower();
1647 fcontents
= BytesIO();
1649 if(ftype
==0 or ftype
==7):
1650 with
open(fname
, "rb") as fpc
:
1651 shutil
.copyfileobj(fpc
, fcontents
);
1652 if(not compresswholefile
):
1653 fcontents
.seek(0, 2);
1654 ucfsize
= fcontents
.tell();
1655 fcontents
.seek(0, 0);
1656 if(compression
=="auto"):
1657 ilsize
= len(compressionlistalt
);
1660 while(ilmin
< ilsize
):
1661 cfcontents
= BytesIO();
1662 shutil
.copyfileobj(fcontents
, cfcontents
);
1663 fcontents
.seek(0, 0);
1664 cfcontents
.seek(0, 0);
1665 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1667 cfcontents
.seek(0, 2);
1668 ilcsize
.append(cfcontents
.tell());
1672 ilcsize
.append(sys
.maxint
);
1673 except AttributeError:
1674 ilcsize
.append(sys
.maxsize
);
1676 ilcmin
= ilcsize
.index(min(ilcsize
));
1677 compression
= compressionlistalt
[ilcmin
];
1678 fcontents
.seek(0, 0);
1679 cfcontents
= BytesIO();
1680 shutil
.copyfileobj(fcontents
, cfcontents
);
1681 cfcontents
.seek(0, 0);
1682 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1683 cfcontents
.seek(0, 2);
1684 cfsize
= cfcontents
.tell();
1685 if(ucfsize
> cfsize
):
1686 fcsize
= format(int(cfsize
), 'x').lower();
1687 fcompression
= compression
;
1689 fcontents
= cfcontents
;
1690 if(followlink
and (ftype
==1 or ftype
==2)):
1691 flstatinfo
= os
.stat(flinkname
);
1692 with
open(flinkname
, "rb") as fpc
:
1693 shutil
.copyfileobj(fpc
, fcontents
);
1694 if(not compresswholefile
):
1695 fcontents
.seek(0, 2);
1696 ucfsize
= fcontents
.tell();
1697 fcontents
.seek(0, 0);
1698 if(compression
=="auto"):
1699 ilsize
= len(compressionlistalt
);
1702 while(ilmin
< ilsize
):
1703 cfcontents
= BytesIO();
1704 shutil
.copyfileobj(fcontents
, cfcontents
);
1705 fcontents
.seek(0, 0);
1706 cfcontents
.seek(0, 0);
1707 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1709 cfcontents
.seek(0, 2);
1710 ilcsize
.append(cfcontents
.tell());
1714 ilcsize
.append(sys
.maxint
);
1715 except AttributeError:
1716 ilcsize
.append(sys
.maxsize
);
1718 ilcmin
= ilcsize
.index(min(ilcsize
));
1719 compression
= compressionlistalt
[ilcmin
];
1720 fcontents
.seek(0, 0);
1721 cfcontents
= BytesIO();
1722 shutil
.copyfileobj(fcontents
, cfcontents
);
1723 cfcontents
.seek(0, 0);
1724 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1725 cfcontents
.seek(0, 2);
1726 cfsize
= cfcontents
.tell();
1727 if(ucfsize
> cfsize
):
1728 fcsize
= format(int(cfsize
), 'x').lower();
1729 fcompression
= compression
;
1731 fcontents
= cfcontents
;
1732 if(fcompression
=="none"):
1734 fcontents
.seek(0, 0);
1735 ftypehex
= format(ftype
, 'x').lower();
1736 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1737 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1739 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1743 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1745 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1746 GetDirList
= inlist
;
1754 inodetocatinode
= {};
1755 numfiles
= int(len(GetDirList
));
1756 fnumfiles
= format(numfiles
, 'x').lower();
1757 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1758 for curfname
in GetDirList
:
1759 ftype
= format(curfname
[0], 'x').lower();
1760 if(re
.findall("^[.|/]", curfname
[1])):
1761 fname
= curfname
[1];
1763 fname
= "./"+curfname
[1];
1764 fbasedir
= os
.path
.dirname(fname
);
1765 flinkname
= curfname
[2];
1766 fsize
= format(curfname
[3], 'x').lower();
1767 fatime
= format(curfname
[4], 'x').lower();
1768 fmtime
= format(curfname
[5], 'x').lower();
1769 fctime
= format(curfname
[6], 'x').lower();
1770 fbtime
= format(curfname
[7], 'x').lower();
1771 fmode
= format(curfname
[8], 'x').lower();
1772 fwinattributes
= format(curfname
[9], 'x').lower();
1773 fcompression
= curfname
[10];
1774 fcsize
= format(curfname
[11], 'x').lower();
1775 fuid
= format(curfname
[12], 'x').lower();
1776 funame
= curfname
[13];
1777 fgid
= format(curfname
[14], 'x').lower();
1778 fgname
= curfname
[15];
1779 fid
= format(curfname
[16], 'x').lower();
1780 finode
= format(curfname
[17], 'x').lower();
1781 flinkcount
= format(curfname
[18], 'x').lower();
1782 fdev_minor
= format(curfname
[19], 'x').lower();
1783 fdev_major
= format(curfname
[20], 'x').lower();
1784 frdev_minor
= format(curfname
[21], 'x').lower();
1785 frdev_major
= format(curfname
[22], 'x').lower();
1786 extradata
= curfname
[23];
1787 fchecksumtype
= curfname
[24];
1788 fcontents
= curfname
[25];
1789 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1790 fcontents
.seek(0, 0);
1791 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1793 fp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1796 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1797 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1798 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1800 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1801 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1802 if(os
.path
.exists(outfile
)):
1805 except OSError as e
:
1809 catfpfp
= BytesIO();
1810 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1812 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1815 fbasename
= os
.path
.splitext(outfile
)[0];
1816 fextname
= os
.path
.splitext(outfile
)[1];
1817 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1818 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1819 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1820 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1823 os
.fsync(catfp
.fileno());
1824 except io
.UnsupportedOperation
:
1826 except AttributeError:
1828 except OSError as e
:
1832 if(hasattr(sys
.stdout
, "buffer")):
1833 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1835 shutil
.copyfileobj(catfp
, sys
.stdout
);
1836 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1837 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1839 upload_file_to_internet_file(catfp
, outfile
);
1847 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1848 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1849 if(os
.path
.exists(outfile
)):
1852 except OSError as e
:
1856 catfpfp
= BytesIO();
1857 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1859 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1862 fbasename
= os
.path
.splitext(outfile
)[0];
1863 fextname
= os
.path
.splitext(outfile
)[1];
1864 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1865 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1866 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1867 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1870 os
.fsync(catfp
.fileno());
1871 except io
.UnsupportedOperation
:
1873 except AttributeError:
1875 except OSError as e
:
1879 if(hasattr(sys
.stdout
, "buffer")):
1880 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1882 shutil
.copyfileobj(catfp
, sys
.stdout
);
1883 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1884 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1886 upload_file_to_internet_file(catfp
, outfile
);
1894 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1895 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1896 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
1898 def PrintPermissionString(fchmode
, ftype
):
1899 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1901 for fmodval
in str(oct(fchmode
))[-3:]:
1902 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
1903 if(ftype
==0 or ftype
==7):
1904 permissionstr
= "-" + permissionstr
;
1906 permissionstr
= "h" + permissionstr
;
1908 permissionstr
= "l" + permissionstr
;
1910 permissionstr
= "c" + permissionstr
;
1912 permissionstr
= "b" + permissionstr
;
1914 permissionstr
= "d" + permissionstr
;
1916 permissionstr
= "f" + permissionstr
;
1918 permissionstr
= "D" + permissionstr
;
1920 permissionstr
= "p" + permissionstr
;
1922 permissionstr
= "w" + permissionstr
;
1924 permissionoutstr
= stat
.filemode(fchmode
);
1925 except AttributeError:
1926 permissionoutstr
= permissionstr
;
1928 permissionoutstr
= permissionstr
;
1929 return permissionoutstr
;
1931 def PrintPermissionStringAlt(fchmode
, ftype
):
1933 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1934 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1936 # Translate file mode into permission string
1937 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
1938 # Append file type indicator
1940 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1941 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1943 file_type
= type_indicators
.get(ftype
, '-');
1944 permissionstr
= file_type
+ permissionstr
;
1946 permissionoutstr
= stat
.filemode(fchmode
);
1947 except AttributeError:
1948 permissionoutstr
= permissionstr
;
1949 return permissionoutstr
;
1951 def CompressionSupport():
1952 compression_list
= [];
1955 compression_list
.append("gz");
1956 compression_list
.append("gzip");
1961 compression_list
.append("bz2");
1962 compression_list
.append("bzip2");
1967 compression_list
.append("lz4");
1972 compression_list
.append("lzo");
1973 compression_list
.append("lzop");
1978 compression_list
.append("zstd");
1979 compression_list
.append("zstandard");
1984 compression_list
.append("lzma");
1985 compression_list
.append("xz");
1988 return compression_list
;
1990 def CheckCompressionType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
1991 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1995 catfp
= open(infile
, "rb");
1996 except FileNotFoundError
:
1999 prefp
= catfp
.read(2);
2001 if(prefp
==binascii
.unhexlify("1f8b")):
2004 prefp
= catfp
.read(3);
2005 if(prefp
==binascii
.unhexlify("425a68")):
2007 if(prefp
==binascii
.unhexlify("5d0000")):
2010 prefp
= catfp
.read(4);
2011 if(prefp
==binascii
.unhexlify("28b52ffd")):
2013 if(prefp
==binascii
.unhexlify("04224d18")):
2015 if(prefp
==binascii
.unhexlify("504B0304")):
2016 filetype
= "zipfile";
2018 prefp
= catfp
.read(5);
2019 if(prefp
==binascii
.unhexlify("7573746172")):
2020 filetype
= "tarfile";
2022 prefp
= catfp
.read(6);
2023 if(prefp
==binascii
.unhexlify("fd377a585a00")):
2025 if(prefp
==binascii
.unhexlify("377abcaf271c")):
2026 filetype
= "7zipfile";
2028 prefp
= catfp
.read(7);
2029 if(prefp
==binascii
.unhexlify("526172211a0700")):
2030 filetype
= "rarfile";
2031 if(prefp
==binascii
.unhexlify("43617446696c65")):
2032 filetype
= "catfile";
2034 prefp
= catfp
.read(8);
2035 if(prefp
==binascii
.unhexlify("526172211a070100")):
2036 filetype
= "rarfile";
2038 prefp
= catfp
.read(formatspecs
['format_len']);
2039 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2040 filetype
= formatspecs
['format_lower'];
2042 prefp
= catfp
.read(9);
2043 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
2046 prefp
= catfp
.read(10);
2047 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2048 filetype
= "tarfile";
2050 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
2051 if(is_tarfile(catfp
)):
2052 filetype
= "tarfile";
2054 if(is_tarfile(catfp
)):
2055 filetype
= "tarfile";
2056 elif(zipfile
.is_zipfile(catfp
)):
2057 filetype
= "zipfile";
2058 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
2059 filetype
= "rarile";
2067 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_dict__
, closefp
=True):
2069 instringsfile
= BytesIO(instring
);
2071 instringsfile
= BytesIO(instring
.encode("UTF-8"));
2072 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
2074 def GetCompressionMimeType(infile
, formatspecs
=__file_format_dict__
):
2075 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2076 if(compresscheck
=="gzip" or compresscheck
=="gz"):
2077 return archivefile_gzip_mimetype
;
2078 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
2079 return archivefile_bzip2_mimetype
;
2080 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
2081 return archivefile_zstandard_mimetype
;
2082 if(compresscheck
=="lz4"):
2083 return archivefile_lz4_mimetype
;
2084 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2085 return archivefile_lzop_mimetype
;
2086 if(compresscheck
=="lzma"):
2087 return archivefile_lzma_mimetype
;
2088 if(compresscheck
=="xz"):
2089 return archivefile_xz_mimetype
;
2090 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
['format_lower']):
2091 return archivefile_cat_mimetype
;
2092 if(not compresscheck
):
2096 def UncompressArchiveFile(fp
, formatspecs
=__file_format_dict__
):
2097 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2099 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2100 if(compresscheck
=="gzip"):
2105 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
2106 if(compresscheck
=="bzip2"):
2112 catfp
.write(bz2
.decompress(fp
.read()));
2113 if(compresscheck
=="zstd"):
2119 catfp
.write(zstandard
.decompress(fp
.read()));
2120 if(compresscheck
=="lz4"):
2126 catfp
.write(lz4
.frame
.decompress(fp
.read()));
2127 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2133 catfp
.write(lzo
.decompress(fp
.read()));
2134 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2140 catfp
.write(lzma
.decompress(fp
.read()));
2141 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2143 if(not compresscheck
):
2149 with fp
as fpcontent
:
2151 catfp
.write(lzma
.decompress(fp
.read()));
2152 except lzma
.LZMAError
:
2154 if(compresscheck
!="catfile" or compresscheck
!=formatspecs
['format_lower']):
2158 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
2160 def UncompressFile(infile
, formatspecs
=__file_format_dict__
, mode
="rb"):
2161 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2162 if(sys
.version_info
[0]==2 and compresscheck
):
2168 if(compresscheck
=="gzip"):
2174 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
2175 except (ValueError, TypeError) as e
:
2176 filefp
= gzip
.open(infile
, mode
);
2177 if(compresscheck
=="bzip2"):
2183 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
2184 except (ValueError, TypeError) as e
:
2185 filefp
= bz2
.open(infile
, mode
);
2186 if(compresscheck
=="zstd"):
2192 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
2193 except (ValueError, TypeError) as e
:
2194 filefp
= zstandard
.open(infile
, mode
);
2195 if(compresscheck
=="lz4"):
2201 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
2202 except (ValueError, TypeError) as e
:
2203 filefp
= lz4
.frame
.open(infile
, mode
);
2204 if(compresscheck
=="lzo"):
2210 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
2211 except (ValueError, TypeError) as e
:
2212 filefp
= lzo
.open(infile
, mode
);
2213 if(compresscheck
=="lzma"):
2219 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
2220 except (ValueError, TypeError) as e
:
2221 filefp
= lzma
.open(infile
, mode
);
2222 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2224 filefp
= open(infile
, mode
, encoding
="UTF-8");
2225 except (ValueError, TypeError) as e
:
2226 filefp
= open(infile
, mode
);
2227 if(not compresscheck
):
2229 filefp
= open(infile
, mode
, encoding
="UTF-8");
2230 except (ValueError, TypeError) as e
:
2231 filefp
= open(infile
, mode
);
2232 except FileNotFoundError
:
2236 def UncompressString(infile
):
2237 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
2238 if(compresscheck
=="gzip"):
2243 fileuz
= gzip
.decompress(infile
);
2244 if(compresscheck
=="bzip2"):
2249 fileuz
= bz2
.decompress(infile
);
2250 if(compresscheck
=="zstd"):
2255 fileuz
= zstandard
.decompress(infile
);
2256 if(compresscheck
=="lz4"):
2261 fileuz
= lz4
.frame
.decompress(infile
);
2262 if(compresscheck
=="lzo"):
2267 fileuz
= lzo
.decompress(infile
);
2268 if(compresscheck
=="lzma"):
2273 fileuz
= lzma
.decompress(infile
);
2274 if(not compresscheck
):
2276 if(hasattr(fileuz
, 'decode')):
2277 fileuz
= fileuz
.decode("UTF-8");
2280 def UncompressStringAlt(infile
):
2281 filefp
= StringIO();
2282 outstring
= UncompressString(infile
);
2283 filefp
.write(outstring
);
2287 def CheckCompressionSubType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2288 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2289 if(not compresscheck
):
2290 fextname
= os
.path
.splitext(infile
)[1];
2291 if(fextname
==".gz"):
2292 compresscheck
= "gzip";
2293 elif(fextname
==".bz2"):
2294 compresscheck
= "bzip2";
2295 elif(fextname
==".zst"):
2296 compresscheck
= "zstd";
2297 elif(fextname
==".lz4"):
2298 compresscheck
= "lz4";
2299 elif(fextname
==".lzo" or fextname
==".lzop"):
2300 compresscheck
= "lzo";
2301 elif(fextname
==".lzma" or fextname
==".xz"):
2302 compresscheck
= "lzma";
2305 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2306 if(is_tarfile(infile
)):
2307 filetype
= "tarfile";
2308 if(not compresscheck
):
2309 if(is_tarfile(infile
)):
2311 elif(zipfile
.is_zipfile(infile
)):
2313 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2318 if(compresscheck
=="catfile"):
2320 if(compresscheck
==formatspecs
['format_lower']):
2321 return formatspecs
['format_lower'];
2322 if(compresscheck
=="tarfile"):
2324 if(compresscheck
=="zipfile"):
2326 if(rarfile_support
and compresscheck
=="rarfile"):
2328 if(py7zr_support
and compresscheck
=="7zipfile"):
2330 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2331 catfp
= UncompressArchiveFile(infile
, formatspecs
['format_lower']);
2334 if(compresscheck
=="gzip"):
2339 catfp
= gzip
.GzipFile(infile
, "rb");
2340 if(compresscheck
=="bzip2"):
2345 catfp
= bz2
.BZ2File(infile
, "rb");
2346 if(compresscheck
=="lz4"):
2351 catfp
= lz4
.frame
.open(infile
, "rb");
2352 if(compresscheck
=="zstd"):
2357 catfp
= zstandard
.open(infile
, "rb");
2358 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2363 catfp
= lzma
.open(infile
, "rb");
2364 except FileNotFoundError
:
2367 prefp
= catfp
.read(5);
2368 if(prefp
==binascii
.unhexlify("7573746172")):
2369 filetype
= "tarfile";
2371 prefp
= catfp
.read(7);
2372 if(prefp
==binascii
.unhexlify("43617446696c65")):
2373 filetype
= "catfile";
2375 prefp
= catfp
.read(formatspecs
['format_len']);
2376 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2377 filetype
= formatspecs
['format_lower'];
2379 prefp
= catfp
.read(10);
2380 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2381 filetype
= "tarfile";
2387 def GZipCompress(data
, compresslevel
=9):
2392 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2394 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2398 catfp
= open(tmpfp
.name
, "rb");
2399 except FileNotFoundError
:
2401 catdata
= catfp
.read();
2405 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_dict__
):
2406 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2409 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2410 compression
= "auto";
2411 if(compression
not in compressionlist
and compression
is None):
2412 compression
= "auto";
2413 if(compression
=="gzip"):
2419 if(compressionlevel
is None):
2420 compressionlevel
= 9;
2422 compressionlevel
= int(compressionlevel
);
2423 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2424 if(compression
=="bzip2"):
2430 if(compressionlevel
is None):
2431 compressionlevel
= 9;
2433 compressionlevel
= int(compressionlevel
);
2434 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2435 if(compression
=="lz4"):
2441 if(compressionlevel
is None):
2442 compressionlevel
= 9;
2444 compressionlevel
= int(compressionlevel
);
2445 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2446 if(compression
=="lzo" or compression
=="lzop"):
2452 if(compressionlevel
is None):
2453 compressionlevel
= 9;
2455 compressionlevel
= int(compressionlevel
);
2456 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2457 if(compression
=="zstd"):
2463 if(compressionlevel
is None):
2464 compressionlevel
= 10;
2466 compressionlevel
= int(compressionlevel
);
2467 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2468 if(compression
=="lzma"):
2474 if(compressionlevel
is None):
2475 compressionlevel
= 9;
2477 compressionlevel
= int(compressionlevel
);
2478 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2479 if(compression
=="xz"):
2485 if(compressionlevel
is None):
2486 compressionlevel
= 9;
2488 compressionlevel
= int(compressionlevel
);
2489 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2490 if(compression
=="auto" or compression
is None):
2495 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2497 def CompressOpenFile(outfile
, compressionenable
=True, compressionlevel
=None):
2498 if(outfile
is None):
2500 fbasename
= os
.path
.splitext(outfile
)[0];
2501 fextname
= os
.path
.splitext(outfile
)[1];
2502 if(compressionlevel
is None and fextname
!=".zst"):
2503 compressionlevel
= 9;
2504 elif(compressionlevel
is None and fextname
==".zst"):
2505 compressionlevel
= 10;
2507 compressionlevel
= int(compressionlevel
);
2508 if(sys
.version_info
[0]==2):
2513 if(fextname
not in outextlistwd
or not compressionenable
):
2515 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2516 except (ValueError, TypeError) as e
:
2517 outfp
= open(outfile
, "wb");
2518 elif(fextname
==".gz"):
2524 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2525 except (ValueError, TypeError) as e
:
2526 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2527 elif(fextname
==".bz2"):
2533 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2534 except (ValueError, TypeError) as e
:
2535 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2536 elif(fextname
==".zst"):
2542 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2543 except (ValueError, TypeError) as e
:
2544 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2545 elif(fextname
==".xz"):
2551 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2552 except (ValueError, TypeError) as e
:
2553 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2554 elif(fextname
==".lz4"):
2560 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2561 except (ValueError, TypeError) as e
:
2562 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2563 elif(fextname
==".lzo"):
2569 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2570 except (ValueError, TypeError) as e
:
2571 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2572 elif(fextname
==".lzma"):
2578 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2579 except (ValueError, TypeError) as e
:
2580 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2581 except FileNotFoundError
:
2585 def GetDevMajorMinor(fdev
):
2587 if(hasattr(os
, "minor")):
2588 retdev
.append(os
.minor(fdev
));
2591 if(hasattr(os
, "major")):
2592 retdev
.append(os
.major(fdev
));
2597 def CheckSumSupport(checkfor
, guaranteed
=True):
2600 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2601 except AttributeError:
2602 hash_list
= sorted(list(hashlib
.algorithms
));
2605 hash_list
= sorted(list(hashlib
.algorithms_available
));
2606 except AttributeError:
2607 hash_list
= sorted(list(hashlib
.algorithms
));
2608 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2609 if(checkfor
in checklistout
):
2614 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2617 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2618 except AttributeError:
2619 hash_list
= sorted(list(hashlib
.algorithms
));
2622 hash_list
= sorted(list(hashlib
.algorithms_available
));
2623 except AttributeError:
2624 hash_list
= sorted(list(hashlib
.algorithms
));
2625 checklistout
= hash_list
;
2626 if(checkfor
in checklistout
):
2631 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2632 advancedlist
= formatspecs
['use_advanced_list'];
2633 altinode
= formatspecs
['use_alt_inode'];
2634 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2635 outfile
= RemoveWindowsPath(outfile
);
2636 checksumtype
= checksumtype
.lower();
2637 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2638 checksumtype
="crc32";
2639 if(checksumtype
=="none"):
2641 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2642 compression
= "auto";
2643 if(compression
not in compressionlist
and compression
is None):
2644 compression
= "auto";
2646 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2647 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2648 if(os
.path
.exists(outfile
)):
2651 except OSError as e
:
2656 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2658 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2661 fbasename
= os
.path
.splitext(outfile
)[0];
2662 fextname
= os
.path
.splitext(outfile
)[1];
2663 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2664 catver
= formatspecs
['format_ver'];
2665 fileheaderver
= str(int(catver
.replace(".", "")));
2668 for line
in sys
.stdin
:
2669 infilelist
.append(line
.strip());
2670 infilelist
= list(filter(None, infilelist
));
2671 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2672 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2674 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2675 for line
in finfile
:
2676 infilelist
.append(line
.strip());
2677 infilelist
= list(filter(None, infilelist
));
2679 if(isinstance(infiles
, (list, tuple, ))):
2680 infilelist
= list(filter(None, infiles
));
2681 elif(isinstance(infiles
, (str, ))):
2682 infilelist
= list(filter(None, [infiles
]));
2684 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2686 GetDirList
= ListDir(infilelist
, followlink
, False);
2694 inodetocatinode
= {};
2695 numfiles
= int(len(GetDirList
));
2696 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2697 for curfname
in GetDirList
:
2698 if(re
.findall("^[.|/]", curfname
)):
2701 fname
= "./"+curfname
;
2703 VerbosePrintOut(fname
);
2704 if(not followlink
or followlink
is None):
2705 fstatinfo
= os
.lstat(fname
);
2707 fstatinfo
= os
.stat(fname
);
2708 fpremode
= fstatinfo
.st_mode
;
2709 finode
= fstatinfo
.st_ino
;
2710 flinkcount
= fstatinfo
.st_nlink
;
2712 if(stat
.S_ISREG(fpremode
)):
2714 elif(stat
.S_ISLNK(fpremode
)):
2716 elif(stat
.S_ISCHR(fpremode
)):
2718 elif(stat
.S_ISBLK(fpremode
)):
2720 elif(stat
.S_ISDIR(fpremode
)):
2722 elif(stat
.S_ISFIFO(fpremode
)):
2724 elif(stat
.S_ISSOCK(fpremode
)):
2726 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2728 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2730 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2735 fcurfid
= format(int(curfid
), 'x').lower();
2736 if(not followlink
and finode
!=0):
2738 if(finode
in inodelist
):
2740 flinkname
= inodetofile
[finode
];
2742 fcurinode
= format(int(finode
), 'x').lower();
2744 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2745 if(finode
not in inodelist
):
2746 inodelist
.append(finode
);
2747 inodetofile
.update({finode
: fname
});
2748 inodetocatinode
.update({finode
: curinode
});
2750 fcurinode
= format(int(finode
), 'x').lower();
2752 fcurinode
= format(int(curinode
), 'x').lower();
2753 curinode
= curinode
+ 1;
2755 fcurinode
= format(int(curinode
), 'x').lower();
2756 curinode
= curinode
+ 1;
2757 curfid
= curfid
+ 1;
2759 flinkname
= os
.readlink(fname
);
2760 fdev
= fstatinfo
.st_dev
;
2761 getfdev
= GetDevMajorMinor(fdev
);
2762 fdev_minor
= getfdev
[0];
2763 fdev_major
= getfdev
[1];
2764 frdev
= fstatinfo
.st_dev
;
2765 if(hasattr(fstatinfo
, "st_rdev")):
2766 frdev
= fstatinfo
.st_rdev
;
2768 frdev
= fstatinfo
.st_dev
;
2769 getfrdev
= GetDevMajorMinor(frdev
);
2770 frdev_minor
= getfrdev
[0];
2771 frdev_major
= getfrdev
[1];
2772 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2773 fsize
= format(int("0"), 'x').lower();
2774 elif(ftype
==0 or ftype
==7):
2775 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2777 fsize
= format(int(fstatinfo
.st_size
)).lower();
2778 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2779 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2780 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2781 if(hasattr(fstatinfo
, "st_birthtime")):
2782 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2784 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2785 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2786 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2787 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2788 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2789 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2794 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2795 funame
= userinfo
.pw_name
;
2804 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2805 fgname
= groupinfo
.gr_name
;
2810 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2811 fdev_major
= format(int(fdev_major
), 'x').lower();
2812 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2813 frdev_major
= format(int(frdev_major
), 'x').lower();
2814 finode
= format(int(finode
), 'x').lower();
2815 flinkcount
= format(int(flinkcount
), 'x').lower();
2816 if(hasattr(fstatinfo
, "st_file_attributes")):
2817 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2819 fwinattributes
= format(int(0), 'x').lower();
2821 fcsize
= format(int(0), 'x').lower();
2822 fcontents
= BytesIO();
2823 if(ftype
==0 or ftype
==7):
2824 with
open(fname
, "rb") as fpc
:
2825 shutil
.copyfileobj(fpc
, fcontents
);
2826 if(not compresswholefile
):
2827 fcontents
.seek(0, 2);
2828 ucfsize
= fcontents
.tell();
2829 fcontents
.seek(0, 0);
2830 if(compression
=="auto"):
2831 ilsize
= len(compressionlistalt
);
2834 while(ilmin
< ilsize
):
2835 cfcontents
= BytesIO();
2836 shutil
.copyfileobj(fcontents
, cfcontents
);
2837 fcontents
.seek(0, 0);
2838 cfcontents
.seek(0, 0);
2839 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2841 cfcontents
.seek(0, 2);
2842 ilcsize
.append(cfcontents
.tell());
2846 ilcsize
.append(sys
.maxint
);
2847 except AttributeError:
2848 ilcsize
.append(sys
.maxsize
);
2850 ilcmin
= ilcsize
.index(min(ilcsize
));
2851 compression
= compressionlistalt
[ilcmin
];
2852 fcontents
.seek(0, 0);
2853 cfcontents
= BytesIO();
2854 shutil
.copyfileobj(fcontents
, cfcontents
);
2855 cfcontents
.seek(0, 0);
2856 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2857 cfcontents
.seek(0, 2);
2858 cfsize
= cfcontents
.tell();
2859 if(ucfsize
> cfsize
):
2860 fcsize
= format(int(cfsize
), 'x').lower();
2861 fcompression
= compression
;
2863 fcontents
= cfcontents
;
2864 if(fcompression
=="none"):
2866 if(followlink
and (ftype
==1 or ftype
==2)):
2867 flstatinfo
= os
.stat(flinkname
);
2868 with
open(flinkname
, "rb") as fpc
:
2869 shutil
.copyfileobj(fpc
, fcontents
);
2870 if(not compresswholefile
):
2871 fcontents
.seek(0, 2);
2872 ucfsize
= fcontents
.tell();
2873 fcontents
.seek(0, 0);
2874 if(compression
=="auto"):
2875 ilsize
= len(compressionlistalt
);
2878 while(ilmin
< ilsize
):
2879 cfcontents
= BytesIO();
2880 shutil
.copyfileobj(fcontents
, cfcontents
);
2881 fcontents
.seek(0, 0);
2882 cfcontents
.seek(0, 0);
2883 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2885 cfcontents
.seek(0, 2);
2886 ilcsize
.append(cfcontents
.tell());
2890 ilcsize
.append(sys
.maxint
);
2891 except AttributeError:
2892 ilcsize
.append(sys
.maxsize
);
2894 ilcmin
= ilcsize
.index(min(ilcsize
));
2895 compression
= compressionlistalt
[ilcmin
];
2896 fcontents
.seek(0, 0);
2897 cfcontents
= BytesIO();
2898 shutil
.copyfileobj(fcontents
, cfcontents
);
2899 cfcontents
.seek(0, 0);
2900 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2901 cfcontents
.seek(0, 2);
2902 cfsize
= cfcontents
.tell();
2903 if(ucfsize
> cfsize
):
2904 fcsize
= format(int(cfsize
), 'x').lower();
2905 fcompression
= compression
;
2907 fcontents
= cfcontents
;
2908 fcontents
.seek(0, 0);
2909 ftypehex
= format(ftype
, 'x').lower();
2910 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
2911 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
2914 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
2915 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2916 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
2919 os
.fsync(catfp
.fileno());
2920 except io
.UnsupportedOperation
:
2922 except AttributeError:
2924 except OSError as e
:
2928 if(hasattr(sys
.stdout
, "buffer")):
2929 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
2931 shutil
.copyfileobj(catfp
, sys
.stdout
);
2932 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2933 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
2935 upload_file_to_internet_file(catfp
, outfile
);
2943 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
2945 if(hasattr(shutil
, "register_archive_format")):
2946 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
2947 return PackArchiveFile(source_dir
, archive_name
, False, "auto", True, None, False, "crc32", [], __file_format_dict__
['format_delimiter'], False, False);
2948 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
2950 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2951 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
2953 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
2955 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2956 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2957 outfile
= RemoveWindowsPath(outfile
);
2958 checksumtype
= checksumtype
.lower();
2959 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2960 checksumtype
="crc32";
2961 if(checksumtype
=="none"):
2963 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2964 compression
= "auto";
2965 if(compression
not in compressionlist
and compression
is None):
2966 compression
= "auto";
2968 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2969 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2970 if(os
.path
.exists(outfile
)):
2973 except OSError as e
:
2978 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2980 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2983 fbasename
= os
.path
.splitext(outfile
)[0];
2984 fextname
= os
.path
.splitext(outfile
)[1];
2985 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2986 catver
= formatspecs
['format_ver'];
2987 fileheaderver
= str(int(catver
.replace(".", "")));
2993 inodetocatinode
= {};
2996 if(hasattr(sys
.stdin
, "buffer")):
2997 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
2999 shutil
.copyfileobj(sys
.stdin
, infile
);
3004 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3005 infile
= download_file_from_internet_file(infile
);
3010 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3012 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
3014 if(not tarfile
.is_tarfile(infile
)):
3016 except AttributeError:
3017 if(not is_tarfile(infile
)):
3022 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3023 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
3025 tarfp
= tarfile
.open(infile
, "r");
3026 except FileNotFoundError
:
3028 numfiles
= int(len(tarfp
.getmembers()));
3029 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3030 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
3031 if(re
.findall("^[.|/]", member
.name
)):
3032 fname
= member
.name
;
3034 fname
= "./"+member
.name
;
3036 VerbosePrintOut(fname
);
3037 fpremode
= member
.mode
;
3038 ffullmode
= member
.mode
;
3042 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3044 elif(member
.isdev()):
3045 ffullmode
= member
.mode
;
3047 elif(member
.islnk()):
3048 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3050 elif(member
.issym()):
3051 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
3053 elif(member
.ischr()):
3054 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
3056 elif(member
.isblk()):
3057 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
3059 elif(member
.isdir()):
3060 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
3062 elif(member
.isfifo()):
3063 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
3065 elif(member
.issparse()):
3066 ffullmode
= member
.mode
;
3069 ffullmode
= member
.mode
;
3072 fcurfid
= format(int(curfid
), 'x').lower();
3073 fcurinode
= format(int(curfid
), 'x').lower();
3074 curfid
= curfid
+ 1;
3076 flinkname
= member
.linkname
;
3077 fdev_minor
= format(int(member
.devminor
), 'x').lower();
3078 fdev_major
= format(int(member
.devmajor
), 'x').lower();
3079 frdev_minor
= format(int(member
.devminor
), 'x').lower();
3080 frdev_major
= format(int(member
.devmajor
), 'x').lower();
3081 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
3082 fsize
= format(int("0"), 'x').lower();
3083 elif(ftype
==0 or ftype
==7):
3084 fsize
= format(int(member
.size
), 'x').lower();
3086 fsize
= format(int(member
.size
), 'x').lower();
3087 fatime
= format(int(member
.mtime
), 'x').lower();
3088 fmtime
= format(int(member
.mtime
), 'x').lower();
3089 fctime
= format(int(member
.mtime
), 'x').lower();
3090 fbtime
= format(int(member
.mtime
), 'x').lower();
3091 fmode
= format(int(ffullmode
), 'x').lower();
3092 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
3093 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
3094 fuid
= format(int(member
.uid
), 'x').lower();
3095 fgid
= format(int(member
.gid
), 'x').lower();
3096 funame
= member
.uname
;
3097 fgname
= member
.gname
;
3098 flinkcount
= format(int(flinkcount
), 'x').lower();
3099 fwinattributes
= format(int(0), 'x').lower();
3101 fcsize
= format(int(0), 'x').lower();
3102 fcontents
= BytesIO();
3103 if(ftype
==0 or ftype
==7):
3104 with tarfp
.extractfile(member
) as fpc
:
3105 shutil
.copyfileobj(fpc
, fcontents
);
3106 if(not compresswholefile
):
3107 fcontents
.seek(0, 2);
3108 ucfsize
= fcontents
.tell();
3109 fcontents
.seek(0, 0);
3110 if(compression
=="auto"):
3111 ilsize
= len(compressionlistalt
);
3114 while(ilmin
< ilsize
):
3115 cfcontents
= BytesIO();
3116 shutil
.copyfileobj(fcontents
, cfcontents
);
3117 fcontents
.seek(0, 0);
3118 cfcontents
.seek(0, 0);
3119 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3121 cfcontents
.seek(0, 2);
3122 ilcsize
.append(cfcontents
.tell());
3126 ilcsize
.append(sys
.maxint
);
3127 except AttributeError:
3128 ilcsize
.append(sys
.maxsize
);
3130 ilcmin
= ilcsize
.index(min(ilcsize
));
3131 compression
= compressionlistalt
[ilcmin
];
3132 fcontents
.seek(0, 0);
3133 cfcontents
= BytesIO();
3134 shutil
.copyfileobj(fcontents
, cfcontents
);
3135 cfcontents
.seek(0, 0);
3136 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3137 cfcontents
.seek(0, 2);
3138 cfsize
= cfcontents
.tell();
3139 if(ucfsize
> cfsize
):
3140 fcsize
= format(int(cfsize
), 'x').lower();
3141 fcompression
= compression
;
3143 fcontents
= cfcontents
;
3144 if(fcompression
=="none"):
3146 fcontents
.seek(0, 0);
3147 ftypehex
= format(ftype
, 'x').lower();
3148 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3149 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3152 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3153 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3154 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3157 os
.fsync(catfp
.fileno());
3158 except io
.UnsupportedOperation
:
3160 except AttributeError:
3162 except OSError as e
:
3166 if(hasattr(sys
.stdout
, "buffer")):
3167 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3169 shutil
.copyfileobj(catfp
, sys
.stdout
);
3170 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3171 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3173 upload_file_to_internet_file(catfp
, outfile
);
3181 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
3183 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3184 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3185 outfile
= RemoveWindowsPath(outfile
);
3186 checksumtype
= checksumtype
.lower();
3187 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3188 checksumtype
="crc32";
3189 if(checksumtype
=="none"):
3191 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3192 compression
= "auto";
3193 if(compression
not in compressionlist
and compression
is None):
3194 compression
= "auto";
3196 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3197 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3198 if(os
.path
.exists(outfile
)):
3201 except OSError as e
:
3206 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3208 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3211 fbasename
= os
.path
.splitext(outfile
)[0];
3212 fextname
= os
.path
.splitext(outfile
)[1];
3213 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3214 catver
= formatspecs
['format_ver'];
3215 fileheaderver
= str(int(catver
.replace(".", "")));
3221 inodetocatinode
= {};
3224 if(hasattr(sys
.stdin
, "buffer")):
3225 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3227 shutil
.copyfileobj(sys
.stdin
, infile
);
3232 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3233 infile
= download_file_from_internet_file(infile
);
3238 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3242 if(not zipfile
.is_zipfile(infile
)):
3245 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
3246 except FileNotFoundError
:
3248 ziptest
= zipfp
.testzip();
3250 VerbosePrintOut("Bad file found!");
3251 numfiles
= int(len(zipfp
.infolist()));
3252 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3253 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
3254 if(re
.findall("^[.|/]", member
.filename
)):
3255 fname
= member
.filename
;
3257 fname
= "./"+member
.filename
;
3258 zipinfo
= zipfp
.getinfo(member
.filename
);
3260 VerbosePrintOut(fname
);
3261 if(not member
.is_dir()):
3262 fpremode
= int(stat
.S_IFREG
+ 438);
3263 elif(member
.is_dir()):
3264 fpremode
= int(stat
.S_IFDIR
+ 511);
3267 if(not member
.is_dir()):
3269 elif(member
.is_dir()):
3272 fcurfid
= format(int(curfid
), 'x').lower();
3273 fcurinode
= format(int(curfid
), 'x').lower();
3274 curfid
= curfid
+ 1;
3275 fdev_minor
= format(int(0), 'x').lower();
3276 fdev_major
= format(int(0), 'x').lower();
3277 frdev_minor
= format(int(0), 'x').lower();
3278 frdev_major
= format(int(0), 'x').lower();
3280 fsize
= format(int("0"), 'x').lower();
3282 fsize
= format(int(member
.file_size
), 'x').lower();
3284 fsize
= format(int(member
.file_size
), 'x').lower();
3285 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3286 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3287 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3288 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3289 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
3290 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
3291 if(not member
.is_dir()):
3292 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3293 fchmode
= stat
.S_IMODE(fmode
);
3294 ftypemod
= stat
.S_IFMT(fmode
);
3295 elif(member
.is_dir()):
3296 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3297 fchmode
= stat
.S_IMODE(fmode
);
3298 ftypemod
= stat
.S_IFMT(fmode
);
3299 elif(zipinfo
.create_system
==3):
3300 fwinattributes
= format(int(0), 'x').lower();
3301 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
3302 fchmode
= stat
.S_IMODE(fmode
);
3303 ftypemod
= stat
.S_IFMT(fmode
);
3305 fwinattributes
= format(int(0), 'x').lower();
3306 if(not member
.is_dir()):
3307 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3308 fchmode
= stat
.S_IMODE(fmode
);
3309 ftypemod
= stat
.S_IFMT(fmode
);
3310 elif(member
.is_dir()):
3311 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3312 fchmode
= stat
.S_IMODE(fmode
);
3313 ftypemod
= stat
.S_IFMT(fmode
);
3315 fcsize
= format(int(0), 'x').lower();
3317 fuid
= format(int(os
.getuid()), 'x').lower();
3318 except AttributeError:
3319 fuid
= format(int(0), 'x').lower();
3321 fuid
= format(int(0), 'x').lower();
3323 fgid
= format(int(os
.getgid()), 'x').lower();
3324 except AttributeError:
3325 fgid
= format(int(0), 'x').lower();
3327 fgid
= format(int(0), 'x').lower();
3331 userinfo
= pwd
.getpwuid(os
.getuid());
3332 funame
= userinfo
.pw_name
;
3335 except AttributeError:
3343 groupinfo
= grp
.getgrgid(os
.getgid());
3344 fgname
= groupinfo
.gr_name
;
3347 except AttributeError:
3351 fcontents
= BytesIO();
3353 fcontents
.write(zipfp
.read(member
.filename
));
3354 if(not compresswholefile
):
3355 fcontents
.seek(0, 2);
3356 ucfsize
= fcontents
.tell();
3357 fcontents
.seek(0, 0);
3358 if(compression
=="auto"):
3359 ilsize
= len(compressionlistalt
);
3362 while(ilmin
< ilsize
):
3363 cfcontents
= BytesIO();
3364 shutil
.copyfileobj(fcontents
, cfcontents
);
3365 fcontents
.seek(0, 0);
3366 cfcontents
.seek(0, 0);
3367 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3368 cfcontents
.seek(0, 2);
3369 ilcsize
.append(cfcontents
.tell());
3372 ilcmin
= ilcsize
.index(min(ilcsize
));
3373 compression
= compressionlistalt
[ilcmin
];
3374 fcontents
.seek(0, 0);
3375 cfcontents
= BytesIO();
3376 shutil
.copyfileobj(fcontents
, cfcontents
);
3377 cfcontents
.seek(0, 0);
3378 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3379 cfcontents
.seek(0, 2);
3380 cfsize
= cfcontents
.tell();
3381 if(ucfsize
> cfsize
):
3382 fcsize
= format(int(cfsize
), 'x').lower();
3383 fcompression
= compression
;
3385 fcontents
= cfcontents
;
3386 if(fcompression
=="none"):
3388 fcontents
.seek(0, 0);
3389 ftypehex
= format(ftype
, 'x').lower();
3390 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3391 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3394 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3395 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3396 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3399 os
.fsync(catfp
.fileno());
3400 except io
.UnsupportedOperation
:
3402 except AttributeError:
3404 except OSError as e
:
3408 if(hasattr(sys
.stdout
, "buffer")):
3409 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3411 shutil
.copyfileobj(catfp
, sys
.stdout
);
3412 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3413 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3415 upload_file_to_internet_file(catfp
, outfile
);
3423 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3425 if(not rarfile_support
):
3426 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3429 if(rarfile_support
):
3430 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3431 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3432 outfile
= RemoveWindowsPath(outfile
);
3433 checksumtype
= checksumtype
.lower();
3434 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3435 checksumtype
="crc32";
3436 if(checksumtype
=="none"):
3438 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3439 compression
= "auto";
3440 if(compression
not in compressionlist
and compression
is None):
3441 compression
= "auto";
3443 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3444 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3445 if(os
.path
.exists(outfile
)):
3448 except OSError as e
:
3453 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3455 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3458 fbasename
= os
.path
.splitext(outfile
)[0];
3459 fextname
= os
.path
.splitext(outfile
)[1];
3460 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3461 catver
= formatspecs
['format_ver'];
3462 fileheaderver
= str(int(catver
.replace(".", "")));
3468 inodetocatinode
= {};
3469 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3471 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3473 rarfp
= rarfile
.RarFile(infile
, "r");
3474 rartest
= rarfp
.testrar();
3476 VerbosePrintOut("Bad file found!");
3477 numfiles
= int(len(rarfp
.infolist()));
3478 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3481 os
.fsync(catfp
.fileno());
3482 except io
.UnsupportedOperation
:
3484 except AttributeError:
3486 except OSError as e
:
3488 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3491 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3494 member
.external_attr
3496 except AttributeError:
3498 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3501 member
.external_attr
3503 except AttributeError:
3508 if(re
.findall("^[.|/]", member
.filename
)):
3509 fname
= member
.filename
;
3511 fname
= "./"+member
.filename
;
3512 rarinfo
= rarfp
.getinfo(member
.filename
);
3514 VerbosePrintOut(fname
);
3515 if(is_unix
and member
.external_attr
!=0):
3516 fpremode
= int(member
.external_attr
);
3517 elif(member
.is_file()):
3518 fpremode
= int(stat
.S_IFREG
+ 438);
3519 elif(member
.is_symlink()):
3520 fpremode
= int(stat
.S_IFLNK
+ 438);
3521 elif(member
.is_dir()):
3522 fpremode
= int(stat
.S_IFDIR
+ 511);
3523 if(is_windows
and member
.external_attr
!=0):
3524 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3526 fwinattributes
= format(int(0), 'x').lower();
3528 fcsize
= format(int(0), 'x').lower();
3531 if(member
.is_file()):
3533 elif(member
.is_symlink()):
3535 elif(member
.is_dir()):
3539 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3540 fcurfid
= format(int(curfid
), 'x').lower();
3541 fcurinode
= format(int(curfid
), 'x').lower();
3542 curfid
= curfid
+ 1;
3543 fdev_minor
= format(int(0), 'x').lower();
3544 fdev_major
= format(int(0), 'x').lower();
3545 frdev_minor
= format(int(0), 'x').lower();
3546 frdev_major
= format(int(0), 'x').lower();
3548 fsize
= format(int("0"), 'x').lower();
3550 fsize
= format(int(member
.file_size
), 'x').lower();
3552 fsize
= format(int(member
.file_size
), 'x').lower();
3555 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3557 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3558 except AttributeError:
3559 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3560 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3563 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3565 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3566 except AttributeError:
3567 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3568 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3569 if(is_unix
and member
.external_attr
!=0):
3570 fmode
= format(int(member
.external_attr
), 'x').lower();
3571 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3572 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3573 elif(member
.is_file()):
3574 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3575 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3576 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3577 elif(member
.is_symlink()):
3578 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3579 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3580 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3581 elif(member
.is_dir()):
3582 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3583 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3584 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3586 fuid
= format(int(os
.getuid()), 'x').lower();
3587 except AttributeError:
3588 fuid
= format(int(0), 'x').lower();
3590 fuid
= format(int(0), 'x').lower();
3592 fgid
= format(int(os
.getgid()), 'x').lower();
3593 except AttributeError:
3594 fgid
= format(int(0), 'x').lower();
3596 fgid
= format(int(0), 'x').lower();
3600 userinfo
= pwd
.getpwuid(os
.getuid());
3601 funame
= userinfo
.pw_name
;
3604 except AttributeError:
3612 groupinfo
= grp
.getgrgid(os
.getgid());
3613 fgname
= groupinfo
.gr_name
;
3616 except AttributeError:
3620 fcontents
= BytesIO();
3622 fcontents
.write(rarfp
.read(member
.filename
));
3623 if(not compresswholefile
):
3624 fcontents
.seek(0, 2);
3625 ucfsize
= fcontents
.tell();
3626 fcontents
.seek(0, 0);
3627 if(compression
=="auto"):
3628 ilsize
= len(compressionlistalt
);
3631 while(ilmin
< ilsize
):
3632 cfcontents
= BytesIO();
3633 shutil
.copyfileobj(fcontents
, cfcontents
);
3634 fcontents
.seek(0, 0);
3635 cfcontents
.seek(0, 0);
3636 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3638 cfcontents
.seek(0, 2);
3639 ilcsize
.append(cfcontents
.tell());
3643 ilcsize
.append(sys
.maxint
);
3644 except AttributeError:
3645 ilcsize
.append(sys
.maxsize
);
3647 ilcmin
= ilcsize
.index(min(ilcsize
));
3648 compression
= compressionlistalt
[ilcmin
];
3649 fcontents
.seek(0, 0);
3650 cfcontents
= BytesIO();
3651 shutil
.copyfileobj(fcontents
, cfcontents
);
3652 cfcontents
.seek(0, 0);
3653 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3654 cfcontents
.seek(0, 2);
3655 cfsize
= cfcontents
.tell();
3656 if(ucfsize
> cfsize
):
3657 fcsize
= format(int(cfsize
), 'x').lower();
3658 fcompression
= compression
;
3660 fcontents
= cfcontents
;
3661 if(fcompression
=="none"):
3663 fcontents
.seek(0, 0);
3664 ftypehex
= format(ftype
, 'x').lower();
3665 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3666 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3669 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3670 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3671 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3674 os
.fsync(catfp
.fileno());
3675 except io
.UnsupportedOperation
:
3677 except AttributeError:
3679 except OSError as e
:
3683 if(hasattr(sys
.stdout
, "buffer")):
3684 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3686 shutil
.copyfileobj(catfp
, sys
.stdout
);
3687 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3688 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3690 upload_file_to_internet_file(catfp
, outfile
);
3698 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3700 if(not py7zr_support
):
3701 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3705 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3706 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3707 outfile
= RemoveWindowsPath(outfile
);
3708 checksumtype
= checksumtype
.lower();
3709 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3710 checksumtype
="crc32";
3711 if(checksumtype
=="none"):
3713 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3714 compression
= "auto";
3715 if(compression
not in compressionlist
and compression
is None):
3716 compression
= "auto";
3718 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3719 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3720 if(os
.path
.exists(outfile
)):
3723 except OSError as e
:
3728 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3730 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3733 fbasename
= os
.path
.splitext(outfile
)[0];
3734 fextname
= os
.path
.splitext(outfile
)[1];
3735 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3736 catver
= formatspecs
['format_ver'];
3737 fileheaderver
= str(int(catver
.replace(".", "")));
3743 inodetocatinode
= {};
3744 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3746 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3747 file_content
= szpfp
.readall();
3748 #sztest = szpfp.testzip();
3749 sztestalt
= szpfp
.test();
3751 VerbosePrintOut("Bad file found!");
3752 numfiles
= int(len(szpfp
.list()));
3753 AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3754 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3755 if(re
.findall("^[.|/]", member
.filename
)):
3756 fname
= member
.filename
;
3758 fname
= "./"+member
.filename
;
3760 VerbosePrintOut(fname
);
3761 if(not member
.is_directory
):
3762 fpremode
= int(stat
.S_IFREG
+ 438);
3763 elif(member
.is_directory
):
3764 fpremode
= int(stat
.S_IFDIR
+ 511);
3765 fwinattributes
= format(int(0), 'x').lower();
3767 fcsize
= format(int(0), 'x').lower();
3770 if(member
.is_directory
):
3775 fcurfid
= format(int(curfid
), 'x').lower();
3776 fcurinode
= format(int(curfid
), 'x').lower();
3777 curfid
= curfid
+ 1;
3778 fdev_minor
= format(int(0), 'x').lower();
3779 fdev_major
= format(int(0), 'x').lower();
3780 frdev_minor
= format(int(0), 'x').lower();
3781 frdev_major
= format(int(0), 'x').lower();
3783 fsize
= format(int("0"), 'x').lower();
3784 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3785 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3786 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3787 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3788 if(member
.is_directory
):
3789 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3790 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3791 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3793 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3794 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3795 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3797 fuid
= format(int(os
.getuid()), 'x').lower();
3798 except AttributeError:
3799 fuid
= format(int(0), 'x').lower();
3801 fuid
= format(int(0), 'x').lower();
3803 fgid
= format(int(os
.getgid()), 'x').lower();
3804 except AttributeError:
3805 fgid
= format(int(0), 'x').lower();
3807 fgid
= format(int(0), 'x').lower();
3811 userinfo
= pwd
.getpwuid(os
.getuid());
3812 funame
= userinfo
.pw_name
;
3815 except AttributeError:
3823 groupinfo
= grp
.getgrgid(os
.getgid());
3824 fgname
= groupinfo
.gr_name
;
3827 except AttributeError:
3831 fcontents
= BytesIO();
3833 fcontents
.write(file_content
[member
.filename
].read());
3834 fsize
= format(fcontents
.tell(), 'x').lower();
3835 file_content
[member
.filename
].close();
3836 if(not compresswholefile
):
3837 fcontents
.seek(0, 2);
3838 ucfsize
= fcontents
.tell();
3839 fcontents
.seek(0, 0);
3840 if(compression
=="auto"):
3841 ilsize
= len(compressionlistalt
);
3844 while(ilmin
< ilsize
):
3845 cfcontents
= BytesIO();
3846 shutil
.copyfileobj(fcontents
, cfcontents
);
3847 fcontents
.seek(0, 0);
3848 cfcontents
.seek(0, 0);
3849 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3851 cfcontents
.seek(0, 2);
3852 ilcsize
.append(cfcontents
.tell());
3856 ilcsize
.append(sys
.maxint
);
3857 except AttributeError:
3858 ilcsize
.append(sys
.maxsize
);
3860 ilcmin
= ilcsize
.index(min(ilcsize
));
3861 compression
= compressionlistalt
[ilcmin
];
3862 fcontents
.seek(0, 0);
3863 cfcontents
= BytesIO();
3864 shutil
.copyfileobj(fcontents
, cfcontents
);
3865 cfcontents
.seek(0, 0);
3866 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3867 cfcontents
.seek(0, 2);
3868 cfsize
= cfcontents
.tell();
3869 if(ucfsize
> cfsize
):
3870 fcsize
= format(int(cfsize
), 'x').lower();
3871 fcompression
= compression
;
3873 fcontents
= cfcontents
;
3874 if(fcompression
=="none"):
3876 fcontents
.seek(0, 0);
3877 ftypehex
= format(ftype
, 'x').lower();
3878 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3879 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3882 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3883 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3884 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3887 os
.fsync(catfp
.fileno());
3888 except io
.UnsupportedOperation
:
3890 except AttributeError:
3892 except OSError as e
:
3896 if(hasattr(sys
.stdout
, "buffer")):
3897 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3899 shutil
.copyfileobj(catfp
, sys
.stdout
);
3900 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3901 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3903 upload_file_to_internet_file(catfp
, outfile
);
3911 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
3913 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3914 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3916 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3917 if(checkcompressfile
=="tarfile"):
3918 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3919 elif(checkcompressfile
=="zipfile"):
3920 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3921 elif(checkcompressfile
=="catfile"):
3922 return RePackArchiveFile(infile
, outfile
, compression
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
3923 elif(rarfile_support
and checkcompressfile
=="rarfile"):
3924 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3925 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
3926 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3931 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
3933 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
3934 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3937 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3938 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
3939 if(checkcompressfile
=="tarfile"):
3940 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3941 if(checkcompressfile
=="zipfile"):
3942 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3943 if(rarfile_support
and checkcompressfile
=="rarfile"):
3944 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3945 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3946 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3947 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
3954 if(hasattr(sys
.stdin
, "buffer")):
3955 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
3957 shutil
.copyfileobj(sys
.stdin
, catfp
);
3959 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3963 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3964 catfp
= download_file_from_internet_file(infile
);
3966 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3971 infile
= RemoveWindowsPath(infile
);
3972 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3973 if(checkcompressfile
=="tarfile"):
3974 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3975 if(checkcompressfile
=="zipfile"):
3976 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3977 if(rarfile_support
and checkcompressfile
=="rarfile"):
3978 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3979 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3980 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3981 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
3983 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
3984 if(not compresscheck
):
3985 fextname
= os
.path
.splitext(infile
)[1];
3986 if(fextname
==".gz"):
3987 compresscheck
= "gzip";
3988 elif(fextname
==".bz2"):
3989 compresscheck
= "bzip2";
3990 elif(fextname
==".zst"):
3991 compresscheck
= "zstd";
3992 elif(fextname
==".lz4" or fextname
==".clz4"):
3993 compresscheck
= "lz4";
3994 elif(fextname
==".lzo" or fextname
==".lzop"):
3995 compresscheck
= "lzo";
3996 elif(fextname
==".lzma" or fextname
==".xz"):
3997 compresscheck
= "lzma";
4000 if(not compresscheck
):
4002 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4007 SeekToEndOfFile(catfp);
4009 SeekToEndOfFile(catfp);
4010 CatSize = catfp.tell();
4011 CatSizeEnd = CatSize;
4019 curloc
= catfp
.tell();
4022 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4024 catfp
.seek(curloc
, 0);
4025 catstring
= catheader
[0];
4026 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4027 fprenumfiles
= catheader
[1];
4028 fnumfiles
= int(fprenumfiles
, 16);
4029 fprechecksumtype
= catheader
[2];
4030 fprechecksum
= catheader
[3];
4031 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4032 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4033 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4034 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4035 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4036 fheadtell
= len(fileheader
);
4037 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4038 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4039 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4041 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4042 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4043 if(seekto
>=fnumfiles
):
4044 seekto
= fnumfiles
- 1;
4050 prefhstart
= catfp
.tell();
4051 if(formatspecs
['new_style']):
4052 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4054 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4055 if(len(preheaderdata
)==0):
4057 prefheadsize
= int(preheaderdata
[0], 16);
4058 prefnumfields
= int(preheaderdata
[1], 16);
4059 preftype
= int(preheaderdata
[2], 16);
4060 if(re
.findall("^[.|/]", preheaderdata
[3])):
4061 prefname
= preheaderdata
[3];
4063 prefname
= "./"+preheaderdata
[3];
4064 prefbasedir
= os
.path
.dirname(prefname
);
4065 preflinkname
= preheaderdata
[4];
4066 prefsize
= int(preheaderdata
[5], 16);
4067 prefatime
= int(preheaderdata
[6], 16);
4068 prefmtime
= int(preheaderdata
[7], 16);
4069 prefctime
= int(preheaderdata
[8], 16);
4070 prefbtime
= int(preheaderdata
[9], 16);
4071 prefmode
= int(preheaderdata
[10], 16);
4072 prefchmode
= stat
.S_IMODE(prefmode
);
4073 preftypemod
= stat
.S_IFMT(prefmode
);
4074 prefwinattributes
= int(preheaderdata
[11], 16);
4075 prefcompression
= preheaderdata
[12];
4076 prefcsize
= int(preheaderdata
[13], 16);
4077 prefuid
= int(preheaderdata
[14], 16);
4078 prefuname
= preheaderdata
[15];
4079 prefgid
= int(preheaderdata
[16], 16);
4080 prefgname
= preheaderdata
[17];
4081 fid
= int(preheaderdata
[18], 16);
4082 finode
= int(preheaderdata
[19], 16);
4083 flinkcount
= int(preheaderdata
[20], 16);
4084 prefdev_minor
= int(preheaderdata
[21], 16);
4085 prefdev_major
= int(preheaderdata
[22], 16);
4086 prefrdev_minor
= int(preheaderdata
[23], 16);
4087 prefrdev_major
= int(preheaderdata
[24], 16);
4088 prefextrasize
= int(preheaderdata
[25], 16);
4089 prefextrafields
= int(preheaderdata
[26], 16);
4090 extrafieldslist
= [];
4092 extraend
= extrastart
+ prefextrafields
;
4093 extrafieldslist
= [];
4094 if(extrastart
<extraend
):
4095 extrafieldslist
.append(preheaderdata
[extrastart
]);
4096 extrastart
= extrastart
+ 1;
4097 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4098 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4099 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4101 hcmax
= len(preheaderdata
) - 2;
4104 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4106 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4107 if(prefcs
!=prenewfcs
and not skipchecksum
):
4108 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4109 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4111 valid_archive
= False;
4112 invalid_archive
= True;
4113 prefhend
= catfp
.tell() - 1;
4114 prefcontentstart
= catfp
.tell();
4116 pyhascontents
= False;
4118 if(prefcompression
):
4119 prefcontents
= catfp
.read(prefsize
);
4121 prefcontents
= catfp
.read(prefcsize
);
4122 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4123 pyhascontents
= True;
4124 if(prefccs
!=prenewfccs
and not skipchecksum
):
4125 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4126 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4130 catfp
.seek(seekstart
, 0);
4132 catfheadsize
= int(preheaderdata
[0], 16);
4133 catfnumfields
= int(preheaderdata
[1], 16);
4134 catftype
= int(preheaderdata
[2], 16);
4135 if(re
.findall("^[.|/]", preheaderdata
[3])):
4136 catfname
= preheaderdata
[3];
4138 catfname
= "./"+preheaderdata
[3];
4139 catflinkname
= preheaderdata
[4];
4140 catfsize
= int(preheaderdata
[5], 16);
4141 catfbasedir
= os
.path
.dirname(catfname
);
4142 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4144 catlist
.update({'catfp': catfp
});
4149 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
4151 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4152 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4155 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4156 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4157 if(checkcompressfile
=="tarfile"):
4158 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4159 if(checkcompressfile
=="zipfile"):
4160 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4161 if(rarfile_support
and checkcompressfile
=="rarfile"):
4162 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4163 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4164 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4165 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4172 if(hasattr(sys
.stdin
, "buffer")):
4173 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4175 shutil
.copyfileobj(sys
.stdin
, catfp
);
4177 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4181 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4182 catfp
= download_file_from_internet_file(infile
);
4183 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4189 infile
= RemoveWindowsPath(infile
);
4190 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4191 if(checkcompressfile
=="tarfile"):
4192 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4193 if(checkcompressfile
=="zipfile"):
4194 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4195 if(rarfile_support
and checkcompressfile
=="rarfile"):
4196 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4197 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4198 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4199 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4201 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4202 if(not compresscheck
):
4203 fextname
= os
.path
.splitext(infile
)[1];
4204 if(fextname
==".gz"):
4205 compresscheck
= "gzip";
4206 elif(fextname
==".bz2"):
4207 compresscheck
= "bzip2";
4208 elif(fextname
==".zst"):
4209 compresscheck
= "zstd";
4210 elif(fextname
==".lz4" or fextname
==".clz4"):
4211 compresscheck
= "lz4";
4212 elif(fextname
==".lzo" or fextname
==".lzop"):
4213 compresscheck
= "lzo";
4214 elif(fextname
==".lzma" or fextname
==".xz"):
4215 compresscheck
= "lzma";
4218 if(not compresscheck
):
4220 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4225 SeekToEndOfFile(catfp);
4227 SeekToEndOfFile(catfp);
4228 CatSize = catfp.tell();
4229 CatSizeEnd = CatSize;
4237 curloc
= catfp
.tell();
4240 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4242 catfp
.seek(curloc
, 0);
4243 catstring
= catheader
[0];
4244 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4245 fprenumfiles
= catheader
[1];
4246 fnumfiles
= int(fprenumfiles
, 16);
4247 fprechecksumtype
= catheader
[2];
4248 fprechecksum
= catheader
[3];
4249 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4250 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4251 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4252 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4253 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4254 fheadtell
= len(fileheader
);
4255 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4256 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4257 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4259 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4260 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4261 seekto
= fnumfiles
- 1
4266 prefhstart
= catfp
.tell();
4267 if(formatspecs
['new_style']):
4268 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4270 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4271 if(len(preheaderdata
)==0):
4273 prefheadsize
= int(preheaderdata
[0], 16);
4274 prefnumfields
= int(preheaderdata
[1], 16);
4275 preftype
= int(preheaderdata
[2], 16);
4276 if(re
.findall("^[.|/]", preheaderdata
[3])):
4277 prefname
= preheaderdata
[3];
4279 prefname
= "./"+preheaderdata
[3];
4280 prefbasedir
= os
.path
.dirname(prefname
);
4281 preflinkname
= preheaderdata
[4];
4282 prefsize
= int(preheaderdata
[5], 16);
4283 prefatime
= int(preheaderdata
[6], 16);
4284 prefmtime
= int(preheaderdata
[7], 16);
4285 prefctime
= int(preheaderdata
[8], 16);
4286 prefbtime
= int(preheaderdata
[9], 16);
4287 prefmode
= int(preheaderdata
[10], 16);
4288 prefchmode
= stat
.S_IMODE(prefmode
);
4289 preftypemod
= stat
.S_IFMT(prefmode
);
4290 prefwinattributes
= int(preheaderdata
[11], 16);
4291 prefcompression
= preheaderdata
[12];
4292 prefcsize
= int(preheaderdata
[13], 16);
4293 prefuid
= int(preheaderdata
[14], 16);
4294 prefuname
= preheaderdata
[15];
4295 prefgid
= int(preheaderdata
[16], 16);
4296 prefgname
= preheaderdata
[17];
4297 fid
= int(preheaderdata
[18], 16);
4298 finode
= int(preheaderdata
[19], 16);
4299 flinkcount
= int(preheaderdata
[20], 16);
4300 prefdev_minor
= int(preheaderdata
[21], 16);
4301 prefdev_major
= int(preheaderdata
[22], 16);
4302 prefrdev_minor
= int(preheaderdata
[23], 16);
4303 prefrdev_major
= int(preheaderdata
[24], 16);
4304 prefextrasize
= int(preheaderdata
[25], 16);
4305 prefextrafields
= int(preheaderdata
[26], 16);
4306 extrafieldslist
= [];
4308 extraend
= extrastart
+ prefextrafields
;
4309 extrafieldslist
= [];
4310 if(extrastart
<extraend
):
4311 extrafieldslist
.append(preheaderdata
[extrastart
]);
4312 extrastart
= extrastart
+ 1;
4313 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4314 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4315 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4317 hcmax
= len(preheaderdata
) - 2;
4320 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4322 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4323 if(prefcs
!=prenewfcs
and not skipchecksum
):
4324 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4325 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4327 valid_archive
= False;
4328 invalid_archive
= True;
4329 prefhend
= catfp
.tell() - 1;
4330 prefcontentstart
= catfp
.tell();
4332 pyhascontents
= False;
4334 if(prefcompression
):
4335 prefcontents
= catfp
.read(prefsize
);
4337 prefcontents
= catfp
.read(prefcsize
);
4338 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4339 pyhascontents
= True;
4340 if(prefccs
!=prenewfccs
and not skipchecksum
):
4341 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4342 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4347 prefname
= preheaderdata
[2];
4348 if(re
.findall("^[.|/]", preheaderdata
[2])):
4349 prefname
= preheaderdata
[2];
4351 prefname
= "./"+preheaderdata
[2];
4352 if(prefname
==seekfile
):
4355 catfp
.seek(seekstart
, 0);
4357 catfheadsize
= int(preheaderdata
[0], 16);
4358 catfnumfields
= int(preheaderdata
[1], 16);
4359 catftype
= int(preheaderdata
[2], 16);
4360 if(re
.findall("^[.|/]", preheaderdata
[3])):
4361 catfname
= preheaderdata
[3];
4363 catfname
= "./"+preheaderdata
[3];
4364 catflinkname
= preheaderdata
[4];
4365 catfsize
= int(preheaderdata
[5], 16);
4366 catfbasedir
= os
.path
.dirname(catfname
);
4368 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4372 catlist
.update({'catfp': catfp
});
4377 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4379 def ArchiveFileValidate(infile
, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4381 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4382 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4385 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4386 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4387 if(checkcompressfile
=="tarfile"):
4388 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4389 if(checkcompressfile
=="zipfile"):
4390 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4391 if(rarfile_support
and checkcompressfile
=="rarfile"):
4392 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4393 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4394 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4395 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4402 if(hasattr(sys
.stdin
, "buffer")):
4403 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4405 shutil
.copyfileobj(sys
.stdin
, catfp
);
4407 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4411 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4412 catfp
= download_file_from_internet_file(infile
);
4413 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4419 infile
= RemoveWindowsPath(infile
);
4420 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4421 if(checkcompressfile
=="tarfile"):
4422 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4423 if(checkcompressfile
=="zipfile"):
4424 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4425 if(rarfile_support
and checkcompressfile
=="rarfile"):
4426 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4427 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4428 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4429 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4431 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4432 if(not compresscheck
):
4433 fextname
= os
.path
.splitext(infile
)[1];
4434 if(fextname
==".gz"):
4435 compresscheck
= "gzip";
4436 elif(fextname
==".bz2"):
4437 compresscheck
= "bzip2";
4438 elif(fextname
==".zst"):
4439 compresscheck
= "zstd";
4440 elif(fextname
==".lz4" or fextname
==".clz4"):
4441 compresscheck
= "lz4";
4442 elif(fextname
==".lzo" or fextname
==".lzop"):
4443 compresscheck
= "lzo";
4444 elif(fextname
==".lzma" or fextname
==".xz"):
4445 compresscheck
= "lzma";
4448 if(not compresscheck
):
4450 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4455 SeekToEndOfFile(catfp);
4457 SeekToEndOfFile(catfp);
4458 CatSize = catfp.tell();
4459 CatSizeEnd = CatSize;
4467 curloc
= catfp
.tell();
4470 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4472 catfp
.seek(curloc
, 0);
4473 catstring
= catheader
[0];
4474 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4475 fprenumfiles
= catheader
[1];
4476 fnumfiles
= int(fprenumfiles
, 16);
4477 fprechecksumtype
= catheader
[2];
4478 fprechecksum
= catheader
[3];
4480 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4481 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4482 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4483 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4484 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4485 valid_archive
= True;
4486 invalid_archive
= False;
4488 VerbosePrintOut(infile
);
4489 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4490 if(fprechecksum
==catfileheadercshex
):
4492 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4493 VerbosePrintOut("'" + str(fprechecksum
) + "' == " + "'" + str(catfileheadercshex
) + "'");
4496 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4497 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4498 valid_archive
= False;
4499 invalid_archive
= True;
4501 VerbosePrintOut("");
4502 while(il
<fnumfiles
):
4503 catfhstart
= catfp
.tell();
4504 if(formatspecs
['new_style']):
4505 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4507 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4508 if(len(catheaderdata
)==0):
4510 catfheadsize
= int(catheaderdata
[0], 16);
4511 catfnumfields
= int(catheaderdata
[1], 16);
4512 catftype
= int(catheaderdata
[2], 16);
4513 if(re
.findall("^[.|/]", catheaderdata
[3])):
4514 catfname
= catheaderdata
[3];
4516 catfname
= "./"+catheaderdata
[3];
4517 catfbasedir
= os
.path
.dirname(catfname
);
4518 catflinkname
= catheaderdata
[4];
4519 catfsize
= int(catheaderdata
[5], 16);
4520 catfatime
= int(catheaderdata
[6], 16);
4521 catfmtime
= int(catheaderdata
[7], 16);
4522 catfctime
= int(catheaderdata
[8], 16);
4523 catfbtime
= int(catheaderdata
[9], 16);
4524 catfmode
= int(catheaderdata
[10], 16);
4525 catfchmode
= stat
.S_IMODE(catfmode
);
4526 catftypemod
= stat
.S_IFMT(catfmode
);
4527 catfwinattributes
= int(catheaderdata
[11], 16);
4528 catfcompression
= catheaderdata
[12];
4529 catfcsize
= int(catheaderdata
[13], 16);
4530 catfuid
= int(catheaderdata
[14], 16);
4531 catfuname
= catheaderdata
[15];
4532 catfgid
= int(catheaderdata
[16], 16);
4533 catfgname
= catheaderdata
[17];
4534 fid
= int(catheaderdata
[18], 16);
4535 finode
= int(catheaderdata
[19], 16);
4536 flinkcount
= int(catheaderdata
[20], 16);
4537 catfdev_minor
= int(catheaderdata
[21], 16);
4538 catfdev_major
= int(catheaderdata
[22], 16);
4539 catfrdev_minor
= int(catheaderdata
[23], 16);
4540 catfrdev_major
= int(catheaderdata
[24], 16);
4541 catfextrasize
= int(catheaderdata
[25], 16);
4542 catfextrafields
= int(catheaderdata
[26], 16);
4543 extrafieldslist
= [];
4545 extraend
= extrastart
+ catfextrafields
;
4546 extrafieldslist
= [];
4547 if(extrastart
<extraend
):
4548 extrafieldslist
.append(catheaderdata
[extrastart
]);
4549 extrastart
= extrastart
+ 1;
4550 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4551 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4552 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4554 hcmax
= len(catheaderdata
) - 2;
4557 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4559 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4561 VerbosePrintOut(catfname
);
4562 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4563 if(catfcs
==catnewfcs
):
4565 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4566 VerbosePrintOut("'" + str(catfcs
) + "' == " + "'" + str(catnewfcs
) + "'");
4569 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4570 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4571 valid_archive
= False;
4572 invalid_archive
= True;
4573 catfhend
= catfp
.tell() - 1;
4574 catfcontentstart
= catfp
.tell();
4576 pyhascontents
= False;
4578 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4579 catfcontents
= catfp
.read(catfsize
);
4581 catfcontents
= catfp
.read(catfcsize
);
4582 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4583 pyhascontents
= True;
4584 if(catfccs
==catnewfccs
):
4586 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4587 VerbosePrintOut("'" + str(catfccs
) + "' == " + "'" + str(catnewfccs
) + "'");
4590 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4591 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4592 valid_archive
= False;
4593 invalid_archive
= True;
4595 VerbosePrintOut("");
4608 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4610 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4611 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4614 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4615 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4616 if(checkcompressfile
=="tarfile"):
4617 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4618 if(checkcompressfile
=="zipfile"):
4619 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4620 if(rarfile_support
and checkcompressfile
=="rarfile"):
4621 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4622 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4623 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4624 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4631 if(hasattr(sys
.stdin
, "buffer")):
4632 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4634 shutil
.copyfileobj(sys
.stdin
, catfp
);
4636 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4640 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4641 catfp
= download_file_from_internet_file(infile
);
4642 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4648 infile
= RemoveWindowsPath(infile
);
4649 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4650 if(checkcompressfile
=="tarfile"):
4651 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4652 if(checkcompressfile
=="zipfile"):
4653 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4654 if(rarfile_support
and checkcompressfile
=="rarfile"):
4655 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4656 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4657 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4658 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4660 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4661 if(not compresscheck
):
4662 fextname
= os
.path
.splitext(infile
)[1];
4663 if(fextname
==".gz"):
4664 compresscheck
= "gzip";
4665 elif(fextname
==".bz2"):
4666 compresscheck
= "bzip2";
4667 elif(fextname
==".zst"):
4668 compresscheck
= "zstd";
4669 elif(fextname
==".lz4" or fextname
==".clz4"):
4670 compresscheck
= "lz4";
4671 elif(fextname
==".lzo" or fextname
==".lzop"):
4672 compresscheck
= "lzo";
4673 elif(fextname
==".lzma" or fextname
==".xz"):
4674 compresscheck
= "lzma";
4677 if(not compresscheck
):
4679 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4684 SeekToEndOfFile(catfp);
4686 SeekToEndOfFile(catfp);
4687 CatSize = catfp.tell();
4688 CatSizeEnd = CatSize;
4696 curloc
= catfp
.tell();
4699 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4701 catfp
.seek(curloc
, 0);
4702 catstring
= catheader
[0];
4703 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4704 fprenumfiles
= catheader
[1];
4705 fnumfiles
= int(fprenumfiles
, 16);
4706 fprechecksumtype
= catheader
[2];
4707 fprechecksum
= catheader
[3];
4708 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4709 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4710 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4711 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4712 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4713 fheadtell
= len(fileheader
);
4714 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4715 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4716 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4718 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4719 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4720 if(seekstart
<0 and seekstart
>fnumfiles
):
4722 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4723 seekend
= fnumfiles
;
4724 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4725 seekend
= fnumfiles
- abs(seekend
);
4728 while(il
< seekstart
):
4729 prefhstart
= catfp
.tell();
4730 if(formatspecs
['new_style']):
4731 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4733 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4734 if(len(preheaderdata
)==0):
4736 prefheadsize
= int(preheaderdata
[0], 16);
4737 prefnumfields
= int(preheaderdata
[1], 16);
4738 if(re
.findall("^[.|/]", preheaderdata
[3])):
4739 prefname
= preheaderdata
[3];
4741 prefname
= "./"+preheaderdata
[3];
4742 prefsize
= int(preheaderdata
[5], 16);
4743 prefcompression
= preheaderdata
[12];
4744 prefcsize
= int(preheaderdata
[13], 16);
4745 prefextrasize
= int(preheaderdata
[24], 16);
4746 prefextrafields
= int(preheaderdata
[25], 16);
4747 extrafieldslist
= [];
4749 extraend
= extrastart
+ prefextrafields
;
4750 extrafieldslist
= [];
4751 if(extrastart
<extraend
):
4752 extrafieldslist
.append(preheaderdata
[extrastart
]);
4753 extrastart
= extrastart
+ 1;
4754 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4755 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4756 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4758 hcmax
= len(preheaderdata
) - 2;
4761 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4763 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4764 if(prefcs
!=prenewfcs
and not skipchecksum
):
4765 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4766 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4768 valid_archive
= False;
4769 invalid_archive
= True;
4770 prefhend
= catfp
.tell() - 1;
4771 prefcontentstart
= catfp
.tell();
4773 pyhascontents
= False;
4775 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
4776 prefcontents
= catfp
.read(prefsize
);
4778 prefcontents
= catfp
.read(prefcsize
);
4779 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4780 pyhascontents
= True;
4781 if(prefccs
!=prenewfccs
and not skipchecksum
):
4782 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4783 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4787 fileidnum
= seekstart
;
4789 while(fileidnum
<seekend
):
4790 catfhstart
= catfp
.tell();
4791 if(formatspecs
['new_style']):
4792 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4794 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4795 if(len(catheaderdata
)==0):
4797 catfheadsize
= int(catheaderdata
[0], 16);
4798 catfnumfields
= int(catheaderdata
[1], 16);
4799 catftype
= int(catheaderdata
[2], 16);
4800 if(re
.findall("^[.|/]", catheaderdata
[3])):
4801 catfname
= catheaderdata
[3];
4803 catfname
= "./"+catheaderdata
[3];
4804 catfbasedir
= os
.path
.dirname(catfname
);
4805 catflinkname
= catheaderdata
[4];
4806 catfsize
= int(catheaderdata
[5], 16);
4807 catfatime
= int(catheaderdata
[6], 16);
4808 catfmtime
= int(catheaderdata
[7], 16);
4809 catfctime
= int(catheaderdata
[8], 16);
4810 catfbtime
= int(catheaderdata
[9], 16);
4811 catfmode
= int(catheaderdata
[10], 16);
4812 catfchmode
= stat
.S_IMODE(catfmode
);
4813 catftypemod
= stat
.S_IFMT(catfmode
);
4814 catfwinattributes
= int(catheaderdata
[11], 16);
4815 catfcompression
= catheaderdata
[12];
4816 catfcsize
= int(catheaderdata
[13], 16);
4817 catfuid
= int(catheaderdata
[14], 16);
4818 catfuname
= catheaderdata
[15];
4819 catfgid
= int(catheaderdata
[16], 16);
4820 catfgname
= catheaderdata
[17];
4821 catfid
= int(catheaderdata
[18], 16);
4822 catfinode
= int(catheaderdata
[19], 16);
4823 catflinkcount
= int(catheaderdata
[20], 16);
4824 catfdev_minor
= int(catheaderdata
[21], 16);
4825 catfdev_major
= int(catheaderdata
[22], 16);
4826 catfrdev_minor
= int(catheaderdata
[23], 16);
4827 catfrdev_major
= int(catheaderdata
[24], 16);
4828 catfextrasize
= int(catheaderdata
[25], 16);
4829 catfextrafields
= int(catheaderdata
[26], 16);
4830 extrafieldslist
= [];
4832 extraend
= extrastart
+ catfextrafields
;
4833 extrafieldslist
= [];
4834 if(extrastart
<extraend
):
4835 extrafieldslist
.append(catheaderdata
[extrastart
]);
4836 extrastart
= extrastart
+ 1;
4837 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4838 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4839 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4841 hcmax
= len(catheaderdata
) - 2;
4844 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4846 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4847 if(catfcs
!=catnewfcs
and not skipchecksum
):
4848 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
4849 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4851 catfhend
= catfp
.tell() - 1;
4852 catfcontentstart
= catfp
.tell();
4853 catfcontents
= BytesIO();
4854 pyhascontents
= False;
4855 if(catfsize
>0 and not listonly
):
4856 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4857 catfcontents
.write(catfp
.read(catfsize
));
4859 catfcontents
.write(catfp
.read(catfcsize
));
4860 catfcontents
.seek(0, 0);
4861 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
4862 pyhascontents
= True;
4863 if(catfccs
!=catnewfccs
and skipchecksum
):
4864 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
4865 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4867 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4870 catfcontents
.seek(0, 0);
4871 catfcontents
= UncompressArchiveFile(catfcontents
, formatspecs
);
4872 if(catfsize
>0 and listonly
):
4873 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4874 catfp
.seek(catfsize
, 1);
4876 catfp
.seek(catfcsize
, 1);
4877 pyhascontents
= False;
4879 catfcontentend
= catfp
.tell() - 1;
4880 catfcontents
.seek(0, 0);
4881 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fcompression': catfcompression
, 'fcsize': catfcsize
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': catfinode
, 'flinkcount': catflinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fchecksumtype': catfchecksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
} });
4882 fileidnum
= fileidnum
+ 1;
4883 realidnum
= realidnum
+ 1;
4885 catlist
.update({'catfp': catfp
});
4890 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
4892 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4893 catfp
= BytesIO(catstr
);
4894 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4895 return listcatfiles
;
4897 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
4899 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4901 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4902 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4903 return listcatfiles
;
4905 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4907 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4908 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4909 return listcatfiles
;
4911 if(not rarfile_support
):
4912 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4915 if(rarfile_support
):
4916 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4918 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4919 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4920 return listcatfiles
;
4922 if(not py7zr_support
):
4923 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4927 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4929 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4930 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4931 return listcatfiles
;
4933 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4934 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4935 if(checkcompressfile
=="tarfile"):
4936 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4937 elif(checkcompressfile
=="zipfile"):
4938 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4939 elif(checkcompressfile
=="catfile"):
4940 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4941 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4942 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4943 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4944 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4949 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
4950 catver
= formatspecs
['format_ver'];
4951 fileheaderver
= str(int(catver
.replace(".", "")));
4952 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
4953 advancedlist
= formatspecs
['use_advanced_list'];
4954 altinode
= formatspecs
['use_alt_inode'];
4957 for line
in sys
.stdin
:
4958 infilelist
.append(line
.strip());
4959 infilelist
= list(filter(None, infilelist
));
4960 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
4961 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
4963 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
4964 for line
in finfile
:
4965 infilelist
.append(line
.strip());
4966 infilelist
= list(filter(None, infilelist
));
4968 if(isinstance(infiles
, (list, tuple, ))):
4969 infilelist
= list(filter(None, infiles
));
4970 elif(isinstance(infiles
, (str, ))):
4971 infilelist
= list(filter(None, [infiles
]));
4973 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
4975 GetDirList
= ListDir(infilelist
, followlink
, False);
4983 inodetocatinode
= {};
4985 fnumfiles
= int(len(GetDirList
));
4986 catver
= formatspecs
['format_ver'];
4987 fileheaderver
= str(int(catver
.replace(".", "")));
4988 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
4989 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4990 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
4991 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
4992 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4993 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4994 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4995 fheadtell
= len(fileheader
);
4996 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
4997 for curfname
in GetDirList
:
4998 if(re
.findall("^[.|/]", curfname
)):
5001 fname
= "./"+curfname
;
5003 VerbosePrintOut(fname
);
5004 if(not followlink
or followlink
is None):
5005 fstatinfo
= os
.lstat(fname
);
5007 fstatinfo
= os
.stat(fname
);
5008 fpremode
= fstatinfo
.st_mode
;
5009 finode
= fstatinfo
.st_ino
;
5010 flinkcount
= fstatinfo
.st_nlink
;
5012 if(stat
.S_ISREG(fpremode
)):
5014 elif(stat
.S_ISLNK(fpremode
)):
5016 elif(stat
.S_ISCHR(fpremode
)):
5018 elif(stat
.S_ISBLK(fpremode
)):
5020 elif(stat
.S_ISDIR(fpremode
)):
5022 elif(stat
.S_ISFIFO(fpremode
)):
5024 elif(stat
.S_ISSOCK(fpremode
)):
5026 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
5028 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
5030 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
5035 fbasedir
= os
.path
.dirname(fname
);
5037 if(not followlink
and finode
!=0):
5039 if(finode
in inodelist
):
5041 flinkname
= inodetofile
[finode
];
5045 fcurinode
= inodetocatinode
[finode
];
5046 if(finode
not in inodelist
):
5047 inodelist
.append(finode
);
5048 inodetofile
.update({finode
: fname
});
5049 inodetocatinode
.update({finode
: curinode
});
5053 fcurinode
= curinode
;
5054 curinode
= curinode
+ 1;
5056 fcurinode
= curinode
;
5057 curinode
= curinode
+ 1;
5058 curfid
= curfid
+ 1;
5060 flinkname
= os
.readlink(fname
);
5061 fdev
= fstatinfo
.st_dev
;
5062 getfdev
= GetDevMajorMinor(fdev
);
5063 fdev_minor
= getfdev
[0];
5064 fdev_major
= getfdev
[1];
5065 frdev
= fstatinfo
.st_dev
;
5066 if(hasattr(fstatinfo
, "st_rdev")):
5067 frdev
= fstatinfo
.st_rdev
;
5069 frdev
= fstatinfo
.st_dev
;
5070 getfrdev
= GetDevMajorMinor(frdev
);
5071 frdev_minor
= getfrdev
[0];
5072 frdev_major
= getfrdev
[1];
5073 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5075 if(ftype
==0 or ftype
==7):
5076 fsize
= fstatinfo
.st_size
;
5077 fatime
= fstatinfo
.st_atime
;
5078 fmtime
= fstatinfo
.st_mtime
;
5079 fctime
= fstatinfo
.st_ctime
;
5080 if(hasattr(fstatinfo
, "st_birthtime")):
5081 fbtime
= fstatinfo
.st_birthtime
;
5083 fbtime
= fstatinfo
.st_ctime
;
5084 fmode
= fstatinfo
.st_mode
;
5085 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
5086 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
5087 fuid
= fstatinfo
.st_uid
;
5088 fgid
= fstatinfo
.st_gid
;
5093 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
5094 funame
= userinfo
.pw_name
;
5103 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
5104 fgname
= groupinfo
.gr_name
;
5109 fdev_minor
= fdev_minor
;
5110 fdev_major
= fdev_major
;
5111 frdev_minor
= frdev_minor
;
5112 frdev_major
= frdev_major
;
5113 flinkcount
= flinkcount
;
5114 if(hasattr(fstatinfo
, "st_file_attributes")):
5115 fwinattributes
= fstatinfo
.st_file_attributes
;
5120 fcontents
= BytesIO();
5121 if(ftype
==0 or ftype
==7):
5122 with
open(fname
, "rb") as fpc
:
5123 shutil
.copyfileobj(fpc
, fcontents
);
5124 if(followlink
and (ftype
==1 or ftype
==2)):
5125 flstatinfo
= os
.stat(flinkname
);
5126 with
open(flinkname
, "rb") as fpc
:
5127 shutil
.copyfileobj(fpc
, fcontents
);
5128 fcontents
.seek(0, 0);
5129 ftypehex
= format(ftype
, 'x').lower();
5130 extrafields
= len(extradata
);
5131 extrafieldslist
= extradata
;
5132 catfextrafields
= extrafields
;
5133 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5134 if(len(extradata
)>0):
5135 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5136 extrasizelen
= len(extrasizestr
);
5137 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5138 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5139 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5140 catoutlenhex
= format(catoutlen
, 'x').lower();
5141 catoutlist
.insert(0, catoutlenhex
);
5142 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5143 if(len(extradata
)>0):
5144 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5145 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5146 catfnumfields
= catoutlen
;
5147 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5148 fcontents
.seek(0, 0);
5149 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5150 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5151 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5152 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5153 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5154 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5155 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5156 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5157 fheadtell
+= len(catfileoutstr
) + 1;
5158 catfcontentend
= fheadtell
- 1;
5159 fcontents
.seek(0, 0);
5160 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5161 pyhascontents
= False;
5162 if(int(fsize
)>0 and not listonly
):
5163 pyhascontents
= True;
5164 if(int(fsize
)>0 and listonly
):
5165 fcontents
= BytesIO();
5166 pyhascontents
= False;
5167 fcontents
.seek(0, 0);
5168 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5169 fileidnum
= fileidnum
+ 1;
5172 def TarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5178 inodetocatinode
= {};
5182 if(hasattr(sys
.stdin
, "buffer")):
5183 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5185 shutil
.copyfileobj(sys
.stdin
, infile
);
5190 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5191 infile
= download_file_from_internet_file(infile
);
5196 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5198 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
5200 if(not tarfile
.is_tarfile(infile
)):
5202 except AttributeError:
5203 if(not is_tarfile(infile
)):
5208 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
5209 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
5211 tarfp
= tarfile
.open(infile
, "r");
5212 except FileNotFoundError
:
5214 fnumfiles
= int(len(tarfp
.getmembers()));
5215 catver
= formatspecs
['format_ver'];
5216 fileheaderver
= str(int(catver
.replace(".", "")));
5217 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5218 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5219 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5220 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5221 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5222 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5223 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5224 fheadtell
= len(fileheader
);
5225 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5226 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
5227 if(re
.findall("^[.|/]", member
.name
)):
5228 fname
= member
.name
;
5230 fname
= "./"+member
.name
;
5232 VerbosePrintOut(fname
);
5233 fpremode
= member
.mode
;
5234 ffullmode
= member
.mode
;
5238 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5240 elif(member
.isdev()):
5241 ffullmode
= member
.mode
;
5243 elif(member
.islnk()):
5244 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5246 elif(member
.issym()):
5247 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
5249 elif(member
.ischr()):
5250 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
5252 elif(member
.isblk()):
5253 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
5255 elif(member
.isdir()):
5256 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
5258 elif(member
.isfifo()):
5259 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
5261 elif(member
.issparse()):
5262 ffullmode
= member
.mode
;
5265 ffullmode
= member
.mode
;
5268 fbasedir
= os
.path
.dirname(fname
);
5272 curfid
= curfid
+ 1;
5274 flinkname
= member
.linkname
;
5275 fdev_minor
= member
.devminor
;
5276 fdev_major
= member
.devmajor
;
5277 frdev_minor
= member
.devminor
;
5278 frdev_major
= member
.devmajor
;
5279 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5281 elif(ftype
==0 or ftype
==7):
5282 fsize
= member
.size
;
5284 fsize
= member
.size
;
5285 fatime
= member
.mtime
;
5286 fmtime
= member
.mtime
;
5287 fctime
= member
.mtime
;
5288 fbtime
= member
.mtime
;
5290 fchmode
= stat
.S_IMODE(ffullmode
);
5291 ftypemod
= stat
.S_IFMT(ffullmode
);
5294 funame
= member
.uname
;
5295 fgname
= member
.gname
;
5296 flinkcount
= flinkcount
;
5297 fwinattributes
= int(0);
5300 fcontents
= BytesIO();
5301 if(ftype
==0 or ftype
==7):
5302 with tarfp
.extractfile(member
) as fpc
:
5303 shutil
.copyfileobj(fpc
, fcontents
);
5304 fcontents
.seek(0, 0);
5305 ftypehex
= format(ftype
, 'x').lower();
5306 extrafields
= len(extradata
);
5307 extrafieldslist
= extradata
;
5308 catfextrafields
= extrafields
;
5309 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5310 if(len(extradata
)>0):
5311 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5312 extrasizelen
= len(extrasizestr
);
5313 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5314 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5315 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5316 catoutlenhex
= format(catoutlen
, 'x').lower();
5317 catoutlist
.insert(0, catoutlenhex
);
5318 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5319 if(len(extradata
)>0):
5320 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5321 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5322 catfnumfields
= catoutlen
;
5323 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5324 fcontents
.seek(0, 0);
5325 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5326 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5327 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5328 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5329 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5330 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5331 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5332 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5333 fheadtell
+= len(catfileoutstr
) + 1;
5334 catfcontentend
= fheadtell
- 1;
5335 fcontents
.seek(0, 0);
5336 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5337 pyhascontents
= False;
5338 if(int(fsize
)>0 and not listonly
):
5339 pyhascontents
= True;
5340 if(int(fsize
)>0 and listonly
):
5341 fcontents
= BytesIO();
5342 pyhascontents
= False;
5343 fcontents
.seek(0, 0);
5344 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5345 fileidnum
= fileidnum
+ 1;
5348 def ZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5354 inodetocatinode
= {};
5358 if(hasattr(sys
.stdin
, "buffer")):
5359 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5361 shutil
.copyfileobj(sys
.stdin
, infile
);
5366 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5367 infile
= download_file_from_internet_file(infile
);
5372 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5376 if(not zipfile
.is_zipfile(infile
)):
5379 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
5380 except FileNotFoundError
:
5382 ziptest
= zipfp
.testzip();
5384 VerbosePrintOut("Bad file found!");
5385 fnumfiles
= int(len(zipfp
.infolist()));
5386 catver
= formatspecs
['format_ver'];
5387 fileheaderver
= str(int(catver
.replace(".", "")));
5388 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5389 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5390 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5391 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5392 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5393 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5394 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5395 fheadtell
= len(fileheader
);
5396 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5397 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5398 if(re
.findall("^[.|/]", member
.filename
)):
5399 fname
= member
.filename
;
5401 fname
= "./"+member
.filename
;
5402 zipinfo
= zipfp
.getinfo(member
.filename
);
5404 VerbosePrintOut(fname
);
5405 if(not member
.is_dir()):
5406 fpremode
= stat
.S_IFREG
+ 438;
5407 elif(member
.is_dir()):
5408 fpremode
= stat
.S_IFDIR
+ 511;
5411 if(not member
.is_dir()):
5413 elif(member
.is_dir()):
5416 fbasedir
= os
.path
.dirname(fname
);
5420 curfid
= curfid
+ 1;
5428 fsize
= member
.file_size
;
5430 fsize
= member
.file_size
;
5431 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5432 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5433 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5434 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5435 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5436 fwinattributes
= int(zipinfo
.external_attr
);
5437 if(not member
.is_dir()):
5438 fmode
= int(stat
.S_IFREG
+ 438);
5439 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5440 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5441 elif(member
.is_dir()):
5442 fmode
= int(stat
.S_IFDIR
+ 511);
5443 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5444 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5445 elif(zipinfo
.create_system
==3):
5446 fwinattributes
= int(0);
5447 fmode
= int(zipinfo
.external_attr
);
5449 fwinattributes
= int(0);
5450 if(not member
.is_dir()):
5451 fmode
= int(stat
.S_IFREG
+ 438);
5452 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5453 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5454 elif(member
.is_dir()):
5455 fmode
= int(stat
.S_IFDIR
+ 511);
5456 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5457 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5462 except AttributeError:
5468 except AttributeError:
5475 userinfo
= pwd
.getpwuid(os
.getuid());
5476 funame
= userinfo
.pw_name
;
5479 except AttributeError:
5487 groupinfo
= grp
.getgrgid(os
.getgid());
5488 fgname
= groupinfo
.gr_name
;
5491 except AttributeError:
5495 fcontents
= BytesIO();
5497 fcontents
.write(zipfp
.read(member
.filename
));
5498 fcontents
.seek(0, 0);
5499 ftypehex
= format(ftype
, 'x').lower();
5500 extrafields
= len(extradata
);
5501 extrafieldslist
= extradata
;
5502 catfextrafields
= extrafields
;
5503 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5504 if(len(extradata
)>0):
5505 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5506 extrasizelen
= len(extrasizestr
);
5507 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5508 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5509 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5510 catoutlenhex
= format(catoutlen
, 'x').lower();
5511 catoutlist
.insert(0, catoutlenhex
);
5512 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5513 if(len(extradata
)>0):
5514 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5515 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5516 catfnumfields
= catoutlen
;
5517 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5518 fcontents
.seek(0, 0);
5519 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5520 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5521 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5522 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5523 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5524 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5525 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5526 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5527 fheadtell
+= len(catfileoutstr
) + 1;
5528 catfcontentend
= fheadtell
- 1;
5529 fcontents
.seek(0, 0);
5530 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5531 pyhascontents
= False;
5532 if(int(fsize
)>0 and not listonly
):
5533 pyhascontents
= True;
5534 if(int(fsize
)>0 and listonly
):
5535 fcontents
= BytesIO();
5536 pyhascontents
= False;
5537 fcontents
.seek(0, 0);
5538 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5539 fileidnum
= fileidnum
+ 1;
5542 if(not rarfile_support
):
5543 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5546 if(rarfile_support
):
5547 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5553 inodetocatinode
= {};
5555 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5557 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5559 rarfp
= rarfile
.RarFile(infile
, "r");
5560 rartest
= rarfp
.testrar();
5562 VerbosePrintOut("Bad file found!");
5563 fnumfiles
= int(len(rarfp
.infolist()));
5564 catver
= formatspecs
['format_ver'];
5565 fileheaderver
= str(int(catver
.replace(".", "")));
5566 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5567 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5568 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5569 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5570 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5571 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5572 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5573 fheadtell
= len(fileheader
);
5574 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5575 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5578 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5581 member
.external_attr
5583 except AttributeError:
5585 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5588 member
.external_attr
5590 except AttributeError:
5595 if(re
.findall("^[.|/]", member
.filename
)):
5596 fname
= member
.filename
;
5598 fname
= "./"+member
.filename
;
5599 rarinfo
= rarfp
.getinfo(member
.filename
);
5601 VerbosePrintOut(fname
);
5602 if(is_unix
and member
.external_attr
!=0):
5603 fpremode
= int(member
.external_attr
);
5604 elif(member
.is_file()):
5605 fpremode
= stat
.S_IFREG
+ 438;
5606 elif(member
.is_symlink()):
5607 fpremode
= stat
.S_IFLNK
+ 438;
5608 elif(member
.is_dir()):
5609 fpremode
= stat
.S_IFDIR
+ 511;
5610 if(is_windows
and member
.external_attr
!=0):
5611 fwinattributes
= int(member
.external_attr
);
5613 fwinattributes
= int(0);
5618 if(member
.is_file()):
5620 elif(member
.is_symlink()):
5622 elif(member
.is_dir()):
5626 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5627 fbasedir
= os
.path
.dirname(fname
);
5631 curfid
= curfid
+ 1;
5639 fsize
= member
.file_size
;
5642 fatime
= int(member
.atime
.timestamp());
5644 fatime
= int(member
.mtime
.timestamp());
5645 except AttributeError:
5646 fatime
= int(member
.mtime
.timestamp());
5647 fmtime
= int(member
.mtime
.timestamp());
5650 fctime
= int(member
.ctime
.timestamp());
5652 fctime
= int(member
.mtime
.timestamp());
5653 except AttributeError:
5654 fctime
= int(member
.mtime
.timestamp());
5655 fbtime
= int(member
.mtime
.timestamp());
5656 if(is_unix
and member
.external_attr
!=0):
5657 fmode
= int(member
.external_attr
);
5658 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5659 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5660 elif(member
.is_file()):
5661 fmode
= int(stat
.S_IFREG
+ 438)
5662 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5663 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5664 elif(member
.is_symlink()):
5665 fmode
= int(stat
.S_IFLNK
+ 438)
5666 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5667 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5668 elif(member
.is_dir()):
5669 fmode
= int(stat
.S_IFDIR
+ 511)
5670 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5671 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5674 except AttributeError:
5680 except AttributeError:
5687 userinfo
= pwd
.getpwuid(os
.getuid());
5688 funame
= userinfo
.pw_name
;
5691 except AttributeError:
5699 groupinfo
= grp
.getgrgid(os
.getgid());
5700 fgname
= groupinfo
.gr_name
;
5703 except AttributeError:
5707 fcontents
= BytesIO();
5709 fcontents
.write(rarfp
.read(member
.filename
));
5710 fcontents
.seek(0, 0);
5711 ftypehex
= format(ftype
, 'x').lower();
5712 extrafields
= len(extradata
);
5713 extrafieldslist
= extradata
;
5714 catfextrafields
= extrafields
;
5715 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5716 if(len(extradata
)>0):
5717 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5718 extrasizelen
= len(extrasizestr
);
5719 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5720 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5721 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5722 catoutlenhex
= format(catoutlen
, 'x').lower();
5723 catoutlist
.insert(0, catoutlenhex
);
5724 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5725 if(len(extradata
)>0):
5726 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5727 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5728 catfnumfields
= 24 + catfextrafields
;
5729 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5730 fcontents
.seek(0, 0);
5731 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5732 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5733 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5734 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5735 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5736 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5737 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5738 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5739 fheadtell
+= len(catfileoutstr
) + 1;
5740 catfcontentend
= fheadtell
- 1;
5741 fcontents
.seek(0, 0);
5742 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5743 pyhascontents
= False;
5744 if(int(fsize
)>0 and not listonly
):
5745 pyhascontents
= True;
5746 if(int(fsize
)>0 and listonly
):
5747 fcontents
= BytesIO();
5748 pyhascontents
= False;
5749 fcontents
.seek(0, 0);
5750 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5751 fileidnum
= fileidnum
+ 1;
5754 if(not py7zr_support
):
5755 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5759 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5765 inodetocatinode
= {};
5767 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5768 file_content
= szpfp
.readall();
5769 #sztest = szpfp.testzip();
5770 sztestalt
= szpfp
.test();
5772 VerbosePrintOut("Bad file found!");
5773 numfiles
= int(len(szpfp
.list()));
5774 catver
= formatspecs
['format_ver'];
5775 fileheaderver
= str(int(catver
.replace(".", "")));
5776 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5777 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5778 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5779 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5780 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5781 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5782 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5783 fheadtell
= len(fileheader
);
5784 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5785 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5786 if(re
.findall("^[.|/]", member
.filename
)):
5787 fname
= member
.filename
;
5789 fname
= "./"+member
.filename
;
5790 if(not member
.is_directory
):
5791 fpremode
= int(stat
.S_IFREG
+ 438);
5792 elif(member
.is_directory
):
5793 fpremode
= int(stat
.S_IFDIR
+ 511);
5794 fwinattributes
= int(0);
5799 if(member
.is_directory
):
5804 fbasedir
= os
.path
.dirname(fname
);
5808 curfid
= curfid
+ 1;
5815 fatime
= int(member
.creationtime
.timestamp());
5816 fmtime
= int(member
.creationtime
.timestamp());
5817 fctime
= int(member
.creationtime
.timestamp());
5818 fbtime
= int(member
.creationtime
.timestamp());
5819 if(member
.is_directory
):
5820 fmode
= int(stat
.S_IFDIR
+ 511)
5821 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5822 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5824 fmode
= int(stat
.S_IFLNK
+ 438)
5825 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5826 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5829 except AttributeError:
5835 except AttributeError:
5842 userinfo
= pwd
.getpwuid(os
.getuid());
5843 funame
= userinfo
.pw_name
;
5846 except AttributeError:
5854 groupinfo
= grp
.getgrgid(os
.getgid());
5855 fgname
= groupinfo
.gr_name
;
5858 except AttributeError:
5862 fcontents
= BytesIO();
5864 fcontents
.write(file_content
[member
.filename
].read());
5865 fsize
= format(fcontents
.tell(), 'x').lower();
5867 fcontents
.seek(0, 0);
5868 ftypehex
= format(ftype
, 'x').lower();
5869 extrafields
= len(extradata
);
5870 extrafieldslist
= extradata
;
5871 catfextrafields
= extrafields
;
5872 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5873 if(len(extradata
)>0):
5874 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5875 extrasizelen
= len(extrasizestr
);
5876 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5877 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5878 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5879 catoutlenhex
= format(catoutlen
, 'x').lower();
5880 catoutlist
.insert(0, catoutlenhex
);
5881 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5882 if(len(extradata
)>0):
5883 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5884 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5885 catfnumfields
= 24 + catfextrafields
;
5886 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5887 fcontents
.seek(0, 0);
5888 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5889 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5890 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5891 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5892 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5893 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5894 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5895 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5896 fheadtell
+= len(catfileoutstr
) + 1;
5897 catfcontentend
= fheadtell
- 1;
5898 fcontents
.seek(0, 0);
5899 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5900 pyhascontents
= False;
5901 if(int(fsize
)>0 and not listonly
):
5902 pyhascontents
= True;
5903 if(int(fsize
)>0 and listonly
):
5904 fcontents
= BytesIO();
5905 pyhascontents
= False;
5906 fcontents
.seek(0, 0);
5907 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5908 fileidnum
= fileidnum
+ 1;
5911 def InFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5912 checkcompressfile
= CheckCompressionSubType(infiles
, formatspecs
, True);
5913 if(checkcompressfile
=="tarfile"):
5914 return TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5915 elif(checkcompressfile
=="zipfile"):
5916 return ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5917 elif(checkcompressfile
=="catfile"):
5918 return ArchiveFileToArray(infile
, 0, 0, listonly
, False, formatspecs
, False);
5919 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5920 return RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5921 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5922 return SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5927 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
5928 outarray
= BytesIO();
5929 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
5930 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5931 return listcatfiles
;
5933 def ArchiveFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5934 if(isinstance(infile
, dict)):
5935 listcatfiles
= infile
;
5937 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
5938 infile
= RemoveWindowsPath(infile
);
5939 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5940 if(not listcatfiles
):
5942 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5944 catarray
.update({'catfp': listcatfiles
['catfp']});
5945 lenlist
= len(listcatfiles
['ffilelist']);
5947 lcfx
= int(listcatfiles
['fnumfiles']);
5948 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5949 lcfx
= int(lenlist
);
5951 lcfx
= int(listcatfiles
['fnumfiles']);
5953 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5954 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5955 catarray
['filetoid'].update(filetoidarray
);
5956 catarray
['idtofile'].update(idtofilearray
);
5957 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5958 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5959 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5960 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5961 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5962 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5963 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5964 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5965 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5966 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5967 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5968 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5969 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5970 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5971 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5972 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5973 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5974 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5975 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5976 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5977 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5978 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5979 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5980 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5981 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5982 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5983 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5984 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5985 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5986 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5987 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5991 create_alias_function("", __file_format_name__
, "ToArrayIndex", ArchiveFileToArrayIndex
);
5993 def ListDirToArrayIndexAlt(infiles
, dirlistfromtxt
=False, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5994 listcatfiles
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5995 if(not listcatfiles
):
5997 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5998 lenlist
= len(listcatfiles
['ffilelist']);
6000 lcfx
= int(listcatfiles
['fnumfiles']);
6001 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6002 lcfx
= int(lenlist
);
6004 lcfx
= int(listcatfiles
['fnumfiles']);
6006 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6007 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6008 catarray
['filetoid'].update(filetoidarray
);
6009 catarray
['idtofile'].update(idtofilearray
);
6010 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6011 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6012 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6013 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6014 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6015 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6016 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6017 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6018 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6019 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6020 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6021 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6022 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6023 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6024 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6025 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6026 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6027 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6028 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6029 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6030 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6031 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6032 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6033 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6034 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6035 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6036 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6037 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6038 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6039 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6040 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6044 def TarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6045 listcatfiles
= TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6046 if(not listcatfiles
):
6048 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6049 lenlist
= len(listcatfiles
['ffilelist']);
6051 lcfx
= int(listcatfiles
['fnumfiles']);
6052 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6053 lcfx
= int(lenlist
);
6055 lcfx
= int(listcatfiles
['fnumfiles']);
6057 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6058 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6059 catarray
['filetoid'].update(filetoidarray
);
6060 catarray
['idtofile'].update(idtofilearray
);
6061 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6062 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6063 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6064 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6065 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6066 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6067 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6068 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6069 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6070 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6071 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6072 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6073 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6074 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6075 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6076 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6077 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6078 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6079 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6080 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6081 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6082 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6083 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6084 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6085 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6086 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6087 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6088 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6089 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6090 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6091 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6095 def ZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6096 listcatfiles
= ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6097 if(not listcatfiles
):
6099 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6100 lenlist
= len(listcatfiles
['ffilelist']);
6102 lcfx
= int(listcatfiles
['fnumfiles']);
6103 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6104 lcfx
= int(lenlist
);
6106 lcfx
= int(listcatfiles
['fnumfiles']);
6108 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6109 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6110 catarray
['filetoid'].update(filetoidarray
);
6111 catarray
['idtofile'].update(idtofilearray
);
6112 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6113 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6114 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6115 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6116 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6117 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6118 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6119 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6120 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6121 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6122 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6123 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6124 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6125 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6126 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6127 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6128 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6129 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6130 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6131 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6132 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6133 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6134 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6135 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6136 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6137 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6138 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6139 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6140 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6141 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6142 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6146 if(not rarfile_support
):
6147 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6150 if(rarfile_support
):
6151 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6152 listcatfiles
= RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6153 if(not listcatfiles
):
6155 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6156 lenlist
= len(listcatfiles
['ffilelist']);
6158 lcfx
= int(listcatfiles
['fnumfiles']);
6159 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6160 lcfx
= int(lenlist
);
6162 lcfx
= int(listcatfiles
['fnumfiles']);
6164 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6165 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6166 catarray
['filetoid'].update(filetoidarray
);
6167 catarray
['idtofile'].update(idtofilearray
);
6168 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6169 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6170 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6171 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6172 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6173 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6174 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6175 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6176 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6177 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6178 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6179 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6180 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6181 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6182 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6183 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6184 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6185 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6186 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6187 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6188 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6189 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6190 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6191 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6192 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6193 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6194 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6195 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6196 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6197 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6198 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6202 if(not py7zr_support
):
6203 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6207 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6208 listcatfiles
= SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6209 if(not listcatfiles
):
6211 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6212 lenlist
= len(listcatfiles
['ffilelist']);
6214 lcfx
= int(listcatfiles
['fnumfiles']);
6215 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6216 lcfx
= int(lenlist
);
6218 lcfx
= int(listcatfiles
['fnumfiles']);
6220 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6221 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6222 catarray
['filetoid'].update(filetoidarray
);
6223 catarray
['idtofile'].update(idtofilearray
);
6224 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6225 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6226 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6227 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6228 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6229 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6230 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6231 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6232 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6233 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6234 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6235 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6236 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6237 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6238 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6239 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6240 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6241 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6242 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6243 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6244 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6245 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6246 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6247 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6248 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6249 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6250 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6251 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6252 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6253 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6254 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6258 def ArchiveFileStringToArrayIndex(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6259 catfp
= BytesIO(catstr
);
6260 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6261 return listcatfiles
;
6263 create_alias_function("", __file_format_name__
, "StringToArrayIndex", ArchiveFileStringToArrayIndex
);
6265 def TarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6267 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6268 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6269 return listcatfiles
;
6271 def ZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6273 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6274 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6275 return listcatfiles
;
6277 if(not rarfile_support
):
6278 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6281 if(rarfile_support
):
6282 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6284 catfp
= PackArchiveFileFromRarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6285 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6286 return listcatfiles
;
6288 if(not py7zr_support
):
6289 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6293 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6295 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6296 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6297 return listcatfiles
;
6299 def ListDirToArrayIndex(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6300 outarray
= BytesIO();
6301 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
, True);
6302 listcatfiles
= ArchiveFileToArrayIndex(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
)
6303 return listcatfiles
;
6305 def RePackArchiveFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6306 if(isinstance(infile
, dict)):
6307 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6308 listcatfiles
= prelistcatfiles
['list'];
6310 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6311 infile
= RemoveWindowsPath(infile
);
6313 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6314 listcatfiles
= prelistcatfiles
['list'];
6316 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6317 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
6318 outfile
= RemoveWindowsPath(outfile
);
6319 checksumtype
= checksumtype
.lower();
6320 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
6321 checksumtype
="crc32";
6322 if(checksumtype
=="none"):
6324 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
6325 compression
= "auto";
6326 if(compression
not in compressionlist
and compression
is None):
6327 compression
= "auto";
6329 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6330 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
6331 if(os
.path
.exists(outfile
)):
6334 except OSError as e
:
6336 if(not listcatfiles
):
6341 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6343 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6346 fbasename
= os
.path
.splitext(outfile
)[0];
6347 fextname
= os
.path
.splitext(outfile
)[1];
6348 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
6349 catver
= formatspecs
['format_ver'];
6350 fileheaderver
= str(int(catver
.replace(".", "")));
6351 lenlist
= len(listcatfiles
['ffilelist']);
6352 fnumfiles
= int(listcatfiles
['fnumfiles']);
6353 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
6354 fnumfiles
= lenlist
;
6355 AppendFileHeader(catfp
, fnumfiles
, checksumtype
, formatspecs
);
6356 lenlist
= len(listcatfiles
['ffilelist']);
6357 fnumfiles
= int(listcatfiles
['fnumfiles']);
6359 lcfx
= int(listcatfiles
['fnumfiles']);
6360 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6361 lcfx
= int(lenlist
);
6363 lcfx
= int(listcatfiles
['fnumfiles']);
6371 if(re
.findall("^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
6372 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
6374 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
6376 VerbosePrintOut(fname
);
6377 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
6378 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
6379 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6380 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
6381 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
6382 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
6383 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
6384 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
6385 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
6386 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
6387 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6388 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6389 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6390 finode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['finode']), 'x').lower();
6391 flinkcount
= format(int(listcatfiles
['ffilelist'][reallcfi
]['flinkcount']), 'x').lower();
6392 fwinattributes
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fwinattributes']), 'x').lower();
6393 fcompression
= listcatfiles
['ffilelist'][reallcfi
]['fcompression'];
6394 fcsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fcsize']), 'x').lower();
6395 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6396 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6397 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6398 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6399 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6400 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6401 if(not followlink
and len(extradata
)<0):
6402 extradata
= listcatfiles
['ffilelist'][reallcfi
]['fextralist'];
6403 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6405 fcsize
= format(int(0), 'x').lower();
6406 if(not compresswholefile
):
6407 fcontents
.seek(0, 2);
6408 ucfsize
= fcontents
.tell();
6409 fcontents
.seek(0, 0);
6410 if(compression
=="auto"):
6411 ilsize
= len(compressionlistalt
);
6414 while(ilmin
< ilsize
):
6415 cfcontents
= BytesIO();
6416 shutil
.copyfileobj(fcontents
, cfcontents
);
6417 fcontents
.seek(0, 0);
6418 cfcontents
.seek(0, 0);
6419 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
6421 cfcontents
.seek(0, 2);
6422 ilcsize
.append(cfcontents
.tell());
6426 ilcsize
.append(sys
.maxint
);
6427 except AttributeError:
6428 ilcsize
.append(sys
.maxsize
);
6430 ilcmin
= ilcsize
.index(min(ilcsize
));
6431 compression
= compressionlistalt
[ilcmin
];
6432 fcontents
.seek(0, 0);
6433 cfcontents
= BytesIO();
6434 shutil
.copyfileobj(fcontents
, cfcontents
);
6435 cfcontents
.seek(0, 0);
6436 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
6437 cfcontents
.seek(0, 2);
6438 cfsize
= cfcontents
.tell();
6439 if(ucfsize
> cfsize
):
6440 fcsize
= format(int(cfsize
), 'x').lower();
6441 fcompression
= compression
;
6443 fcontents
= cfcontents
;
6445 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6446 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6447 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6448 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6449 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6450 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6451 flinkname
= flinkinfo
['flinkname'];
6452 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6453 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6454 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6455 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6456 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6457 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6458 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6459 funame
= flinkinfo
['funame'];
6460 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6461 fgname
= flinkinfo
['fgname'];
6462 finode
= format(int(flinkinfo
['finode']), 'x').lower();
6463 flinkcount
= format(int(flinkinfo
['flinkcount']), 'x').lower();
6464 fwinattributes
= format(int(flinkinfo
['fwinattributes']), 'x').lower();
6465 fcompression
= flinkinfo
['fcompression'];
6466 fcsize
= format(int(flinkinfo
['fcsize']), 'x').lower();
6467 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6468 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6469 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6470 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6471 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6472 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6473 if(len(extradata
)<0):
6474 extradata
= flinkinfo
['fextralist'];
6475 fcontents
= flinkinfo
['fcontents'];
6476 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6478 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6479 fcurfid
= format(curfid
, 'x').lower();
6480 if(not followlink
and finode
!=0):
6481 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6482 fcurinode
= format(int(curinode
), 'x').lower();
6483 inodetofile
.update({curinode
: fname
});
6484 filetoinode
.update({fname
: curinode
});
6485 curinode
= curinode
+ 1;
6487 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6489 fcurinode
= format(int(curinode
), 'x').lower();
6490 curinode
= curinode
+ 1;
6491 curfid
= curfid
+ 1;
6492 if(fcompression
=="none"):
6494 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
6495 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
6498 reallcfi
= reallcfi
+ 1;
6500 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
6501 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6502 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6505 os
.fsync(catfp
.fileno());
6506 except io
.UnsupportedOperation
:
6508 except AttributeError:
6510 except OSError as e
:
6514 if(hasattr(sys
.stdout
, "buffer")):
6515 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6517 shutil
.copyfileobj(catfp
, sys
.stdout
);
6518 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6519 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6521 upload_file_to_internet_file(catfp
, outfile
);
6529 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6531 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6532 catfp
= BytesIO(catstr
);
6533 listcatfiles
= RePackArchiveFile(catfp
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6534 return listcatfiles
;
6536 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6538 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6539 outarray
= BytesIO();
6540 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6541 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6542 return listcatfiles
;
6544 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6546 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6547 if(outdir
is not None):
6548 outdir
= RemoveWindowsPath(outdir
);
6550 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6551 if(isinstance(infile
, dict)):
6552 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6553 listcatfiles
= prelistcatfiles
['list'];
6555 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6556 infile
= RemoveWindowsPath(infile
);
6558 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6559 listcatfiles
= prelistcatfiles
['list'];
6561 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6562 if(not listcatfiles
):
6564 lenlist
= len(listcatfiles
['ffilelist']);
6565 fnumfiles
= int(listcatfiles
['fnumfiles']);
6567 lcfx
= int(listcatfiles
['fnumfiles']);
6568 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6569 lcfx
= int(lenlist
);
6571 lcfx
= int(listcatfiles
['fnumfiles']);
6577 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6578 funame
= userinfo
.pw_name
;
6587 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6588 fgname
= groupinfo
.gr_name
;
6594 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6595 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6596 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6597 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6598 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6601 os
.fsync(fpc
.fileno());
6602 except io
.UnsupportedOperation
:
6604 except AttributeError:
6606 except OSError as e
:
6608 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6609 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6610 if(preservepermissions
):
6611 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6613 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6614 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6616 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6617 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6618 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6623 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6624 funame
= userinfo
.pw_name
;
6633 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6634 fgname
= groupinfo
.gr_name
;
6639 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6640 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6641 flinkinfo
['fcontents'].seek(0, 0);
6642 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6645 os
.fsync(fpc
.fileno());
6646 except io
.UnsupportedOperation
:
6648 except AttributeError:
6650 except OSError as e
:
6652 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6653 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6654 if(preservepermissions
):
6655 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6657 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6658 if(flinkinfo
['ftype']==1):
6659 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6660 if(flinkinfo
['ftype']==2):
6661 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6662 if(flinkinfo
['ftype']==5):
6663 if(preservepermissions
):
6664 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6666 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6667 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6668 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6669 if(preservepermissions
):
6670 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6672 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6673 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6674 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6676 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6677 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6679 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6680 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6681 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6686 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6687 funame
= userinfo
.pw_name
;
6696 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6697 fgname
= groupinfo
.gr_name
;
6702 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6703 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6704 flinkinfo
['fcontents'].seek(0, 0);
6705 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6708 os
.fsync(fpc
.fileno());
6709 except io
.UnsupportedOperation
:
6711 except AttributeError:
6713 except OSError as e
:
6715 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6716 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6717 if(preservepermissions
):
6718 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6720 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6721 if(flinkinfo
['ftype']==1):
6722 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6723 if(flinkinfo
['ftype']==2):
6724 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6725 if(flinkinfo
['ftype']==5):
6726 if(preservepermissions
):
6727 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6729 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6730 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6731 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6732 if(preservepermissions
):
6733 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6735 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6736 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6737 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6739 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6740 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6741 if(preservepermissions
):
6742 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6744 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6745 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6746 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6747 if(preservepermissions
):
6748 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6750 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6751 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6752 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6755 return listcatfiles
['ffilelist']['catfp'];
6759 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6761 if(hasattr(shutil
, "register_unpack_format")):
6762 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6763 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_dict__
['format_delimiter'], False, False);
6764 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6766 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6767 catfp
= BytesIO(catstr
);
6768 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6769 return listcatfiles
;
6771 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6773 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6774 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6775 if(isinstance(infile
, dict)):
6776 listcatfiles
= infile
;
6778 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6779 infile
= RemoveWindowsPath(infile
);
6780 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, skipchecksum
, formatspecs
, returnfp
);
6781 if(not listcatfiles
):
6783 lenlist
= len(listcatfiles
['ffilelist']);
6784 fnumfiles
= int(listcatfiles
['fnumfiles']);
6786 lcfx
= int(listcatfiles
['fnumfiles']);
6787 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6788 lcfx
= int(lenlist
);
6790 lcfx
= int(listcatfiles
['fnumfiles']);
6793 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6795 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6797 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6798 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6799 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6800 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6801 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6802 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6803 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6804 if(len(fuprint
)<=0):
6805 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6806 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6807 if(len(fgprint
)<=0):
6808 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6809 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6812 return listcatfiles
['catfp'];
6816 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6818 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6819 catfp
= BytesIO(catstr
);
6820 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6821 return listcatfiles
;
6823 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6825 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6826 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6829 if(hasattr(sys
.stdin
, "buffer")):
6830 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6832 shutil
.copyfileobj(sys
.stdin
, infile
);
6837 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6838 infile
= download_file_from_internet_file(infile
);
6843 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6845 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
6847 if(not tarfile
.is_tarfile(infile
)):
6849 except AttributeError:
6850 if(not is_tarfile(infile
)):
6855 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
6856 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
6858 tarfp
= tarfile
.open(infile
, "r");
6859 except FileNotFoundError
:
6863 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
6864 returnval
.update({lcfi
: member
.name
});
6865 fpremode
= member
.mode
;
6866 ffullmode
= member
.mode
;
6870 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6872 elif(member
.isdev()):
6873 ffullmode
= member
.mode
;
6875 elif(member
.islnk()):
6876 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6878 elif(member
.issym()):
6879 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
6881 elif(member
.ischr()):
6882 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
6884 elif(member
.isblk()):
6885 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
6887 elif(member
.isdir()):
6888 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
6890 elif(member
.isfifo()):
6891 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
6893 elif(member
.issparse()):
6894 ffullmode
= member
.mode
;
6897 VerbosePrintOut(member
.name
);
6899 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6900 printfname
= member
.name
;
6902 printfname
= member
.name
+ " link to " + member
.linkname
;
6903 elif(member
.issym()):
6904 printfname
= member
.name
+ " -> " + member
.linkname
;
6905 fuprint
= member
.uname
;
6906 if(len(fuprint
)<=0):
6907 fuprint
= member
.uid
;
6908 fgprint
= member
.gname
;
6909 if(len(fgprint
)<=0):
6910 fgprint
= member
.gid
;
6911 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6914 return listcatfiles
['catfp'];
6918 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6919 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6922 if(hasattr(sys
.stdin
, "buffer")):
6923 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6925 shutil
.copyfileobj(sys
.stdin
, infile
);
6930 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6931 infile
= download_file_from_internet_file(infile
);
6936 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6940 if(not zipfile
.is_zipfile(infile
)):
6943 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
6944 except FileNotFoundError
:
6948 ziptest
= zipfp
.testzip();
6950 VerbosePrintOut("Bad file found!");
6951 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
6952 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
6953 fwinattributes
= int(zipinfo
.external_attr
);
6954 if(not member
.is_dir()):
6955 fmode
= int(stat
.S_IFREG
+ 438);
6956 fchmode
= int(stat
.S_IMODE(fmode
));
6957 ftypemod
= int(stat
.S_IFMT(fmode
));
6958 elif(member
.is_dir()):
6959 fmode
= int(stat
.S_IFDIR
+ 511);
6960 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6961 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6962 elif(zipinfo
.create_system
==3):
6963 fwinattributes
=int(0);
6964 fmode
= int(zipinfo
.external_attr
);
6965 fchmode
= int(stat
.S_IMODE(fmode
));
6966 ftypemod
= int(stat
.S_IFMT(fmode
));
6968 fwinattributes
= int(0);
6969 if(not member
.is_dir()):
6970 fmode
= int(stat
.S_IFREG
+ 438);
6971 fchmode
= int(stat
.S_IMODE(fmode
));
6972 ftypemod
= int(stat
.S_IFMT(fmode
));
6973 elif(member
.is_dir()):
6974 fmode
= int(stat
.S_IFDIR
+ 511);
6975 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6976 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6977 returnval
.update({lcfi
: member
.filename
});
6979 VerbosePrintOut(member
.filename
);
6981 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6983 for fmodval
in str(oct(fmode
))[-3:]:
6984 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6985 if(not member
.is_dir()):
6987 permissionstr
= "-" + permissionstr
;
6988 elif(member
.is_dir()):
6990 permissionstr
= "d" + permissionstr
;
6991 printfname
= member
.filename
;
6993 fuid
= int(os
.getuid());
6994 except AttributeError:
6999 fgid
= int(os
.getgid());
7000 except AttributeError:
7007 userinfo
= pwd
.getpwuid(os
.getuid());
7008 funame
= userinfo
.pw_name
;
7011 except AttributeError:
7019 groupinfo
= grp
.getgrgid(os
.getgid());
7020 fgname
= groupinfo
.gr_name
;
7023 except AttributeError:
7028 if(len(fuprint
)<=0):
7029 fuprint
= str(fuid
);
7031 if(len(fgprint
)<=0):
7032 fgprint
= str(fgid
);
7033 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7036 return listcatfiles
['catfp'];
7040 if(not rarfile_support
):
7041 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
7042 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7043 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7046 if(rarfile_support
):
7047 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
7048 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7049 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7051 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
7055 rarfp
= rarfile
.RarFile(infile
, "r");
7056 rartest
= rarfp
.testrar();
7058 VerbosePrintOut("Bad file found!");
7059 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
7062 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
7065 member
.external_attr
7067 except AttributeError:
7069 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
7072 member
.external_attr
7074 except AttributeError:
7079 if(is_unix
and member
.external_attr
!=0):
7080 fpremode
= int(member
.external_attr
);
7081 elif(member
.is_file()):
7082 fpremode
= int(stat
.S_IFREG
+ 438);
7083 elif(member
.is_symlink()):
7084 fpremode
= int(stat
.S_IFLNK
+ 438);
7085 elif(member
.is_dir()):
7086 fpremode
= int(stat
.S_IFDIR
+ 511);
7087 if(is_windows
and member
.external_attr
!=0):
7088 fwinattributes
= int(member
.external_attr
);
7090 fwinattributes
= int(0);
7091 if(is_unix
and member
.external_attr
!=0):
7092 fmode
= int(member
.external_attr
);
7093 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
7094 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
7095 elif(member
.is_file()):
7096 fmode
= int(stat
.S_IFREG
+ 438);
7097 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
7098 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
7099 elif(member
.is_symlink()):
7100 fmode
= int(stat
.S_IFLNK
+ 438);
7101 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7102 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7103 elif(member
.is_dir()):
7104 fmode
= int(stat
.S_IFDIR
+ 511);
7105 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7106 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7107 returnval
.update({lcfi
: member
.filename
});
7109 VerbosePrintOut(member
.filename
);
7111 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7113 for fmodval
in str(oct(fmode
))[-3:]:
7114 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7115 if(member
.is_file()):
7117 permissionstr
= "-" + permissionstr
;
7118 printfname
= member
.filename
;
7119 elif(member
.is_symlink()):
7121 permissionstr
= "l" + permissionstr
;
7122 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
7123 elif(member
.is_dir()):
7125 permissionstr
= "d" + permissionstr
;
7126 printfname
= member
.filename
;
7128 fuid
= int(os
.getuid());
7129 except AttributeError:
7134 fgid
= int(os
.getgid());
7135 except AttributeError:
7142 userinfo
= pwd
.getpwuid(os
.getuid());
7143 funame
= userinfo
.pw_name
;
7146 except AttributeError:
7154 groupinfo
= grp
.getgrgid(os
.getgid());
7155 fgname
= groupinfo
.gr_name
;
7158 except AttributeError:
7163 if(len(fuprint
)<=0):
7164 fuprint
= str(fuid
);
7166 if(len(fgprint
)<=0):
7167 fgprint
= str(fgid
);
7168 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7171 return listcatfiles
['catfp'];
7175 if(not py7zr_support
):
7176 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7177 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7178 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7182 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7183 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7184 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7188 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
7189 file_content
= szpfp
.readall();
7190 #sztest = szpfp.testzip();
7191 sztestalt
= szpfp
.test();
7193 VerbosePrintOut("Bad file found!");
7194 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
7195 if(re
.findall("^[.|/]", member
.filename
)):
7196 fname
= member
.filename
;
7198 fname
= "./"+member
.filename
;
7199 if(not member
.is_directory
):
7200 fpremode
= int(stat
.S_IFREG
+ 438);
7201 elif(member
.is_directory
):
7202 fpremode
= int(stat
.S_IFDIR
+ 511);
7203 fwinattributes
= int(0);
7204 if(member
.is_directory
):
7205 fmode
= int(stat
.S_IFDIR
+ 511);
7206 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7207 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7209 fmode
= int(stat
.S_IFLNK
+ 438);
7210 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7211 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7212 returnval
.update({lcfi
: member
.filename
});
7214 VerbosePrintOut(member
.filename
);
7216 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7218 for fmodval
in str(oct(fmode
))[-3:]:
7219 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7221 if(not member
.is_directory
):
7223 permissionstr
= "-" + permissionstr
;
7224 printfname
= member
.filename
;
7225 elif(member
.is_directory
):
7227 permissionstr
= "d" + permissionstr
;
7228 printfname
= member
.filename
;
7230 fsize
= len(file_content
[member
.filename
].read());
7231 file_content
[member
.filename
].close();
7233 fuid
= int(os
.getuid());
7234 except AttributeError:
7239 fgid
= int(os
.getgid());
7240 except AttributeError:
7247 userinfo
= pwd
.getpwuid(os
.getuid());
7248 funame
= userinfo
.pw_name
;
7251 except AttributeError:
7259 groupinfo
= grp
.getgrgid(os
.getgid());
7260 fgname
= groupinfo
.gr_name
;
7263 except AttributeError:
7268 if(len(fuprint
)<=0):
7269 fuprint
= str(fuid
);
7271 if(len(fgprint
)<=0):
7272 fgprint
= str(fgid
);
7273 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7276 return listcatfiles
['catfp'];
7280 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
7281 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7282 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
7283 if(checkcompressfile
=="tarfile"):
7284 return TarFileListFiles(infile
, verbose
, returnfp
);
7285 elif(checkcompressfile
=="zipfile"):
7286 return ZipFileListFiles(infile
, verbose
, returnfp
);
7287 elif(checkcompressfile
=="catfile"):
7288 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
7289 elif(rarfile_support
and checkcompressfile
=="rarfile"):
7290 return RarFileListFiles(infile
, verbose
, returnfp
);
7291 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
7292 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
7297 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7298 outarray
= BytesIO();
7299 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
7300 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7301 return listcatfiles
;
7303 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7304 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
7305 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7306 return listcatfiles
;
7308 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7309 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
7310 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
7311 return listcatfiles
;
7313 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
7315 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7316 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7317 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7318 return listcatfiles
;
7320 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
7322 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7323 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7324 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7325 return listcatfiles
;
7327 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
7329 if(not rarfile_support
):
7330 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7333 if(rarfile_support
):
7334 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7335 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7336 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7337 return listcatfiles
;
7339 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
7341 if(not py7zr_support
):
7342 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7346 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7347 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7348 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7349 return listcatfiles
;
7351 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
7353 def download_file_from_ftp_file(url
):
7354 urlparts
= urlparse(url
);
7355 file_name
= os
.path
.basename(urlparts
.path
);
7356 file_dir
= os
.path
.dirname(urlparts
.path
);
7357 if(urlparts
.username
is not None):
7358 ftp_username
= urlparts
.username
;
7360 ftp_username
= "anonymous";
7361 if(urlparts
.password
is not None):
7362 ftp_password
= urlparts
.password
;
7363 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7364 ftp_password
= "anonymous";
7367 if(urlparts
.scheme
=="ftp"):
7369 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7373 if(urlparts
.scheme
=="sftp"):
7375 return download_file_from_pysftp_file(url
);
7377 return download_file_from_sftp_file(url
);
7378 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7379 return download_file_from_http_file(url
);
7380 ftp_port
= urlparts
.port
;
7381 if(urlparts
.port
is None):
7384 ftp
.connect(urlparts
.hostname
, ftp_port
);
7385 except socket
.gaierror
:
7386 log
.info("Error With URL "+url
);
7388 except socket
.timeout
:
7389 log
.info("Error With URL "+url
);
7391 ftp
.login(urlparts
.username
, urlparts
.password
);
7392 if(urlparts
.scheme
=="ftps"):
7394 ftpfile
= BytesIO();
7395 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7396 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7401 def download_file_from_ftp_string(url
):
7402 ftpfile
= download_file_from_ftp_file(url
);
7403 return ftpfile
.read();
7405 def upload_file_to_ftp_file(ftpfile
, url
):
7406 urlparts
= urlparse(url
);
7407 file_name
= os
.path
.basename(urlparts
.path
);
7408 file_dir
= os
.path
.dirname(urlparts
.path
);
7409 if(urlparts
.username
is not None):
7410 ftp_username
= urlparts
.username
;
7412 ftp_username
= "anonymous";
7413 if(urlparts
.password
is not None):
7414 ftp_password
= urlparts
.password
;
7415 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7416 ftp_password
= "anonymous";
7419 if(urlparts
.scheme
=="ftp"):
7421 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7425 if(urlparts
.scheme
=="sftp"):
7427 return upload_file_to_pysftp_file(url
);
7429 return upload_file_to_sftp_file(url
);
7430 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7432 ftp_port
= urlparts
.port
;
7433 if(urlparts
.port
is None):
7436 ftp
.connect(urlparts
.hostname
, ftp_port
);
7437 except socket
.gaierror
:
7438 log
.info("Error With URL "+url
);
7440 except socket
.timeout
:
7441 log
.info("Error With URL "+url
);
7443 ftp
.login(urlparts
.username
, urlparts
.password
);
7444 if(urlparts
.scheme
=="ftps"):
7446 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7451 def upload_file_to_ftp_string(ftpstring
, url
):
7452 ftpfileo
= BytesIO(ftpstring
);
7453 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7457 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7458 # Parse the URL to extract username and password if present
7459 urlparts
= urlparse(url
);
7460 username
= urlparts
.username
;
7461 password
= urlparts
.password
;
7462 # Rebuild the URL without the username and password
7463 netloc
= urlparts
.hostname
;
7464 if(urlparts
.scheme
=="sftp"):
7466 return download_file_from_pysftp_file(url
);
7468 return download_file_from_sftp_file(url
);
7469 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7470 return download_file_from_ftp_file(url
);
7472 netloc
+= ':' + str(urlparts
.port
);
7473 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7474 # Create a temporary file object
7475 httpfile
= BytesIO();
7477 # Use the requests library if available
7478 if username
and password
:
7479 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7481 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7482 response
.raw
.decode_content
= True
7483 shutil
.copyfileobj(response
.raw
, httpfile
);
7485 # Build a Request object for urllib
7486 request
= Request(rebuilt_url
, headers
=headers
);
7487 # Create an opener object for handling URLs
7488 if username
and password
:
7489 # Create a password manager
7490 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7491 # Add the username and password
7492 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7493 # Create an authentication handler using the password manager
7494 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7495 # Build the opener with the authentication handler
7496 opener
= build_opener(auth_handler
);
7498 opener
= build_opener();
7499 with opener
.open(request
) as response
:
7500 shutil
.copyfileobj(response
, httpfile
);
7501 # Reset file pointer to the start
7502 httpfile
.seek(0, 0);
7503 # Return the temporary file object
7506 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7507 httpfile
= download_file_from_http_file(url
, headers
);
7508 return ftpfile
.read();
7511 def download_file_from_sftp_file(url
):
7512 urlparts
= urlparse(url
);
7513 file_name
= os
.path
.basename(urlparts
.path
);
7514 file_dir
= os
.path
.dirname(urlparts
.path
);
7515 sftp_port
= urlparts
.port
;
7516 if(urlparts
.port
is None):
7519 sftp_port
= urlparts
.port
;
7520 if(urlparts
.username
is not None):
7521 sftp_username
= urlparts
.username
;
7523 sftp_username
= "anonymous";
7524 if(urlparts
.password
is not None):
7525 sftp_password
= urlparts
.password
;
7526 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7527 sftp_password
= "anonymous";
7530 if(urlparts
.scheme
=="ftp"):
7531 return download_file_from_ftp_file(url
);
7532 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7533 return download_file_from_http_file(url
);
7534 if(urlparts
.scheme
!="sftp"):
7536 ssh
= paramiko
.SSHClient();
7537 ssh
.load_system_host_keys();
7538 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7540 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7541 except paramiko
.ssh_exception
.SSHException
:
7543 except socket
.gaierror
:
7544 log
.info("Error With URL "+url
);
7546 except socket
.timeout
:
7547 log
.info("Error With URL "+url
);
7549 sftp
= ssh
.open_sftp();
7550 sftpfile
= BytesIO();
7551 sftp
.getfo(urlparts
.path
, sftpfile
);
7554 sftpfile
.seek(0, 0);
7557 def download_file_from_sftp_file(url
):
7561 def download_file_from_sftp_string(url
):
7562 sftpfile
= download_file_from_sftp_file(url
);
7563 return sftpfile
.read();
7565 def download_file_from_ftp_string(url
):
7569 def upload_file_to_sftp_file(sftpfile
, url
):
7570 urlparts
= urlparse(url
);
7571 file_name
= os
.path
.basename(urlparts
.path
);
7572 file_dir
= os
.path
.dirname(urlparts
.path
);
7573 sftp_port
= urlparts
.port
;
7574 if(urlparts
.port
is None):
7577 sftp_port
= urlparts
.port
;
7578 if(urlparts
.username
is not None):
7579 sftp_username
= urlparts
.username
;
7581 sftp_username
= "anonymous";
7582 if(urlparts
.password
is not None):
7583 sftp_password
= urlparts
.password
;
7584 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7585 sftp_password
= "anonymous";
7588 if(urlparts
.scheme
=="ftp"):
7589 return upload_file_to_ftp_file(url
);
7590 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7592 if(urlparts
.scheme
!="sftp"):
7594 ssh
= paramiko
.SSHClient();
7595 ssh
.load_system_host_keys();
7596 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7598 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7599 except paramiko
.ssh_exception
.SSHException
:
7601 except socket
.gaierror
:
7602 log
.info("Error With URL "+url
);
7604 except socket
.timeout
:
7605 log
.info("Error With URL "+url
);
7607 sftp
= ssh
.open_sftp();
7608 sftp
.putfo(sftpfile
, urlparts
.path
);
7611 sftpfile
.seek(0, 0);
7614 def upload_file_to_sftp_file(sftpfile
, url
):
7618 def upload_file_to_sftp_string(sftpstring
, url
):
7619 sftpfileo
= BytesIO(sftpstring
);
7620 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7624 def upload_file_to_sftp_string(url
):
7628 def download_file_from_pysftp_file(url
):
7629 urlparts
= urlparse(url
);
7630 file_name
= os
.path
.basename(urlparts
.path
);
7631 file_dir
= os
.path
.dirname(urlparts
.path
);
7632 sftp_port
= urlparts
.port
;
7633 if(urlparts
.port
is None):
7636 sftp_port
= urlparts
.port
;
7637 if(urlparts
.username
is not None):
7638 sftp_username
= urlparts
.username
;
7640 sftp_username
= "anonymous";
7641 if(urlparts
.password
is not None):
7642 sftp_password
= urlparts
.password
;
7643 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7644 sftp_password
= "anonymous";
7647 if(urlparts
.scheme
=="ftp"):
7648 return download_file_from_ftp_file(url
);
7649 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7650 return download_file_from_http_file(url
);
7651 if(urlparts
.scheme
!="sftp"):
7654 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7655 except paramiko
.ssh_exception
.SSHException
:
7657 except socket
.gaierror
:
7658 log
.info("Error With URL "+url
);
7660 except socket
.timeout
:
7661 log
.info("Error With URL "+url
);
7663 sftp
= ssh
.open_sftp();
7664 sftpfile
= BytesIO();
7665 sftp
.getfo(urlparts
.path
, sftpfile
);
7668 sftpfile
.seek(0, 0);
7671 def download_file_from_pysftp_file(url
):
7675 def download_file_from_pysftp_string(url
):
7676 sftpfile
= download_file_from_pysftp_file(url
);
7677 return sftpfile
.read();
7679 def download_file_from_ftp_string(url
):
7683 def upload_file_to_pysftp_file(sftpfile
, url
):
7684 urlparts
= urlparse(url
);
7685 file_name
= os
.path
.basename(urlparts
.path
);
7686 file_dir
= os
.path
.dirname(urlparts
.path
);
7687 sftp_port
= urlparts
.port
;
7688 if(urlparts
.port
is None):
7691 sftp_port
= urlparts
.port
;
7692 if(urlparts
.username
is not None):
7693 sftp_username
= urlparts
.username
;
7695 sftp_username
= "anonymous";
7696 if(urlparts
.password
is not None):
7697 sftp_password
= urlparts
.password
;
7698 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7699 sftp_password
= "anonymous";
7702 if(urlparts
.scheme
=="ftp"):
7703 return upload_file_to_ftp_file(url
);
7704 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7706 if(urlparts
.scheme
!="sftp"):
7709 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7710 except paramiko
.ssh_exception
.SSHException
:
7712 except socket
.gaierror
:
7713 log
.info("Error With URL "+url
);
7715 except socket
.timeout
:
7716 log
.info("Error With URL "+url
);
7718 sftp
= ssh
.open_sftp();
7719 sftp
.putfo(sftpfile
, urlparts
.path
);
7722 sftpfile
.seek(0, 0);
7725 def upload_file_to_pysftp_file(sftpfile
, url
):
7729 def upload_file_to_pysftp_string(sftpstring
, url
):
7730 sftpfileo
= BytesIO(sftpstring
);
7731 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7735 def upload_file_to_pysftp_string(url
):
7738 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7739 urlparts
= urlparse(url
);
7740 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7741 return download_file_from_http_file(url
, headers
);
7742 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7743 return download_file_from_ftp_file(url
);
7744 elif(urlparts
.scheme
=="sftp"):
7745 if(__use_pysftp__
and havepysftp
):
7746 return download_file_from_pysftp_file(url
);
7748 return download_file_from_sftp_file(url
);
7753 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7754 fp
= download_file_from_internet_file(url
);
7755 fp
= UncompressArchiveFile(fp
, formatspecs
);
7761 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7762 urlparts
= urlparse(url
);
7763 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7764 return download_file_from_http_string(url
, headers
);
7765 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7766 return download_file_from_ftp_string(url
);
7767 elif(urlparts
.scheme
=="sftp"):
7768 if(__use_pysftp__
and havepysftp
):
7769 return download_file_from_pysftp_string(url
);
7771 return download_file_from_sftp_string(url
);
7776 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7777 fp
= download_file_from_internet_string(url
);
7778 fp
= UncompressArchiveFile(fp
, formatspecs
);
7784 def upload_file_to_internet_file(ifp
, url
):
7785 urlparts
= urlparse(url
);
7786 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7788 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7789 return upload_file_to_ftp_file(ifp
, url
);
7790 elif(urlparts
.scheme
=="sftp"):
7791 if(__use_pysftp__
and havepysftp
):
7792 return upload_file_to_pysftp_file(ifp
, url
);
7794 return upload_file_to_sftp_file(ifp
, url
);
7799 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_dict__
):
7800 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
7804 upload_file_to_internet_file(catfp
, outfile
);
7807 def upload_file_to_internet_string(ifp
, url
):
7808 urlparts
= urlparse(url
);
7809 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7811 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7812 return upload_file_to_ftp_string(ifp
, url
);
7813 elif(urlparts
.scheme
=="sftp"):
7814 if(__use_pysftp__
and havepysftp
):
7815 return upload_file_to_pysftp_string(ifp
, url
);
7817 return upload_file_to_sftp_string(ifp
, url
);
7822 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_dict__
):
7823 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, compressionlevel
, formatspecs
);
7827 upload_file_to_internet_file(catfp
, outfile
);
7831 if(hasattr(shutil
, "register_archive_format")):
7832 # Register the packing format
7833 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
7834 except shutil
.RegistryError
:
7838 if(hasattr(shutil
, "register_unpack_format")):
7839 # Register the unpacking format
7840 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
7841 except shutil
.RegistryError
: