2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/15/2024 Ver. 0.11.2 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, tempfile
, zipfile
, platform
;
25 from ftplib
import FTP
, FTP_TLS
;
28 from ftplib
import FTP
;
29 if(sys
.version
[0]=="2"):
30 from urlparse
import urlparse
, urlunparse
;
31 elif(sys
.version
[0]>="3"):
32 from urllib
.parse
import urlunparse
;
33 from urllib
.parse
import urlparse
;
35 if os
.name
== 'nt': # Only modify if on Windows
36 if sys
.version
[0] == "2":
38 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
39 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
42 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
43 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
45 hashlib_guaranteed
= False;
46 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
47 os
.environ
["LC_CTYPE"] = "UTF-8";
51 sys
.setdefaultencoding('UTF-8');
54 except AttributeError:
58 except AttributeError:
62 from zlib
import crc32
;
64 from binascii
import crc32
;
66 if(sys
.version_info
[0]==2):
67 FileNotFoundError
= IOError;
69 rarfile_support
= False;
72 rarfile_support
= True;
74 rarfile_support
= False;
76 py7zr_support
= False;
81 py7zr_support
= False;
84 from xtarfile
import is_tarfile
;
87 from safetar
import is_tarfile
;
89 from tarfile
import is_tarfile
;
92 import xtarfile
as tarfile
;
95 import safetar
as tarfile
;
104 haveparamiko
= False;
113 haverequests
= False;
118 haverequests
= False;
122 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
126 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
127 from urllib
.parse
import urlparse
;
130 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
131 from urlparse
import urlparse
;
133 if(sys
.version
[0]=="2"):
135 from io
import StringIO
, BytesIO
;
138 from cStringIO
import StringIO
;
139 from cStringIO
import StringIO
as BytesIO
;
141 from StringIO
import StringIO
;
142 from StringIO
import StringIO
as BytesIO
;
143 elif(sys
.version
[0]>="3"):
144 from io
import StringIO
, BytesIO
;
149 from cStringIO
import StringIO
as BytesIO
;
155 from StringIO
import StringIO
as BytesIO
;
161 from io
import BytesIO
;
166 __use_pysftp__
= False;
168 __use_pysftp__
= False;
169 __file_format_name__
= "CatFile";
170 __program_name__
= "Py"+__file_format_name__
;
171 __file_format_lower__
= __file_format_name__
.lower();
172 __file_format_magic__
= __file_format_name__
;
173 __file_format_len__
= len(__file_format_magic__
);
174 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__
= "\x00";
176 __file_format_ver__
= "001";
177 __use_new_style__
= True;
178 __use_advanced_list__
= True;
179 __use_alt_inode__
= False;
180 __file_format_extension__
= ".cat";
181 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
182 __file_format_dict__
= {'format_name': __file_format_name__
, 'format_magic': __file_format_magic__
, 'format_lower': __file_format_lower__
, 'format_len': __file_format_len__
, 'format_hex': __file_format_hex__
, 'format_delimiter': __file_format_delimiter__
, 'format_ver': __file_format_ver__
, 'new_style': __use_new_style__
, 'use_advanced_list': __use_advanced_list__
, 'use_alt_inode': __use_alt_inode__
};
183 __project__
= __program_name__
;
184 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
185 __version_info__
= (0, 11, 2, "RC 1", 1);
186 __version_date_info__
= (2024, 5, 15, "RC 1", 1);
187 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
188 __revision__
= __version_info__
[3];
189 __revision_id__
= "$Id$";
190 if(__version_info__
[4] is not None):
191 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
192 if(__version_info__
[4] is None):
193 __version_date_plusrc__
= __version_date__
;
194 if(__version_info__
[3] is not None):
195 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
196 if(__version_info__
[3] is None):
197 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
199 PyBitness
= platform
.architecture();
200 if(PyBitness
=="32bit" or PyBitness
=="32"):
202 elif(PyBitness
=="64bit" or PyBitness
=="64"):
207 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
208 if(platform
.python_implementation()!=""):
209 py_implementation
= platform
.python_implementation();
210 if(platform
.python_implementation()==""):
211 py_implementation
= "Python";
212 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
213 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
214 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
215 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
216 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
217 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
220 compressionsupport
= CompressionSupport();
221 compressionlist
= ['auto'];
222 compressionlistalt
= [];
225 if('gzip' in compressionsupport
):
226 compressionlist
.append('gzip');
227 compressionlistalt
.append('gzip');
228 outextlist
.append('gz');
229 outextlistwd
.append('.gz');
230 if('bzip2' in compressionsupport
):
231 compressionlist
.append('bzip2');
232 compressionlistalt
.append('bzip2');
233 outextlist
.append('bz2');
234 outextlistwd
.append('.bz2');
235 if('zstd' in compressionsupport
):
236 compressionlist
.append('zstd');
237 compressionlistalt
.append('zstd');
238 outextlist
.append('zst');
239 outextlistwd
.append('.zst');
240 if('lz4' in compressionsupport
):
241 compressionlist
.append('lz4');
242 compressionlistalt
.append('lz4');
243 outextlist
.append('lz4');
244 outextlistwd
.append('.lz4');
245 if('lzo' in compressionsupport
):
246 compressionlist
.append('lzo');
247 compressionlistalt
.append('lzo');
248 outextlist
.append('lzo');
249 outextlistwd
.append('.lzo');
250 if('lzop' in compressionsupport
):
251 compressionlist
.append('lzop');
252 compressionlistalt
.append('lzop');
253 outextlist
.append('lzop');
254 outextlistwd
.append('.lzop');
255 if('lzma' in compressionsupport
):
256 compressionlist
.append('lzma');
257 compressionlistalt
.append('lzma');
258 outextlist
.append('lzma');
259 outextlistwd
.append('.lzma');
260 if('xz' in compressionsupport
):
261 compressionlist
.append('xz');
262 compressionlistalt
.append('xz');
263 outextlist
.append('xz');
264 outextlistwd
.append('.xz');
266 tarfile_mimetype
= "application/tar";
267 tarfile_tar_mimetype
= tarfile_mimetype
;
268 zipfile_mimetype
= "application/zip";
269 zipfile_zip_mimetype
= zipfile_mimetype
;
270 rarfile_mimetype
= "application/rar";
271 rarfile_rar_mimetype
= rarfile_mimetype
;
272 archivefile_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"";
273 archivefile_cat_mimetype
= archivefile_mimetype
;
274 archivefile_gzip_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+gzip";
275 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
276 archivefile_bzip2_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+bzip2";
277 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
278 archivefile_lz4_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lz4";
279 archivefile_lzop_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzop";
280 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
281 archivefile_zstandard_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+zstandard";
282 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
283 archivefile_lzma_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzma";
284 archivefile_xz_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+xz";
285 archivefile_extensions
= [__file_format_extension__
, __file_format_extension__
+".gz", __file_format_extension__
+".bz2", __file_format_extension__
+".zst", __file_format_extension__
+".lz4", __file_format_extension__
+".lzo", __file_format_extension__
+".lzop", __file_format_extension__
+".lzma", __file_format_extension__
+".xz"];
287 if __name__
== "__main__":
289 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
293 curscrpath
= curscrpath
.replace(os
.sep
, "/");
294 curscrpath
= curscrpath
+ "/";
295 scrfile
= curscrpath
+ "catfile.py";
296 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
297 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
300 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
306 "warning": logging
.warning
,
307 "error": logging
.error
,
308 "critical": logging
.critical
,
309 "exception": logging
.exception
,
310 "logalt": lambda x
: logging
.log(dgblevel
, x
),
311 "debug": logging
.debug
313 log_function
= log_functions
.get(outtype
);
315 log_function(dbgtxt
);
319 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
320 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
323 def RemoveWindowsPath(dpath
):
327 dpath
= dpath
.replace(os
.path
.sep
, "/");
328 dpath
= dpath
.rstrip("/");
329 if(dpath
=="." or dpath
==".."):
333 def NormalizeRelativePath(inpath
):
334 inpath
= RemoveWindowsPath(inpath
);
335 if(os
.path
.isabs(inpath
)):
338 if(inpath
.startswith("./") or inpath
.startswith("../")):
341 outpath
= "./" + inpath
;
344 def PrependPath(base_dir
, child_path
):
345 # Check if base_dir is None or empty, if so, return child_path as is
348 # Ensure base_dir ends with exactly one slash
349 if not base_dir
.endswith('/'):
351 # Check if child_path starts with ./ or ../ (indicating a relative path)
352 if child_path
.startswith('./') or child_path
.startswith('../'):
353 # For relative paths, we don't alter the child_path
354 return base_dir
+ child_path
;
356 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
357 return base_dir
+ child_path
.lstrip('/');
359 def ListDir(dirpath
, followlink
=False, duplicates
=False):
360 if(isinstance(dirpath
, (list, tuple, ))):
361 dirpath
= list(filter(None, dirpath
));
362 elif(isinstance(dirpath
, (str, ))):
363 dirpath
= list(filter(None, [dirpath
]));
365 for mydirfile
in dirpath
:
366 if(not os
.path
.exists(mydirfile
)):
368 mydirfile
= NormalizeRelativePath(mydirfile
);
369 if(os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
)):
370 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
371 if(os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
)):
372 for root
, dirs
, filenames
in os
.walk(mydirfile
):
374 dpath
= RemoveWindowsPath(dpath
);
375 if(dpath
not in retlist
and not duplicates
):
376 retlist
.append(dpath
);
378 retlist
.append(dpath
);
379 for file in filenames
:
380 fpath
= os
.path
.join(root
, file);
381 fpath
= RemoveWindowsPath(fpath
);
382 if(fpath
not in retlist
and not duplicates
):
383 retlist
.append(fpath
);
385 retlist
.append(fpath
);
387 retlist
.append(RemoveWindowsPath(mydirfile
));
390 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
391 if isinstance(dirpath
, (list, tuple)):
392 dirpath
= list(filter(None, dirpath
));
393 elif isinstance(dirpath
, str):
394 dirpath
= list(filter(None, [dirpath
]));
396 for mydirfile
in dirpath
:
397 if not os
.path
.exists(mydirfile
):
399 mydirfile
= NormalizeRelativePath(mydirfile
);
400 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
401 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
))
402 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
403 for root
, dirs
, filenames
in os
.walk(mydirfile
):
404 # Sort dirs and filenames alphabetically in place
405 dirs
.sort(key
=lambda x
: x
.lower());
406 filenames
.sort(key
=lambda x
: x
.lower());
407 dpath
= RemoveWindowsPath(root
);
408 if not duplicates
and dpath
not in retlist
:
409 retlist
.append(dpath
);
411 retlist
.append(dpath
);
412 for file in filenames
:
413 fpath
= os
.path
.join(root
, file);
414 fpath
= RemoveWindowsPath(fpath
);
415 if not duplicates
and fpath
not in retlist
:
416 retlist
.append(fpath
);
418 retlist
.append(fpath
);
420 retlist
.append(RemoveWindowsPath(mydirfile
));
423 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
424 # Define a new function that wraps the target function
425 def alias_function(*args
, **kwargs
):
426 return target_function(*args
, **kwargs
);
428 # Create the function name by combining the prefix, base name, and the suffix
429 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
431 # Add the new function to the global namespace
432 globals()[function_name
] = alias_function
;
434 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
435 # Create the function name by combining the prefix, base name, and the suffix
436 # Use the format method for string formatting, compatible with Python 2 and 3
437 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
438 # Add the new function (alias of the target_function) to the global namespace
439 # This line is compatible as-is with both Python 2 and 3
440 globals()[function_name
] = target_function
442 def FormatSpecsListToDict(formatspecs
=__file_format_list__
):
443 if(isinstance(formatspecs
, (list, tuple, ))):
444 return {'format_name': formatspecs
[0], 'format_magic': formatspecs
[1], 'format_lower': formatspecs
[2], 'format_len': formatspecs
[3], 'format_hex': formatspecs
[4], 'format_delimiter': formatspecs
[5], 'format_ver': formatspecs
[6], 'new_style': formatspecs
[7], 'use_advanced_list': formatspecs
[8], 'use_alt_inode': formatspecs
[9]};
445 elif(isinstance(formatspecs
, (dict, ))):
448 return __file_format_dict__
;
449 return __file_format_dict__
;
451 # initial_value can be 0xFFFF or 0x0000
452 def crc16_ansi(msg
, initial_value
=0xFFFF):
453 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
454 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
455 crc
= initial_value
; # Initial value
457 crc ^
= b
<< 8; # XOR byte into CRC top byte
458 for _
in range(8): # Process each bit
459 if crc
& 0x8000: # If the top bit is set
460 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
462 crc
= crc
<< 1; # Just shift left
463 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
466 # initial_value can be 0xFFFF or 0x0000
467 def crc16_ibm(msg
, initial_value
=0xFFFF):
468 return crc16_ansi(msg
, initial_value
);
470 # initial_value is 0xFFFF
472 return crc16_ansi(msg
, 0xFFFF);
474 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
475 def crc16_ccitt(msg
, initial_value
=0xFFFF):
476 # CRC-16-CCITT polynomial
477 poly
= 0x1021; # Polynomial for CRC-16-CCITT
478 # Use the specified initial value
481 crc ^
= b
<< 8; # XOR byte into CRC top byte
482 for _
in range(8): # Process each bit
483 if crc
& 0x8000: # If the top bit is set
484 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
486 crc
= crc
<< 1; # Just shift left
487 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
490 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
491 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
492 # CRC-64-ECMA polynomial and initial value
493 poly
= 0x42F0E1EBA9EA3693;
494 crc
= initial_value
; # Initial value for CRC-64-ECMA
496 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
497 for _
in range(8): # Process each bit
498 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
499 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
501 crc
<<= 1; # Just shift left if the MSB is 0
502 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
505 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
506 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
507 # CRC-64-ISO polynomial and initial value
508 poly
= 0x000000000000001B;
509 crc
= initial_value
; # Common initial value for CRC-64-ISO
511 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
512 for _
in range(8): # Process each bit
513 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
514 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
516 crc
<<= 1; # Just shift left if the MSB is 0
517 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
520 def GetDataFromArray(data
, path
, default
=None):
524 element
= element
[key
];
526 except (KeyError, TypeError, IndexError):
529 def GetDataFromArrayAlt(structure
, path
, default
=None):
532 if isinstance(element
, dict) and key
in element
:
533 element
= element
[key
];
534 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
535 element
= element
[key
];
540 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
541 formatspecs
= FormatSpecsListToDict(formatspecs
);
542 if isinstance(inlist
, list):
543 fileheader
= AppendNullBytes(inlist
, formatspecs
['format_delimiter']);
545 fileheader
= AppendNullByte(inlist
, formatspecs
['format_delimiter']);
547 fileheader
= fileheader
.encode('UTF-8');
548 if(checksumtype
=="none" or checksumtype
==""):
549 catfileheadercshex
= format(0, 'x').lower();
550 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
551 catfileheadercshex
= format(crc16(fileheader
) & 0xffff, '04x').lower();
552 elif(checksumtype
=="crc16_ccitt"):
553 catfileheadercshex
= format(crc16_ccitt(fileheader
) & 0xffff, '04x').lower();
554 elif(checksumtype
=="adler32"):
555 catfileheadercshex
= format(zlib
.adler32(fileheader
) & 0xffffffff, '08x').lower();
556 elif(checksumtype
=="crc32"):
557 catfileheadercshex
= format(crc32(fileheader
) & 0xffffffff, '08x').lower();
558 elif(checksumtype
=="crc64_ecma"):
559 catfileheadercshex
= format(crc64_ecma(fileheader
) & 0xffffffffffffffff, '016x').lower();
560 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
561 catfileheadercshex
= format(crc64_iso(fileheader
) & 0xffffffffffffffff, '016x').lower();
562 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
563 checksumoutstr
= hashlib
.new(checksumtype
);
564 checksumoutstr
.update(fileheader
);
565 catfileheadercshex
= checksumoutstr
.hexdigest().lower();
567 catfileheadercshex
= format(0, 'x').lower();
568 return catfileheadercshex
;
570 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
571 formatspecs
= FormatSpecsListToDict(formatspecs
);
573 instr
= instr
.encode('UTF-8');
574 if(checksumtype
=="none" or checksumtype
==""):
575 catinstrcshex
= format(0, 'x').lower();
576 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
577 catinstrcshex
= format(crc16(instr
) & 0xffff, '04x').lower();
578 elif(checksumtype
=="crc16_ccitt"):
579 catinstrcshex
= format(crc16_ccitt(instr
) & 0xffff, '04x').lower();
580 elif(checksumtype
=="adler32"):
581 catinstrcshex
= format(zlib
.adler32(instr
) & 0xffffffff, '08x').lower();
582 elif(checksumtype
=="crc32"):
583 catinstrcshex
= format(crc32(instr
) & 0xffffffff, '08x').lower();
584 elif(checksumtype
=="crc64_ecma"):
585 catinstrcshex
= format(crc64_ecma(instr
) & 0xffffffffffffffff, '016x').lower();
586 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
587 catinstrcshex
= format(crc64_iso(instr
) & 0xffffffffffffffff, '016x').lower();
588 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
589 checksumoutstr
= hashlib
.new(checksumtype
);
590 checksumoutstr
.update(instr
);
591 catinstrcshex
= checksumoutstr
.hexdigest().lower();
593 catinstrcshex
= format(0, 'x').lower();
594 return catinstrcshex
;
596 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
597 formatspecs
= FormatSpecsListToDict(formatspecs
);
598 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
);
599 inchecksum
= inchecksum
.lower();
600 catfileheadercshex
= catfileheadercshex
.lower();
601 if(inchecksum
==catfileheadercshex
):
606 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
607 formatspecs
= FormatSpecsListToDict(formatspecs
);
608 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
);
609 inchecksum
= inchecksum
.lower();
610 catinfilecshex
= catinfilecshex
.lower();
611 if(inchecksum
==catinfilecshex
):
616 def ReadTillNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
619 nullbyte
= delimiter
.encode("UTF-8");
621 curbyte
= fp
.read(1);
622 if(curbyte
==nullbyte
or not curbyte
):
624 curfullbyte
= curfullbyte
+ curbyte
;
625 return curfullbyte
.decode('UTF-8');
627 def ReadUntilNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
628 return ReadTillNullByteOld(fp
, delimiter
);
630 def ReadTillNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
631 delimiter
= delimiter
.encode('UTF-8') # Ensure the delimiter is in bytes
632 buffer = bytearray();
634 delimiter_length
= len(delimiter
);
636 chunk
= fp
.read(chunk_size
)
638 # End of file reached without finding the delimiter
640 buffer.extend(chunk
);
641 total_read
+= len(chunk
);
642 if delimiter
in buffer:
643 # Delimiter found, calculate where to reset the file pointer
644 index
= buffer.find(delimiter
);
645 # Calculate how many extra bytes were read after the delimiter
646 extra_bytes_read
= len(buffer) - (index
+ delimiter_length
);
647 # Move the file pointer back to just after the delimiter
648 fp
.seek(-extra_bytes_read
, 1);
649 buffer = buffer[:index
];
651 if total_read
>= max_read
:
652 # Stop reading if max limit is reached to prevent excessive memory usage
653 raise MemoryError("Maximum read limit reached without finding the delimiter.");
654 # Check for incomplete UTF-8 sequences at the end of the buffer
655 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
656 # This suggests that the last byte might be the start of a multi-byte character
657 # Try to read one more byte to complete the character
658 extra_byte
= fp
.read(1);
660 buffer.extend(extra_byte
);
662 # No more data available
665 return buffer.decode('UTF-8', errors
='replace');
666 except UnicodeDecodeError:
667 return buffer.decode('UTF-8', errors
='replace');
669 def ReadUntilNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
670 return ReadTillNullByteAlt(fp
, delimiter
, chunk_size
, max_read
);
672 def ReadTillNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
673 curfullbyte
= bytearray();
674 nullbyte
= delimiter
.encode("UTF-8");
675 total_read
= 0; # Track the total number of bytes read
677 curbyte
= fp
.read(1);
678 if curbyte
== nullbyte
or not curbyte
:
680 curfullbyte
.extend(curbyte
);
682 if total_read
>= max_read
:
683 raise MemoryError("Maximum read limit reached without finding the delimiter.");
684 # Decode the full byte array to string once out of the loop
686 return curfullbyte
.decode('UTF-8');
687 except UnicodeDecodeError:
688 # Handle potential partial UTF-8 characters
689 for i
in range(1, 4):
691 return curfullbyte
[:-i
].decode('UTF-8');
692 except UnicodeDecodeError:
694 raise; # Re-raise if decoding fails even after trimming
696 def ReadUntilNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
697 return ReadTillNullByte(fp
, delimiter
, max_read
);
699 def ReadTillNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
700 delimiter
= delimiter
.encode('UTF-8'); # Ensure the delimiter is in bytes
701 buffer = bytearray();
703 delimiter_length
= len(delimiter
);
705 while len(results
) < num_delimiters
:
706 chunk
= fp
.read(chunk_size
);
708 # End of file reached; decode whatever is collected if it's the last needed part
710 results
.append(buffer.decode('UTF-8', errors
='replace'));
713 total_read
+= len(chunk
);
714 # Check if we have found the delimiter
715 while delimiter
in buffer:
716 index
= buffer.find(delimiter
);
717 # Decode the section before the delimiter
718 results
.append(buffer[:index
].decode('UTF-8', errors
='replace'));
719 # Remove the processed part from the buffer
720 buffer = buffer[index
+ delimiter_length
:];
721 if len(results
) == num_delimiters
:
722 # If reached the required number of delimiters, adjust the file pointer and stop
723 fp
.seek(-len(buffer), 1);
725 if total_read
>= max_read
:
726 # Stop reading if max limit is reached to prevent excessive memory usage
727 raise MemoryError("Maximum read limit reached without finding the delimiter.");
728 # Check for incomplete UTF-8 sequences at the end of the buffer
729 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
730 # This suggests that the last byte might be the start of a multi-byte character
731 # Try to read one more byte to complete the character
732 extra_byte
= fp
.read(1);
734 buffer.extend(extra_byte
);
736 # No more data available
738 # Process remaining buffer if less than the required number of delimiters were found
739 if len(buffer) > 0 and len(results
) < num_delimiters
:
740 results
.append(buffer.decode('UTF-8', errors
='replace'));
743 def ReadUntilNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
744 return ReadTillNullByteByNum(fp
, delimiter
, num_delimiters
, chunk_size
, max_read
);
746 def SeekToEndOfFile(fp
):
750 if(lasttell
==fp
.tell()):
752 lasttell
= fp
.tell();
755 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_dict__
['format_delimiter']):
759 while(rocount
<roend
):
760 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
761 rocount
= rocount
+ 1;
764 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
765 headerpresize
= ReadTillNullByte(fp
, delimiter
);
766 headersize
= int(headerpresize
, 16);
769 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
772 roend
= int(len(headercontent
));
773 HeaderOut
= [headerpresize
];
774 while(rocount
<roend
):
775 HeaderOut
.append(headercontent
[rocount
]);
776 rocount
= rocount
+ 1;
779 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
780 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
781 headersize
= int(preheaderdata
[0], 16);
782 headernumfields
= int(preheaderdata
[1], 16);
783 if(headersize
<=0 or headernumfields
<=0):
785 headerdata
= ReadTillNullByteByNum(fp
, delimiter
, headernumfields
);
786 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
787 HeaderOut
= preheaderdata
+ headerdata
;
790 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
791 formatspecs
= FormatSpecsListToDict(formatspecs
);
792 delimiter
= formatspecs
['format_delimiter'];
793 fheaderstart
= fp
.tell();
794 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
795 if(len(HeaderOut
)==0):
797 if(re
.findall("^[.|/]", HeaderOut
[3])):
798 fname
= HeaderOut
[3];
800 fname
= "./"+HeaderOut
[3];
801 fchecksumtype
= HeaderOut
[-3].lower();
802 fcs
= HeaderOut
[-2].lower();
803 fccs
= HeaderOut
[-1].lower();
804 fsize
= int(HeaderOut
[5], 16);
805 fcompression
= HeaderOut
[12];
806 fcsize
= int(HeaderOut
[13], 16);
807 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
808 if(fcs
!=newfcs
and not skipchecksum
):
809 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
810 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
812 fhend
= fp
.tell() - 1;
813 fcontentstart
= fp
.tell();
814 fcontents
= BytesIO();
815 if(fsize
>0 and not listonly
):
816 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
817 fcontents
.write(fp
.read(fsize
));
819 fcontents
.write(fp
.read(fcsize
));
820 elif(fsize
>0 and listonly
):
821 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
825 fcontents
.seek(0, 0);
826 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
827 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
828 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
829 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
831 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
834 fcontents
.seek(0, 0);
836 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
838 fcontentend
= fp
.tell() - 1;
839 HeaderOut
.append(fcontents
);
842 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
843 formatspecs
= FormatSpecsListToDict(formatspecs
);
844 delimiter
= formatspecs
['format_delimiter'];
845 fheaderstart
= fp
.tell();
846 if(formatspecs
['new_style']):
847 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
849 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
850 if(len(HeaderOut
)==0):
852 fheadsize
= int(HeaderOut
[0], 16);
853 fnumfields
= int(HeaderOut
[1], 16);
854 ftype
= int(HeaderOut
[2], 16);
855 if(re
.findall("^[.|/]", HeaderOut
[3])):
856 fname
= HeaderOut
[3];
858 fname
= "./"+HeaderOut
[3];
859 fbasedir
= os
.path
.dirname(fname
);
860 flinkname
= HeaderOut
[4];
861 fsize
= int(HeaderOut
[5], 16);
862 fatime
= int(HeaderOut
[6], 16);
863 fmtime
= int(HeaderOut
[7], 16);
864 fctime
= int(HeaderOut
[8], 16);
865 fbtime
= int(HeaderOut
[9], 16);
866 fmode
= int(HeaderOut
[10], 16);
867 fchmode
= stat
.S_IMODE(fmode
);
868 ftypemod
= stat
.S_IFMT(fmode
);
869 fwinattributes
= int(HeaderOut
[11], 16);
870 fcompression
= HeaderOut
[12];
871 fcsize
= int(HeaderOut
[13], 16);
872 fuid
= int(HeaderOut
[14], 16);
873 funame
= HeaderOut
[15];
874 fgid
= int(HeaderOut
[16], 16);
875 fgname
= HeaderOut
[17];
876 fid
= int(HeaderOut
[18], 16);
877 finode
= int(HeaderOut
[19], 16);
878 flinkcount
= int(HeaderOut
[20], 16);
879 fdev_minor
= int(HeaderOut
[21], 16);
880 fdev_major
= int(HeaderOut
[22], 16);
881 frdev_minor
= int(HeaderOut
[23], 16);
882 frdev_major
= int(HeaderOut
[24], 16);
883 fextrasize
= int(HeaderOut
[25], 16);
884 fextrafields
= int(HeaderOut
[26], 16);
885 extrafieldslist
= [];
887 extraend
= extrastart
+ fextrafields
;
888 extrafieldslist
= [];
889 if(extrastart
<extraend
):
890 extrafieldslist
.append(HeaderOut
[extrastart
]);
891 extrastart
= extrastart
+ 1;
892 fchecksumtype
= HeaderOut
[extrastart
].lower();
893 fcs
= HeaderOut
[extrastart
+ 1].lower();
894 fccs
= HeaderOut
[extrastart
+ 2].lower();
895 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
896 if(fcs
!=newfcs
and not skipchecksum
):
897 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
898 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
900 fhend
= fp
.tell() - 1;
901 fcontentstart
= fp
.tell();
902 fcontents
= BytesIO();
903 pyhascontents
= False;
904 if(fsize
>0 and not listonly
):
905 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
906 fcontents
.write(fp
.read(fsize
));
908 fcontents
.write(fp
.read(fcsize
));
909 pyhascontents
= True;
910 elif(fsize
>0 and listonly
):
911 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
915 pyhascontents
= False;
916 fcontents
.seek(0, 0);
917 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
918 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
919 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
920 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
922 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
925 fcontents
.seek(0, 0);
927 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
928 fcontents
.seek(0, 0);
929 fccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
931 fcontentend
= fp
.tell() - 1;
932 fcontents
.seek(0, 0);
933 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': fchecksumtype
, 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
936 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
937 formatspecs
= FormatSpecsListToDict(formatspecs
);
938 delimiter
= formatspecs
['format_delimiter'];
939 fheaderstart
= fp
.tell();
940 if(formatspecs
['new_style']):
941 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
943 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
944 if(len(HeaderOut
)==0):
946 fheadsize
= int(HeaderOut
[0], 16);
947 fnumfields
= int(HeaderOut
[1], 16);
948 ftype
= int(HeaderOut
[2], 16);
949 if(re
.findall("^[.|/]", HeaderOut
[3])):
950 fname
= HeaderOut
[3];
952 fname
= "./"+HeaderOut
[3];
953 fbasedir
= os
.path
.dirname(fname
);
954 flinkname
= HeaderOut
[4];
955 fsize
= int(HeaderOut
[5], 16);
956 fatime
= int(HeaderOut
[6], 16);
957 fmtime
= int(HeaderOut
[7], 16);
958 fctime
= int(HeaderOut
[8], 16);
959 fbtime
= int(HeaderOut
[9], 16);
960 fmode
= int(HeaderOut
[10], 16);
961 fchmode
= stat
.S_IMODE(fmode
);
962 ftypemod
= stat
.S_IFMT(fmode
);
963 fwinattributes
= int(HeaderOut
[11], 16);
964 fcompression
= HeaderOut
[12];
965 fcsize
= int(HeaderOut
[13], 16);
966 fuid
= int(HeaderOut
[14], 16);
967 funame
= HeaderOut
[15];
968 fgid
= int(HeaderOut
[16], 16);
969 fgname
= HeaderOut
[17];
970 fid
= int(HeaderOut
[18], 16);
971 finode
= int(HeaderOut
[19], 16);
972 flinkcount
= int(HeaderOut
[20], 16);
973 fdev_minor
= int(HeaderOut
[21], 16);
974 fdev_major
= int(HeaderOut
[22], 16);
975 frdev_minor
= int(HeaderOut
[23], 16);
976 frdev_major
= int(HeaderOut
[24], 16);
977 fextrasize
= int(HeaderOut
[25], 16);
978 fextrafields
= int(HeaderOut
[26], 16);
979 extrafieldslist
= [];
981 extraend
= extrastart
+ fextrafields
;
982 extrafieldslist
= [];
983 if(extrastart
<extraend
):
984 extrafieldslist
.append(HeaderOut
[extrastart
]);
985 extrastart
= extrastart
+ 1;
986 fchecksumtype
= HeaderOut
[extrastart
].lower();
987 fcs
= HeaderOut
[extrastart
+ 1].lower();
988 fccs
= HeaderOut
[extrastart
+ 2].lower();
989 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
990 if(fcs
!=newfcs
and not skipchecksum
):
991 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
992 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
994 fhend
= fp
.tell() - 1;
995 fcontentstart
= fp
.tell();
996 fcontents
= BytesIO();
997 pyhascontents
= False;
998 if(fsize
>0 and not listonly
):
999 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1000 fcontents
.write(fp
.read(fsize
));
1002 fcontents
.write(fp
.read(fcsize
));
1003 pyhascontents
= True;
1004 elif(fsize
>0 and listonly
):
1005 if(fcompression
=="none" or fcompression
=="" or fcompression
=="atuo"):
1009 pyhascontents
= False;
1010 fcontents
.seek(0, 0);
1011 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1012 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
1013 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
1014 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
1016 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1019 fcontents
.seek(0, 0);
1021 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
1022 fcontents
.seek(0, 0);
1024 fcontentend
= fp
.tell() - 1;
1025 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrafieldslist
, fchecksumtype
, fcontents
];
1028 def ReadFileDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1029 formatspecs
= FormatSpecsListToDict(formatspecs
);
1030 delimiter
= formatspecs
['format_delimiter'];
1034 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1037 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1038 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1039 if(not headercheck
and not skipchecksum
):
1040 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1041 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1043 fnumfiles
= int(catheader
[1], 16);
1046 while(countnum
< fnumfiles
):
1047 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
)
1048 if(len(HeaderOut
)==0):
1050 flist
.append(HeaderOut
);
1051 countnum
= countnum
+ 1;
1054 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1055 formatspecs
= FormatSpecsListToDict(formatspecs
);
1056 delimiter
= formatspecs
['format_delimiter'];
1060 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1063 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1064 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1065 if(not headercheck
and not skipchecksum
):
1066 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1067 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1069 catstring
= catheader
[0];
1070 catversion
= re
.findall(r
"([\d]+)", catstring
);
1071 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1072 fprenumfiles
= catheader
[1];
1073 fnumfiles
= int(fprenumfiles
, 16);
1074 fprechecksumtype
= catheader
[2];
1075 fprechecksum
= catheader
[3];
1076 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
1077 if(seekstart
<0 and seekstart
>fnumfiles
):
1079 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1080 seekend
= fnumfiles
;
1081 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1082 seekend
= fnumfiles
- abs(seekend
);
1085 while(il
< seekstart
):
1086 prefhstart
= fp
.tell();
1087 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1088 if(len(preheaderdata
)==0):
1090 prefsize
= int(preheaderdata
[5], 16);
1091 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
1092 prefcs
= preheaderdata
[-2];
1093 if(prefcs
!=prenewfcs
and not skipchecksum
):
1094 VVerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1095 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1097 valid_archive
= False;
1098 invalid_archive
= True;
1099 prefhend
= fp
.tell() - 1;
1100 prefcontentstart
= fp
.tell();
1101 prefcontents
= BytesIO();
1102 pyhascontents
= False;
1104 prefcontents
.write(fp
.read(prefsize
));
1105 prefcontents
.seek(0, 0);
1106 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
1107 prefccs
= preheaderdata
[-1];
1108 pyhascontents
= True;
1109 if(prefccs
!=prenewfccs
and not skipchecksum
):
1110 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1111 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1116 countnum
= seekstart
;
1117 while(countnum
< seekend
):
1118 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': realidnum
}});
1119 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1120 if(len(HeaderOut
)==0):
1122 catlist
['ffilelist'][realidnum
].update(HeaderOut
);
1123 countnum
= countnum
+ 1;
1124 realidnum
= realidnum
+ 1;
1127 def ReadFileDataBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1128 formatspecs
= FormatSpecsListToDict(formatspecs
);
1129 if(isinstance(infile
, dict)):
1130 listcatfiles
= infile
;
1132 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1134 listcatfiles
= ReadFileDataBySizeWithContentToArray(infile
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1135 if(not listcatfiles
):
1137 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1138 lenlist
= len(listcatfiles
['ffilelist']);
1140 lcfx
= int(listcatfiles
['fnumfiles']);
1141 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1142 lcfx
= int(lenlist
);
1144 lcfx
= int(listcatfiles
['fnumfiles']);
1146 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1147 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1148 catarray
['filetoid'].update(filetoidarray
);
1149 catarray
['idtofile'].update(idtofilearray
);
1150 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1151 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1152 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1153 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1154 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1155 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1156 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1157 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1158 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1159 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1160 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1161 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1162 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1163 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1164 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1165 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1166 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1167 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1168 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1169 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1170 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1171 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1172 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1173 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1174 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1175 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1176 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1177 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1178 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1179 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1180 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1184 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1185 formatspecs
= FormatSpecsListToDict(formatspecs
);
1186 delimiter
= formatspecs
['format_delimiter'];
1190 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1193 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1194 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1195 if(not headercheck
and not skipchecksum
):
1196 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1197 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1199 catstring
= catheader
[0];
1200 catversion
= re
.findall(r
"([\d]+)", catstring
);
1201 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1202 fprenumfiles
= catheader
[1];
1203 fnumfiles
= int(fprenumfiles
, 16);
1204 fprechecksumtype
= catheader
[2];
1205 fprechecksum
= catheader
[3];
1207 if(seekstart
<0 and seekstart
>fnumfiles
):
1209 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1210 seekend
= fnumfiles
;
1211 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1212 seekend
= fnumfiles
- abs(seekend
);
1215 while(il
< seekstart
):
1216 prefhstart
= fp
.tell();
1217 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1218 if(len(preheaderdata
)==0):
1220 prefsize
= int(preheaderdata
[5], 16);
1221 prefcompression
= preheaderdata
[12];
1222 prefcsize
= int(preheaderdata
[13], 16);
1223 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
1224 prefcs
= preheaderdata
[-2];
1225 if(prefcs
!=prenewfcs
and not skipchecksum
):
1226 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1227 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1229 valid_archive
= False;
1230 invalid_archive
= True;
1231 prefhend
= fp
.tell() - 1;
1232 prefcontentstart
= fp
.tell();
1234 pyhascontents
= False;
1236 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
1237 prefcontents
= catfp
.read(prefsize
);
1239 prefcontents
= catfp
.read(prefcsize
);
1240 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
1241 prefccs
= preheaderdata
[-1];
1242 pyhascontents
= True;
1243 if(prefccs
!=prenewfccs
and not skipchecksum
):
1244 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1245 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1250 countnum
= seekstart
;
1251 while(countnum
< seekend
):
1252 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1253 if(len(HeaderOut
)==0):
1255 catlist
.append(HeaderOut
);
1256 countnum
= countnum
+ 1;
1257 realidnum
= realidnum
+ 1;
1260 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1261 formatspecs
= FormatSpecsListToDict(formatspecs
);
1262 delimiter
= formatspecs
['format_delimiter'];
1263 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1266 fp
= UncompressArchiveFile(fp
, formatspecs
);
1267 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1268 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1275 if(hasattr(sys
.stdin
, "buffer")):
1276 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1278 shutil
.copyfileobj(sys
.stdin
, fp
);
1280 fp
= UncompressArchiveFile(fp
, formatspecs
);
1284 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1285 fp
= download_file_from_internet_file(infile
);
1286 fp
= UncompressArchiveFile(fp
, formatspecs
);
1292 infile
= RemoveWindowsPath(infile
);
1293 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1294 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1296 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1297 if(not compresscheck
):
1298 fextname
= os
.path
.splitext(infile
)[1];
1299 if(fextname
==".gz"):
1300 compresscheck
= "gzip";
1301 elif(fextname
==".bz2"):
1302 compresscheck
= "bzip2";
1303 elif(fextname
==".zst"):
1304 compresscheck
= "zstd";
1305 elif(fextname
==".lz4" or fextname
==".clz4"):
1306 compresscheck
= "lz4";
1307 elif(fextname
==".lzo" or fextname
==".lzop"):
1308 compresscheck
= "lzo";
1309 elif(fextname
==".lzma" or fextname
==".xz"):
1310 compresscheck
= "lzma";
1313 if(not compresscheck
):
1315 fp
= UncompressFile(infile
, formatspecs
, "rb");
1316 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1318 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1319 formatspecs
= FormatSpecsListToDict(formatspecs
);
1320 delimiter
= formatspecs
['format_delimiter'];
1321 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1324 fp
= UncompressArchiveFile(fp
, formatspecs
);
1325 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1326 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1333 if(hasattr(sys
.stdin
, "buffer")):
1334 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1336 shutil
.copyfileobj(sys
.stdin
, fp
);
1338 fp
= UncompressArchiveFile(fp
, formatspecs
);
1342 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1343 fp
= download_file_from_internet_file(infile
);
1344 fp
= UncompressArchiveFile(fp
, formatspecs
);
1350 infile
= RemoveWindowsPath(infile
);
1351 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1352 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1354 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1355 if(not compresscheck
):
1356 fextname
= os
.path
.splitext(infile
)[1];
1357 if(fextname
==".gz"):
1358 compresscheck
= "gzip";
1359 elif(fextname
==".bz2"):
1360 compresscheck
= "bzip2";
1361 elif(fextname
==".zst"):
1362 compresscheck
= "zstd";
1363 elif(fextname
==".lz4" or fextname
==".clz4"):
1364 compresscheck
= "lz4";
1365 elif(fextname
==".lzo" or fextname
==".lzop"):
1366 compresscheck
= "lzo";
1367 elif(fextname
==".lzma" or fextname
==".xz"):
1368 compresscheck
= "lzma";
1371 if(not compresscheck
):
1373 fp
= UncompressFile(infile
, formatspecs
, "rb");
1374 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1376 def ReadInFileBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1377 formatspecs
= FormatSpecsListToDict(formatspecs
);
1378 if(isinstance(infile
, dict)):
1379 listcatfiles
= infile
;
1381 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1382 infile
= RemoveWindowsPath(infile
);
1383 listcatfiles
= ReadInFileBySizeWithContentToArray(infile
, listonly
, skipchecksum
, formatspecs
);
1384 if(not listcatfiles
):
1386 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1387 lenlist
= len(listcatfiles
['ffilelist']);
1389 lcfx
= int(listcatfiles
['fnumfiles']);
1390 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1391 lcfx
= int(lenlist
);
1393 lcfx
= int(listcatfiles
['fnumfiles']);
1395 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1396 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1397 catarray
['filetoid'].update(filetoidarray
);
1398 catarray
['idtofile'].update(idtofilearray
);
1399 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1400 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1401 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1402 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1403 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1404 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1405 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1406 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1407 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1408 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1409 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1410 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1411 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1412 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1413 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1414 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1415 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1416 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1417 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1418 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1419 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1420 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1421 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1422 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1423 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1424 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1425 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1426 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1427 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1428 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1429 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1433 def AppendNullByte(indata
, delimiter
=__file_format_dict__
['format_delimiter']):
1434 outdata
= str(indata
) + delimiter
;
1437 def AppendNullBytes(indata
=[], delimiter
=__file_format_dict__
['format_delimiter']):
1442 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1446 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1447 formatspecs
= FormatSpecsListToDict(formatspecs
);
1448 delimiter
= formatspecs
['format_delimiter'];
1449 catver
= formatspecs
['format_ver'];
1450 fileheaderver
= str(int(catver
.replace(".", "")));
1451 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
1452 fp
.write(fileheader
.encode('UTF-8'));
1453 fnumfiles
= format(int(numfiles
), 'x').lower();
1454 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
['format_delimiter']);
1455 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1456 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
1457 fp
.write(fnumfilesa
.encode('UTF-8'));
1460 os
.fsync(fp
.fileno());
1461 except io
.UnsupportedOperation
:
1463 except AttributeError:
1465 except OSError as e
:
1469 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1470 formatspecs
= FormatSpecsListToDict(formatspecs
);
1471 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1474 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_dict__
, returnfp
=False):
1475 formatspecs
= FormatSpecsListToDict(formatspecs
);
1476 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1477 if(os
.path
.exists(outfile
)):
1480 except OSError as e
:
1484 catfpfp
= BytesIO();
1485 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1487 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1490 fbasename
= os
.path
.splitext(outfile
)[0];
1491 fextname
= os
.path
.splitext(outfile
)[1];
1492 catfp
= CompressOpenFile(outfile
, True, compressionlevel
);
1493 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1494 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1495 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1498 os
.fsync(catfp
.fileno());
1499 except io
.UnsupportedOperation
:
1501 except AttributeError:
1503 except OSError as e
:
1507 if(hasattr(sys
.stdout
, "buffer")):
1508 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1510 shutil
.copyfileobj(catfp
, sys
.stdout
);
1511 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1512 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1514 upload_file_to_internet_file(catfp
, outfile
);
1522 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1523 formatspecs
= FormatSpecsListToDict(formatspecs
);
1524 extrafields
= format(len(extradata
), 'x').lower();
1525 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
1526 if(len(extradata
)>0):
1527 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1528 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1529 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1530 catoutlenhex
= format(catoutlen
, 'x').lower();
1531 catoutlist
= filevalues
;
1532 catoutlist
.insert(0, catoutlenhex
);
1533 catoutlist
.append(extrasizelen
);
1534 catoutlist
.append(extrafields
);
1535 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
1536 if(len(extradata
)>0):
1537 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1538 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
1539 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1540 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1541 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1542 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1543 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
1544 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1545 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1546 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1547 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
1548 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1549 fp
.write(catfileout
);
1552 os
.fsync(fp
.fileno());
1553 except io
.UnsupportedOperation
:
1555 except AttributeError:
1557 except OSError as e
:
1561 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1562 formatspecs
= FormatSpecsListToDict(formatspecs
);
1563 advancedlist
= formatspecs
['use_advanced_list'];
1564 altinode
= formatspecs
['use_alt_inode'];
1566 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1568 for line
in sys
.stdin
:
1569 infilelist
.append(line
.strip());
1570 infilelist
= list(filter(None, infilelist
));
1571 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1572 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1574 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1575 for line
in finfile
:
1576 infilelist
.append(line
.strip());
1577 infilelist
= list(filter(None, infilelist
));
1579 if(isinstance(infiles
, (list, tuple, ))):
1580 infilelist
= list(filter(None, infiles
));
1581 elif(isinstance(infiles
, (str, ))):
1582 infilelist
= list(filter(None, [infiles
]));
1584 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1586 GetDirList
= ListDir(infilelist
, followlink
, False);
1594 inodetocatinode
= {};
1595 numfiles
= int(len(GetDirList
));
1596 fnumfiles
= format(numfiles
, 'x').lower();
1597 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1598 for curfname
in GetDirList
:
1599 if(re
.findall("^[.|/]", curfname
)):
1602 fname
= "./"+curfname
;
1604 VerbosePrintOut(fname
);
1605 if(not followlink
or followlink
is None):
1606 fstatinfo
= os
.lstat(fname
);
1608 fstatinfo
= os
.stat(fname
);
1609 fpremode
= fstatinfo
.st_mode
;
1610 finode
= fstatinfo
.st_ino
;
1611 flinkcount
= fstatinfo
.st_nlink
;
1613 if(stat
.S_ISREG(fpremode
)):
1615 elif(stat
.S_ISLNK(fpremode
)):
1617 elif(stat
.S_ISCHR(fpremode
)):
1619 elif(stat
.S_ISBLK(fpremode
)):
1621 elif(stat
.S_ISDIR(fpremode
)):
1623 elif(stat
.S_ISFIFO(fpremode
)):
1625 elif(stat
.S_ISSOCK(fpremode
)):
1627 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1629 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1631 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1636 fcurfid
= format(int(curfid
), 'x').lower();
1637 if(not followlink
and finode
!=0):
1639 if(finode
in inodelist
):
1641 flinkname
= inodetofile
[finode
];
1643 fcurinode
= format(int(finode
), 'x').lower();
1645 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1646 if(finode
not in inodelist
):
1647 inodelist
.append(finode
);
1648 inodetofile
.update({finode
: fname
});
1649 inodetocatinode
.update({finode
: curinode
});
1651 fcurinode
= format(int(finode
), 'x').lower();
1653 fcurinode
= format(int(curinode
), 'x').lower();
1654 curinode
= curinode
+ 1;
1656 fcurinode
= format(int(curinode
), 'x').lower();
1657 curinode
= curinode
+ 1;
1658 curfid
= curfid
+ 1;
1660 flinkname
= os
.readlink(fname
);
1661 fdev
= fstatinfo
.st_dev
;
1662 getfdev
= GetDevMajorMinor(fdev
);
1663 fdev_minor
= getfdev
[0];
1664 fdev_major
= getfdev
[1];
1665 frdev
= fstatinfo
.st_dev
;
1666 if(hasattr(fstatinfo
, "st_rdev")):
1667 frdev
= fstatinfo
.st_rdev
;
1669 frdev
= fstatinfo
.st_dev
;
1670 getfrdev
= GetDevMajorMinor(frdev
);
1671 frdev_minor
= getfrdev
[0];
1672 frdev_major
= getfrdev
[1];
1673 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1674 fsize
= format(int("0"), 'x').lower();
1675 elif(ftype
==0 or ftype
==7):
1676 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1678 fsize
= format(int(fstatinfo
.st_size
)).lower();
1679 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1680 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1681 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1682 if(hasattr(fstatinfo
, "st_birthtime")):
1683 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1685 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1686 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1687 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1688 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1689 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1690 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1695 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1696 funame
= userinfo
.pw_name
;
1705 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1706 fgname
= groupinfo
.gr_name
;
1711 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1712 fdev_major
= format(int(fdev_major
), 'x').lower();
1713 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1714 frdev_major
= format(int(frdev_major
), 'x').lower();
1715 finode
= format(int(finode
), 'x').lower();
1716 flinkcount
= format(int(flinkcount
), 'x').lower();
1717 if(hasattr(fstatinfo
, "st_file_attributes")):
1718 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1720 fwinattributes
= format(int(0), 'x').lower();
1722 fcsize
= format(int(0), 'x').lower();
1723 fcontents
= BytesIO();
1725 if(ftype
==0 or ftype
==7):
1726 with
open(fname
, "rb") as fpc
:
1727 shutil
.copyfileobj(fpc
, fcontents
);
1728 if(not compresswholefile
):
1729 fcontents
.seek(0, 2);
1730 ucfsize
= fcontents
.tell();
1731 fcontents
.seek(0, 0);
1732 if(compression
=="auto"):
1733 ilsize
= len(compressionlistalt
);
1736 while(ilmin
< ilsize
):
1737 cfcontents
= BytesIO();
1738 shutil
.copyfileobj(fcontents
, cfcontents
);
1739 fcontents
.seek(0, 0);
1740 cfcontents
.seek(0, 0);
1741 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1743 cfcontents
.seek(0, 2);
1744 ilcsize
.append(cfcontents
.tell());
1748 ilcsize
.append(sys
.maxint
);
1749 except AttributeError:
1750 ilcsize
.append(sys
.maxsize
);
1752 ilcmin
= ilcsize
.index(min(ilcsize
));
1753 compression
= compressionlistalt
[ilcmin
];
1754 fcontents
.seek(0, 0);
1755 cfcontents
= BytesIO();
1756 shutil
.copyfileobj(fcontents
, cfcontents
);
1757 cfcontents
.seek(0, 0);
1758 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1759 cfcontents
.seek(0, 2);
1760 cfsize
= cfcontents
.tell();
1761 if(ucfsize
> cfsize
):
1762 fcsize
= format(int(cfsize
), 'x').lower();
1763 fcompression
= compression
;
1765 fcontents
= cfcontents
;
1766 if(followlink
and (ftype
==1 or ftype
==2)):
1767 flstatinfo
= os
.stat(flinkname
);
1768 with
open(flinkname
, "rb") as fpc
:
1769 shutil
.copyfileobj(fpc
, fcontents
);
1770 if(not compresswholefile
):
1771 fcontents
.seek(0, 2);
1772 ucfsize
= fcontents
.tell();
1773 fcontents
.seek(0, 0);
1774 if(compression
=="auto"):
1775 ilsize
= len(compressionlistalt
);
1778 while(ilmin
< ilsize
):
1779 cfcontents
= BytesIO();
1780 shutil
.copyfileobj(fcontents
, cfcontents
);
1781 fcontents
.seek(0, 0);
1782 cfcontents
.seek(0, 0);
1783 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1785 cfcontents
.seek(0, 2);
1786 ilcsize
.append(cfcontents
.tell());
1790 ilcsize
.append(sys
.maxint
);
1791 except AttributeError:
1792 ilcsize
.append(sys
.maxsize
);
1794 ilcmin
= ilcsize
.index(min(ilcsize
));
1795 compression
= compressionlistalt
[ilcmin
];
1796 fcontents
.seek(0, 0);
1797 cfcontents
= BytesIO();
1798 shutil
.copyfileobj(fcontents
, cfcontents
);
1799 cfcontents
.seek(0, 0);
1800 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1801 cfcontents
.seek(0, 2);
1802 cfsize
= cfcontents
.tell();
1803 if(ucfsize
> cfsize
):
1804 fcsize
= format(int(cfsize
), 'x').lower();
1805 fcompression
= compression
;
1807 fcontents
= cfcontents
;
1808 if(fcompression
=="none"):
1810 fcontents
.seek(0, 0);
1811 ftypehex
= format(ftype
, 'x').lower();
1812 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1813 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1815 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1819 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1820 formatspecs
= FormatSpecsListToDict(formatspecs
);
1822 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1823 GetDirList
= inlist
;
1831 inodetocatinode
= {};
1832 numfiles
= int(len(GetDirList
));
1833 fnumfiles
= format(numfiles
, 'x').lower();
1834 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1835 for curfname
in GetDirList
:
1836 ftype
= format(curfname
[0], 'x').lower();
1837 if(re
.findall("^[.|/]", curfname
[1])):
1838 fname
= curfname
[1];
1840 fname
= "./"+curfname
[1];
1841 fbasedir
= os
.path
.dirname(fname
);
1842 flinkname
= curfname
[2];
1843 fsize
= format(curfname
[3], 'x').lower();
1844 fatime
= format(curfname
[4], 'x').lower();
1845 fmtime
= format(curfname
[5], 'x').lower();
1846 fctime
= format(curfname
[6], 'x').lower();
1847 fbtime
= format(curfname
[7], 'x').lower();
1848 fmode
= format(curfname
[8], 'x').lower();
1849 fwinattributes
= format(curfname
[9], 'x').lower();
1850 fcompression
= curfname
[10];
1851 fcsize
= format(curfname
[11], 'x').lower();
1852 fuid
= format(curfname
[12], 'x').lower();
1853 funame
= curfname
[13];
1854 fgid
= format(curfname
[14], 'x').lower();
1855 fgname
= curfname
[15];
1856 fid
= format(curfname
[16], 'x').lower();
1857 finode
= format(curfname
[17], 'x').lower();
1858 flinkcount
= format(curfname
[18], 'x').lower();
1859 fdev_minor
= format(curfname
[19], 'x').lower();
1860 fdev_major
= format(curfname
[20], 'x').lower();
1861 frdev_minor
= format(curfname
[21], 'x').lower();
1862 frdev_major
= format(curfname
[22], 'x').lower();
1863 extradata
= curfname
[23];
1864 fchecksumtype
= curfname
[24];
1865 fcontents
= curfname
[25];
1866 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1867 fcontents
.seek(0, 0);
1868 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1870 fp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1873 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1874 formatspecs
= FormatSpecsListToDict(formatspecs
);
1875 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
1876 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1878 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1879 formatspecs
= FormatSpecsListToDict(formatspecs
);
1880 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1881 if(os
.path
.exists(outfile
)):
1884 except OSError as e
:
1888 catfpfp
= BytesIO();
1889 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1891 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1894 fbasename
= os
.path
.splitext(outfile
)[0];
1895 fextname
= os
.path
.splitext(outfile
)[1];
1896 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1897 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1898 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1899 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1902 os
.fsync(catfp
.fileno());
1903 except io
.UnsupportedOperation
:
1905 except AttributeError:
1907 except OSError as e
:
1911 if(hasattr(sys
.stdout
, "buffer")):
1912 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1914 shutil
.copyfileobj(catfp
, sys
.stdout
);
1915 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1916 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1918 upload_file_to_internet_file(catfp
, outfile
);
1926 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1927 formatspecs
= FormatSpecsListToDict(formatspecs
);
1928 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1929 if(os
.path
.exists(outfile
)):
1932 except OSError as e
:
1936 catfpfp
= BytesIO();
1937 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1939 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1942 fbasename
= os
.path
.splitext(outfile
)[0];
1943 fextname
= os
.path
.splitext(outfile
)[1];
1944 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1945 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1946 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1947 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1950 os
.fsync(catfp
.fileno());
1951 except io
.UnsupportedOperation
:
1953 except AttributeError:
1955 except OSError as e
:
1959 if(hasattr(sys
.stdout
, "buffer")):
1960 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1962 shutil
.copyfileobj(catfp
, sys
.stdout
);
1963 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1964 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1966 upload_file_to_internet_file(catfp
, outfile
);
1974 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1975 formatspecs
= FormatSpecsListToDict(formatspecs
);
1976 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
1977 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
1979 def PrintPermissionString(fchmode
, ftype
):
1980 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1982 for fmodval
in str(oct(fchmode
))[-3:]:
1983 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
1984 if(ftype
==0 or ftype
==7):
1985 permissionstr
= "-" + permissionstr
;
1987 permissionstr
= "h" + permissionstr
;
1989 permissionstr
= "l" + permissionstr
;
1991 permissionstr
= "c" + permissionstr
;
1993 permissionstr
= "b" + permissionstr
;
1995 permissionstr
= "d" + permissionstr
;
1997 permissionstr
= "f" + permissionstr
;
1999 permissionstr
= "D" + permissionstr
;
2001 permissionstr
= "p" + permissionstr
;
2003 permissionstr
= "w" + permissionstr
;
2005 permissionoutstr
= stat
.filemode(fchmode
);
2006 except AttributeError:
2007 permissionoutstr
= permissionstr
;
2009 permissionoutstr
= permissionstr
;
2010 return permissionoutstr
;
2012 def PrintPermissionStringAlt(fchmode
, ftype
):
2014 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2015 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2017 # Translate file mode into permission string
2018 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
2019 # Append file type indicator
2021 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2022 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2024 file_type
= type_indicators
.get(ftype
, '-');
2025 permissionstr
= file_type
+ permissionstr
;
2027 permissionoutstr
= stat
.filemode(fchmode
);
2028 except AttributeError:
2029 permissionoutstr
= permissionstr
;
2030 return permissionoutstr
;
2032 def CompressionSupport():
2033 compression_list
= [];
2036 compression_list
.append("gz");
2037 compression_list
.append("gzip");
2042 compression_list
.append("bz2");
2043 compression_list
.append("bzip2");
2048 compression_list
.append("lz4");
2053 compression_list
.append("lzo");
2054 compression_list
.append("lzop");
2059 compression_list
.append("zstd");
2060 compression_list
.append("zstandard");
2065 compression_list
.append("lzma");
2066 compression_list
.append("xz");
2069 return compression_list
;
2071 def CheckCompressionType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2072 formatspecs
= FormatSpecsListToDict(formatspecs
);
2073 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2077 catfp
= open(infile
, "rb");
2078 except FileNotFoundError
:
2081 prefp
= catfp
.read(2);
2083 if(prefp
==binascii
.unhexlify("1f8b")):
2086 prefp
= catfp
.read(3);
2087 if(prefp
==binascii
.unhexlify("425a68")):
2089 if(prefp
==binascii
.unhexlify("5d0000")):
2092 prefp
= catfp
.read(4);
2093 if(prefp
==binascii
.unhexlify("28b52ffd")):
2095 if(prefp
==binascii
.unhexlify("04224d18")):
2097 if(prefp
==binascii
.unhexlify("504B0304")):
2098 filetype
= "zipfile";
2100 prefp
= catfp
.read(5);
2101 if(prefp
==binascii
.unhexlify("7573746172")):
2102 filetype
= "tarfile";
2104 prefp
= catfp
.read(6);
2105 if(prefp
==binascii
.unhexlify("fd377a585a00")):
2107 if(prefp
==binascii
.unhexlify("377abcaf271c")):
2108 filetype
= "7zipfile";
2110 prefp
= catfp
.read(7);
2111 if(prefp
==binascii
.unhexlify("526172211a0700")):
2112 filetype
= "rarfile";
2113 if(prefp
==binascii
.unhexlify("43617446696c65")):
2114 filetype
= "catfile";
2116 prefp
= catfp
.read(8);
2117 if(prefp
==binascii
.unhexlify("526172211a070100")):
2118 filetype
= "rarfile";
2120 prefp
= catfp
.read(formatspecs
['format_len']);
2121 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2122 filetype
= formatspecs
['format_lower'];
2124 prefp
= catfp
.read(9);
2125 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
2128 prefp
= catfp
.read(10);
2129 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2130 filetype
= "tarfile";
2132 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
2133 if(is_tarfile(catfp
)):
2134 filetype
= "tarfile";
2136 if(is_tarfile(catfp
)):
2137 filetype
= "tarfile";
2138 elif(zipfile
.is_zipfile(catfp
)):
2139 filetype
= "zipfile";
2140 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
2141 filetype
= "rarile";
2149 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_dict__
, closefp
=True):
2150 formatspecs
= FormatSpecsListToDict(formatspecs
);
2152 instringsfile
= BytesIO(instring
);
2154 instringsfile
= BytesIO(instring
.encode("UTF-8"));
2155 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
2157 def GetCompressionMimeType(infile
, formatspecs
=__file_format_dict__
):
2158 formatspecs
= FormatSpecsListToDict(formatspecs
);
2159 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2160 if(compresscheck
=="gzip" or compresscheck
=="gz"):
2161 return archivefile_gzip_mimetype
;
2162 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
2163 return archivefile_bzip2_mimetype
;
2164 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
2165 return archivefile_zstandard_mimetype
;
2166 if(compresscheck
=="lz4"):
2167 return archivefile_lz4_mimetype
;
2168 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2169 return archivefile_lzop_mimetype
;
2170 if(compresscheck
=="lzma"):
2171 return archivefile_lzma_mimetype
;
2172 if(compresscheck
=="xz"):
2173 return archivefile_xz_mimetype
;
2174 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
['format_lower']):
2175 return archivefile_cat_mimetype
;
2176 if(not compresscheck
):
2180 def UncompressArchiveFile(fp
, formatspecs
=__file_format_dict__
):
2181 formatspecs
= FormatSpecsListToDict(formatspecs
);
2182 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2184 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2185 if(compresscheck
=="gzip"):
2190 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
2191 if(compresscheck
=="bzip2"):
2197 catfp
.write(bz2
.decompress(fp
.read()));
2198 if(compresscheck
=="zstd"):
2204 catfp
.write(zstandard
.decompress(fp
.read()));
2205 if(compresscheck
=="lz4"):
2211 catfp
.write(lz4
.frame
.decompress(fp
.read()));
2212 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2218 catfp
.write(lzo
.decompress(fp
.read()));
2219 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2225 catfp
.write(lzma
.decompress(fp
.read()));
2226 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2228 if(not compresscheck
):
2234 with fp
as fpcontent
:
2236 catfp
.write(lzma
.decompress(fp
.read()));
2237 except lzma
.LZMAError
:
2239 if(compresscheck
!="catfile" or compresscheck
!=formatspecs
['format_lower']):
2243 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
2245 def UncompressFile(infile
, formatspecs
=__file_format_dict__
, mode
="rb"):
2246 formatspecs
= FormatSpecsListToDict(formatspecs
);
2247 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2248 if(sys
.version_info
[0]==2 and compresscheck
):
2254 if(compresscheck
=="gzip"):
2260 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
2261 except (ValueError, TypeError) as e
:
2262 filefp
= gzip
.open(infile
, mode
);
2263 if(compresscheck
=="bzip2"):
2269 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
2270 except (ValueError, TypeError) as e
:
2271 filefp
= bz2
.open(infile
, mode
);
2272 if(compresscheck
=="zstd"):
2278 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
2279 except (ValueError, TypeError) as e
:
2280 filefp
= zstandard
.open(infile
, mode
);
2281 if(compresscheck
=="lz4"):
2287 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
2288 except (ValueError, TypeError) as e
:
2289 filefp
= lz4
.frame
.open(infile
, mode
);
2290 if(compresscheck
=="lzo"):
2296 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
2297 except (ValueError, TypeError) as e
:
2298 filefp
= lzo
.open(infile
, mode
);
2299 if(compresscheck
=="lzma"):
2305 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
2306 except (ValueError, TypeError) as e
:
2307 filefp
= lzma
.open(infile
, mode
);
2308 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2310 filefp
= open(infile
, mode
, encoding
="UTF-8");
2311 except (ValueError, TypeError) as e
:
2312 filefp
= open(infile
, mode
);
2313 if(not compresscheck
):
2315 filefp
= open(infile
, mode
, encoding
="UTF-8");
2316 except (ValueError, TypeError) as e
:
2317 filefp
= open(infile
, mode
);
2318 except FileNotFoundError
:
2322 def UncompressString(infile
):
2323 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
2324 if(compresscheck
=="gzip"):
2329 fileuz
= gzip
.decompress(infile
);
2330 if(compresscheck
=="bzip2"):
2335 fileuz
= bz2
.decompress(infile
);
2336 if(compresscheck
=="zstd"):
2341 fileuz
= zstandard
.decompress(infile
);
2342 if(compresscheck
=="lz4"):
2347 fileuz
= lz4
.frame
.decompress(infile
);
2348 if(compresscheck
=="lzo"):
2353 fileuz
= lzo
.decompress(infile
);
2354 if(compresscheck
=="lzma"):
2359 fileuz
= lzma
.decompress(infile
);
2360 if(not compresscheck
):
2362 if(hasattr(fileuz
, 'decode')):
2363 fileuz
= fileuz
.decode("UTF-8");
2366 def UncompressStringAlt(infile
):
2367 filefp
= StringIO();
2368 outstring
= UncompressString(infile
);
2369 filefp
.write(outstring
);
2373 def CheckCompressionSubType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2374 formatspecs
= FormatSpecsListToDict(formatspecs
);
2375 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2376 if(not compresscheck
):
2377 fextname
= os
.path
.splitext(infile
)[1];
2378 if(fextname
==".gz"):
2379 compresscheck
= "gzip";
2380 elif(fextname
==".bz2"):
2381 compresscheck
= "bzip2";
2382 elif(fextname
==".zst"):
2383 compresscheck
= "zstd";
2384 elif(fextname
==".lz4"):
2385 compresscheck
= "lz4";
2386 elif(fextname
==".lzo" or fextname
==".lzop"):
2387 compresscheck
= "lzo";
2388 elif(fextname
==".lzma" or fextname
==".xz"):
2389 compresscheck
= "lzma";
2392 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2393 if(is_tarfile(infile
)):
2394 filetype
= "tarfile";
2395 if(not compresscheck
):
2396 if(is_tarfile(infile
)):
2398 elif(zipfile
.is_zipfile(infile
)):
2400 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2405 if(compresscheck
=="catfile"):
2407 if(compresscheck
==formatspecs
['format_lower']):
2408 return formatspecs
['format_lower'];
2409 if(compresscheck
=="tarfile"):
2411 if(compresscheck
=="zipfile"):
2413 if(rarfile_support
and compresscheck
=="rarfile"):
2415 if(py7zr_support
and compresscheck
=="7zipfile"):
2417 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2418 catfp
= UncompressArchiveFile(infile
, formatspecs
['format_lower']);
2421 if(compresscheck
=="gzip"):
2426 catfp
= gzip
.GzipFile(infile
, "rb");
2427 if(compresscheck
=="bzip2"):
2432 catfp
= bz2
.BZ2File(infile
, "rb");
2433 if(compresscheck
=="lz4"):
2438 catfp
= lz4
.frame
.open(infile
, "rb");
2439 if(compresscheck
=="zstd"):
2444 catfp
= zstandard
.open(infile
, "rb");
2445 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2450 catfp
= lzma
.open(infile
, "rb");
2451 except FileNotFoundError
:
2454 prefp
= catfp
.read(5);
2455 if(prefp
==binascii
.unhexlify("7573746172")):
2456 filetype
= "tarfile";
2458 prefp
= catfp
.read(7);
2459 if(prefp
==binascii
.unhexlify("43617446696c65")):
2460 filetype
= "catfile";
2462 prefp
= catfp
.read(formatspecs
['format_len']);
2463 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2464 filetype
= formatspecs
['format_lower'];
2466 prefp
= catfp
.read(10);
2467 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2468 filetype
= "tarfile";
2474 def GZipCompress(data
, compresslevel
=9):
2479 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2481 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2485 catfp
= open(tmpfp
.name
, "rb");
2486 except FileNotFoundError
:
2488 catdata
= catfp
.read();
2492 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_dict__
):
2493 formatspecs
= FormatSpecsListToDict(formatspecs
);
2494 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2497 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2498 compression
= "auto";
2499 if(compression
not in compressionlist
and compression
is None):
2500 compression
= "auto";
2501 if(compression
=="gzip"):
2507 if(compressionlevel
is None):
2508 compressionlevel
= 9;
2510 compressionlevel
= int(compressionlevel
);
2511 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2512 if(compression
=="bzip2"):
2518 if(compressionlevel
is None):
2519 compressionlevel
= 9;
2521 compressionlevel
= int(compressionlevel
);
2522 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2523 if(compression
=="lz4"):
2529 if(compressionlevel
is None):
2530 compressionlevel
= 9;
2532 compressionlevel
= int(compressionlevel
);
2533 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2534 if(compression
=="lzo" or compression
=="lzop"):
2540 if(compressionlevel
is None):
2541 compressionlevel
= 9;
2543 compressionlevel
= int(compressionlevel
);
2544 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2545 if(compression
=="zstd"):
2551 if(compressionlevel
is None):
2552 compressionlevel
= 10;
2554 compressionlevel
= int(compressionlevel
);
2555 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2556 if(compression
=="lzma"):
2562 if(compressionlevel
is None):
2563 compressionlevel
= 9;
2565 compressionlevel
= int(compressionlevel
);
2566 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2567 if(compression
=="xz"):
2573 if(compressionlevel
is None):
2574 compressionlevel
= 9;
2576 compressionlevel
= int(compressionlevel
);
2577 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2578 if(compression
=="auto" or compression
is None):
2583 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2585 def CompressOpenFile(outfile
, compressionenable
=True, compressionlevel
=None):
2586 if(outfile
is None):
2588 fbasename
= os
.path
.splitext(outfile
)[0];
2589 fextname
= os
.path
.splitext(outfile
)[1];
2590 if(compressionlevel
is None and fextname
!=".zst"):
2591 compressionlevel
= 9;
2592 elif(compressionlevel
is None and fextname
==".zst"):
2593 compressionlevel
= 10;
2595 compressionlevel
= int(compressionlevel
);
2596 if(sys
.version_info
[0]==2):
2601 if(fextname
not in outextlistwd
or not compressionenable
):
2603 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2604 except (ValueError, TypeError) as e
:
2605 outfp
= open(outfile
, "wb");
2606 elif(fextname
==".gz"):
2612 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2613 except (ValueError, TypeError) as e
:
2614 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2615 elif(fextname
==".bz2"):
2621 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2622 except (ValueError, TypeError) as e
:
2623 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2624 elif(fextname
==".zst"):
2630 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2631 except (ValueError, TypeError) as e
:
2632 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2633 elif(fextname
==".xz"):
2639 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2640 except (ValueError, TypeError) as e
:
2641 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2642 elif(fextname
==".lz4"):
2648 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2649 except (ValueError, TypeError) as e
:
2650 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2651 elif(fextname
==".lzo"):
2657 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2658 except (ValueError, TypeError) as e
:
2659 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2660 elif(fextname
==".lzma"):
2666 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2667 except (ValueError, TypeError) as e
:
2668 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2669 except FileNotFoundError
:
2673 def GetDevMajorMinor(fdev
):
2675 if(hasattr(os
, "minor")):
2676 retdev
.append(os
.minor(fdev
));
2679 if(hasattr(os
, "major")):
2680 retdev
.append(os
.major(fdev
));
2685 def CheckSumSupport(checkfor
, guaranteed
=True):
2688 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2689 except AttributeError:
2690 hash_list
= sorted(list(hashlib
.algorithms
));
2693 hash_list
= sorted(list(hashlib
.algorithms_available
));
2694 except AttributeError:
2695 hash_list
= sorted(list(hashlib
.algorithms
));
2696 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2697 if(checkfor
in checklistout
):
2702 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2705 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2706 except AttributeError:
2707 hash_list
= sorted(list(hashlib
.algorithms
));
2710 hash_list
= sorted(list(hashlib
.algorithms_available
));
2711 except AttributeError:
2712 hash_list
= sorted(list(hashlib
.algorithms
));
2713 checklistout
= hash_list
;
2714 if(checkfor
in checklistout
):
2719 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2720 formatspecs
= FormatSpecsListToDict(formatspecs
);
2721 advancedlist
= formatspecs
['use_advanced_list'];
2722 altinode
= formatspecs
['use_alt_inode'];
2723 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2724 outfile
= RemoveWindowsPath(outfile
);
2725 checksumtype
= checksumtype
.lower();
2726 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2727 checksumtype
="crc32";
2728 if(checksumtype
=="none"):
2730 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2731 compression
= "auto";
2732 if(compression
not in compressionlist
and compression
is None):
2733 compression
= "auto";
2735 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2736 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2737 if(os
.path
.exists(outfile
)):
2740 except OSError as e
:
2745 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2747 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2750 fbasename
= os
.path
.splitext(outfile
)[0];
2751 fextname
= os
.path
.splitext(outfile
)[1];
2752 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2753 catver
= formatspecs
['format_ver'];
2754 fileheaderver
= str(int(catver
.replace(".", "")));
2757 for line
in sys
.stdin
:
2758 infilelist
.append(line
.strip());
2759 infilelist
= list(filter(None, infilelist
));
2760 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2761 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2763 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2764 for line
in finfile
:
2765 infilelist
.append(line
.strip());
2766 infilelist
= list(filter(None, infilelist
));
2768 if(isinstance(infiles
, (list, tuple, ))):
2769 infilelist
= list(filter(None, infiles
));
2770 elif(isinstance(infiles
, (str, ))):
2771 infilelist
= list(filter(None, [infiles
]));
2773 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2775 GetDirList
= ListDir(infilelist
, followlink
, False);
2783 inodetocatinode
= {};
2784 numfiles
= int(len(GetDirList
));
2785 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2786 for curfname
in GetDirList
:
2787 if(re
.findall("^[.|/]", curfname
)):
2790 fname
= "./"+curfname
;
2792 VerbosePrintOut(fname
);
2793 if(not followlink
or followlink
is None):
2794 fstatinfo
= os
.lstat(fname
);
2796 fstatinfo
= os
.stat(fname
);
2797 fpremode
= fstatinfo
.st_mode
;
2798 finode
= fstatinfo
.st_ino
;
2799 flinkcount
= fstatinfo
.st_nlink
;
2801 if(stat
.S_ISREG(fpremode
)):
2803 elif(stat
.S_ISLNK(fpremode
)):
2805 elif(stat
.S_ISCHR(fpremode
)):
2807 elif(stat
.S_ISBLK(fpremode
)):
2809 elif(stat
.S_ISDIR(fpremode
)):
2811 elif(stat
.S_ISFIFO(fpremode
)):
2813 elif(stat
.S_ISSOCK(fpremode
)):
2815 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2817 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2819 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2824 fcurfid
= format(int(curfid
), 'x').lower();
2825 if(not followlink
and finode
!=0):
2827 if(finode
in inodelist
):
2829 flinkname
= inodetofile
[finode
];
2831 fcurinode
= format(int(finode
), 'x').lower();
2833 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2834 if(finode
not in inodelist
):
2835 inodelist
.append(finode
);
2836 inodetofile
.update({finode
: fname
});
2837 inodetocatinode
.update({finode
: curinode
});
2839 fcurinode
= format(int(finode
), 'x').lower();
2841 fcurinode
= format(int(curinode
), 'x').lower();
2842 curinode
= curinode
+ 1;
2844 fcurinode
= format(int(curinode
), 'x').lower();
2845 curinode
= curinode
+ 1;
2846 curfid
= curfid
+ 1;
2848 flinkname
= os
.readlink(fname
);
2849 fdev
= fstatinfo
.st_dev
;
2850 getfdev
= GetDevMajorMinor(fdev
);
2851 fdev_minor
= getfdev
[0];
2852 fdev_major
= getfdev
[1];
2853 frdev
= fstatinfo
.st_dev
;
2854 if(hasattr(fstatinfo
, "st_rdev")):
2855 frdev
= fstatinfo
.st_rdev
;
2857 frdev
= fstatinfo
.st_dev
;
2858 getfrdev
= GetDevMajorMinor(frdev
);
2859 frdev_minor
= getfrdev
[0];
2860 frdev_major
= getfrdev
[1];
2861 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2862 fsize
= format(int("0"), 'x').lower();
2863 elif(ftype
==0 or ftype
==7):
2864 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2866 fsize
= format(int(fstatinfo
.st_size
)).lower();
2867 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2868 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2869 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2870 if(hasattr(fstatinfo
, "st_birthtime")):
2871 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2873 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2874 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2875 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2876 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2877 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2878 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2883 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2884 funame
= userinfo
.pw_name
;
2893 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2894 fgname
= groupinfo
.gr_name
;
2899 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2900 fdev_major
= format(int(fdev_major
), 'x').lower();
2901 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2902 frdev_major
= format(int(frdev_major
), 'x').lower();
2903 finode
= format(int(finode
), 'x').lower();
2904 flinkcount
= format(int(flinkcount
), 'x').lower();
2905 if(hasattr(fstatinfo
, "st_file_attributes")):
2906 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2908 fwinattributes
= format(int(0), 'x').lower();
2910 fcsize
= format(int(0), 'x').lower();
2911 fcontents
= BytesIO();
2912 if(ftype
==0 or ftype
==7):
2913 with
open(fname
, "rb") as fpc
:
2914 shutil
.copyfileobj(fpc
, fcontents
);
2915 if(not compresswholefile
):
2916 fcontents
.seek(0, 2);
2917 ucfsize
= fcontents
.tell();
2918 fcontents
.seek(0, 0);
2919 if(compression
=="auto"):
2920 ilsize
= len(compressionlistalt
);
2923 while(ilmin
< ilsize
):
2924 cfcontents
= BytesIO();
2925 shutil
.copyfileobj(fcontents
, cfcontents
);
2926 fcontents
.seek(0, 0);
2927 cfcontents
.seek(0, 0);
2928 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2930 cfcontents
.seek(0, 2);
2931 ilcsize
.append(cfcontents
.tell());
2935 ilcsize
.append(sys
.maxint
);
2936 except AttributeError:
2937 ilcsize
.append(sys
.maxsize
);
2939 ilcmin
= ilcsize
.index(min(ilcsize
));
2940 compression
= compressionlistalt
[ilcmin
];
2941 fcontents
.seek(0, 0);
2942 cfcontents
= BytesIO();
2943 shutil
.copyfileobj(fcontents
, cfcontents
);
2944 cfcontents
.seek(0, 0);
2945 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2946 cfcontents
.seek(0, 2);
2947 cfsize
= cfcontents
.tell();
2948 if(ucfsize
> cfsize
):
2949 fcsize
= format(int(cfsize
), 'x').lower();
2950 fcompression
= compression
;
2952 fcontents
= cfcontents
;
2953 if(fcompression
=="none"):
2955 if(followlink
and (ftype
==1 or ftype
==2)):
2956 flstatinfo
= os
.stat(flinkname
);
2957 with
open(flinkname
, "rb") as fpc
:
2958 shutil
.copyfileobj(fpc
, fcontents
);
2959 if(not compresswholefile
):
2960 fcontents
.seek(0, 2);
2961 ucfsize
= fcontents
.tell();
2962 fcontents
.seek(0, 0);
2963 if(compression
=="auto"):
2964 ilsize
= len(compressionlistalt
);
2967 while(ilmin
< ilsize
):
2968 cfcontents
= BytesIO();
2969 shutil
.copyfileobj(fcontents
, cfcontents
);
2970 fcontents
.seek(0, 0);
2971 cfcontents
.seek(0, 0);
2972 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2974 cfcontents
.seek(0, 2);
2975 ilcsize
.append(cfcontents
.tell());
2979 ilcsize
.append(sys
.maxint
);
2980 except AttributeError:
2981 ilcsize
.append(sys
.maxsize
);
2983 ilcmin
= ilcsize
.index(min(ilcsize
));
2984 compression
= compressionlistalt
[ilcmin
];
2985 fcontents
.seek(0, 0);
2986 cfcontents
= BytesIO();
2987 shutil
.copyfileobj(fcontents
, cfcontents
);
2988 cfcontents
.seek(0, 0);
2989 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2990 cfcontents
.seek(0, 2);
2991 cfsize
= cfcontents
.tell();
2992 if(ucfsize
> cfsize
):
2993 fcsize
= format(int(cfsize
), 'x').lower();
2994 fcompression
= compression
;
2996 fcontents
= cfcontents
;
2997 fcontents
.seek(0, 0);
2998 ftypehex
= format(ftype
, 'x').lower();
2999 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3000 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3003 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3004 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3005 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3008 os
.fsync(catfp
.fileno());
3009 except io
.UnsupportedOperation
:
3011 except AttributeError:
3013 except OSError as e
:
3017 if(hasattr(sys
.stdout
, "buffer")):
3018 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3020 shutil
.copyfileobj(catfp
, sys
.stdout
);
3021 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3022 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3024 upload_file_to_internet_file(catfp
, outfile
);
3032 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
3034 if(hasattr(shutil
, "register_archive_format")):
3035 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
3036 return PackArchiveFile(source_dir
, archive_name
, False, "auto", True, None, False, "crc32", [], __file_format_dict__
['format_delimiter'], False, False);
3037 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
3039 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3040 formatspecs
= FormatSpecsListToDict(formatspecs
);
3041 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3043 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
3045 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3046 formatspecs
= FormatSpecsListToDict(formatspecs
);
3047 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3048 outfile
= RemoveWindowsPath(outfile
);
3049 checksumtype
= checksumtype
.lower();
3050 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3051 checksumtype
="crc32";
3052 if(checksumtype
=="none"):
3054 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3055 compression
= "auto";
3056 if(compression
not in compressionlist
and compression
is None):
3057 compression
= "auto";
3059 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3060 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3061 if(os
.path
.exists(outfile
)):
3064 except OSError as e
:
3069 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3071 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3074 fbasename
= os
.path
.splitext(outfile
)[0];
3075 fextname
= os
.path
.splitext(outfile
)[1];
3076 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3077 catver
= formatspecs
['format_ver'];
3078 fileheaderver
= str(int(catver
.replace(".", "")));
3084 inodetocatinode
= {};
3087 if(hasattr(sys
.stdin
, "buffer")):
3088 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3090 shutil
.copyfileobj(sys
.stdin
, infile
);
3095 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3096 infile
= download_file_from_internet_file(infile
);
3101 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3103 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
3105 if(not tarfile
.is_tarfile(infile
)):
3107 except AttributeError:
3108 if(not is_tarfile(infile
)):
3113 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3114 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
3116 tarfp
= tarfile
.open(infile
, "r");
3117 except FileNotFoundError
:
3119 numfiles
= int(len(tarfp
.getmembers()));
3120 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3121 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
3122 if(re
.findall("^[.|/]", member
.name
)):
3123 fname
= member
.name
;
3125 fname
= "./"+member
.name
;
3127 VerbosePrintOut(fname
);
3128 fpremode
= member
.mode
;
3129 ffullmode
= member
.mode
;
3133 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3135 elif(member
.isdev()):
3136 ffullmode
= member
.mode
;
3138 elif(member
.islnk()):
3139 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3141 elif(member
.issym()):
3142 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
3144 elif(member
.ischr()):
3145 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
3147 elif(member
.isblk()):
3148 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
3150 elif(member
.isdir()):
3151 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
3153 elif(member
.isfifo()):
3154 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
3156 elif(member
.issparse()):
3157 ffullmode
= member
.mode
;
3160 ffullmode
= member
.mode
;
3163 fcurfid
= format(int(curfid
), 'x').lower();
3164 fcurinode
= format(int(curfid
), 'x').lower();
3165 curfid
= curfid
+ 1;
3167 flinkname
= member
.linkname
;
3168 fdev_minor
= format(int(member
.devminor
), 'x').lower();
3169 fdev_major
= format(int(member
.devmajor
), 'x').lower();
3170 frdev_minor
= format(int(member
.devminor
), 'x').lower();
3171 frdev_major
= format(int(member
.devmajor
), 'x').lower();
3172 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
3173 fsize
= format(int("0"), 'x').lower();
3174 elif(ftype
==0 or ftype
==7):
3175 fsize
= format(int(member
.size
), 'x').lower();
3177 fsize
= format(int(member
.size
), 'x').lower();
3178 fatime
= format(int(member
.mtime
), 'x').lower();
3179 fmtime
= format(int(member
.mtime
), 'x').lower();
3180 fctime
= format(int(member
.mtime
), 'x').lower();
3181 fbtime
= format(int(member
.mtime
), 'x').lower();
3182 fmode
= format(int(ffullmode
), 'x').lower();
3183 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
3184 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
3185 fuid
= format(int(member
.uid
), 'x').lower();
3186 fgid
= format(int(member
.gid
), 'x').lower();
3187 funame
= member
.uname
;
3188 fgname
= member
.gname
;
3189 flinkcount
= format(int(flinkcount
), 'x').lower();
3190 fwinattributes
= format(int(0), 'x').lower();
3192 fcsize
= format(int(0), 'x').lower();
3193 fcontents
= BytesIO();
3194 if(ftype
==0 or ftype
==7):
3195 with tarfp
.extractfile(member
) as fpc
:
3196 shutil
.copyfileobj(fpc
, fcontents
);
3197 if(not compresswholefile
):
3198 fcontents
.seek(0, 2);
3199 ucfsize
= fcontents
.tell();
3200 fcontents
.seek(0, 0);
3201 if(compression
=="auto"):
3202 ilsize
= len(compressionlistalt
);
3205 while(ilmin
< ilsize
):
3206 cfcontents
= BytesIO();
3207 shutil
.copyfileobj(fcontents
, cfcontents
);
3208 fcontents
.seek(0, 0);
3209 cfcontents
.seek(0, 0);
3210 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3212 cfcontents
.seek(0, 2);
3213 ilcsize
.append(cfcontents
.tell());
3217 ilcsize
.append(sys
.maxint
);
3218 except AttributeError:
3219 ilcsize
.append(sys
.maxsize
);
3221 ilcmin
= ilcsize
.index(min(ilcsize
));
3222 compression
= compressionlistalt
[ilcmin
];
3223 fcontents
.seek(0, 0);
3224 cfcontents
= BytesIO();
3225 shutil
.copyfileobj(fcontents
, cfcontents
);
3226 cfcontents
.seek(0, 0);
3227 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3228 cfcontents
.seek(0, 2);
3229 cfsize
= cfcontents
.tell();
3230 if(ucfsize
> cfsize
):
3231 fcsize
= format(int(cfsize
), 'x').lower();
3232 fcompression
= compression
;
3234 fcontents
= cfcontents
;
3235 if(fcompression
=="none"):
3237 fcontents
.seek(0, 0);
3238 ftypehex
= format(ftype
, 'x').lower();
3239 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3240 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3243 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3244 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3245 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3248 os
.fsync(catfp
.fileno());
3249 except io
.UnsupportedOperation
:
3251 except AttributeError:
3253 except OSError as e
:
3257 if(hasattr(sys
.stdout
, "buffer")):
3258 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3260 shutil
.copyfileobj(catfp
, sys
.stdout
);
3261 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3262 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3264 upload_file_to_internet_file(catfp
, outfile
);
3272 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
3274 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3275 formatspecs
= FormatSpecsListToDict(formatspecs
);
3276 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3277 outfile
= RemoveWindowsPath(outfile
);
3278 checksumtype
= checksumtype
.lower();
3279 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3280 checksumtype
="crc32";
3281 if(checksumtype
=="none"):
3283 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3284 compression
= "auto";
3285 if(compression
not in compressionlist
and compression
is None):
3286 compression
= "auto";
3288 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3289 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3290 if(os
.path
.exists(outfile
)):
3293 except OSError as e
:
3298 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3300 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3303 fbasename
= os
.path
.splitext(outfile
)[0];
3304 fextname
= os
.path
.splitext(outfile
)[1];
3305 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3306 catver
= formatspecs
['format_ver'];
3307 fileheaderver
= str(int(catver
.replace(".", "")));
3313 inodetocatinode
= {};
3316 if(hasattr(sys
.stdin
, "buffer")):
3317 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3319 shutil
.copyfileobj(sys
.stdin
, infile
);
3324 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3325 infile
= download_file_from_internet_file(infile
);
3330 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3334 if(not zipfile
.is_zipfile(infile
)):
3337 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
3338 except FileNotFoundError
:
3340 ziptest
= zipfp
.testzip();
3342 VerbosePrintOut("Bad file found!");
3343 numfiles
= int(len(zipfp
.infolist()));
3344 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3345 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
3346 if(re
.findall("^[.|/]", member
.filename
)):
3347 fname
= member
.filename
;
3349 fname
= "./"+member
.filename
;
3350 zipinfo
= zipfp
.getinfo(member
.filename
);
3352 VerbosePrintOut(fname
);
3353 if(not member
.is_dir()):
3354 fpremode
= int(stat
.S_IFREG
+ 438);
3355 elif(member
.is_dir()):
3356 fpremode
= int(stat
.S_IFDIR
+ 511);
3359 if(not member
.is_dir()):
3361 elif(member
.is_dir()):
3364 fcurfid
= format(int(curfid
), 'x').lower();
3365 fcurinode
= format(int(curfid
), 'x').lower();
3366 curfid
= curfid
+ 1;
3367 fdev_minor
= format(int(0), 'x').lower();
3368 fdev_major
= format(int(0), 'x').lower();
3369 frdev_minor
= format(int(0), 'x').lower();
3370 frdev_major
= format(int(0), 'x').lower();
3372 fsize
= format(int("0"), 'x').lower();
3374 fsize
= format(int(member
.file_size
), 'x').lower();
3376 fsize
= format(int(member
.file_size
), 'x').lower();
3377 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3378 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3379 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3380 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3381 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
3382 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
3383 if(not member
.is_dir()):
3384 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3385 fchmode
= stat
.S_IMODE(fmode
);
3386 ftypemod
= stat
.S_IFMT(fmode
);
3387 elif(member
.is_dir()):
3388 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3389 fchmode
= stat
.S_IMODE(fmode
);
3390 ftypemod
= stat
.S_IFMT(fmode
);
3391 elif(zipinfo
.create_system
==3):
3392 fwinattributes
= format(int(0), 'x').lower();
3393 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
3394 fchmode
= stat
.S_IMODE(fmode
);
3395 ftypemod
= stat
.S_IFMT(fmode
);
3397 fwinattributes
= format(int(0), 'x').lower();
3398 if(not member
.is_dir()):
3399 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3400 fchmode
= stat
.S_IMODE(fmode
);
3401 ftypemod
= stat
.S_IFMT(fmode
);
3402 elif(member
.is_dir()):
3403 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3404 fchmode
= stat
.S_IMODE(fmode
);
3405 ftypemod
= stat
.S_IFMT(fmode
);
3407 fcsize
= format(int(0), 'x').lower();
3409 fuid
= format(int(os
.getuid()), 'x').lower();
3410 except AttributeError:
3411 fuid
= format(int(0), 'x').lower();
3413 fuid
= format(int(0), 'x').lower();
3415 fgid
= format(int(os
.getgid()), 'x').lower();
3416 except AttributeError:
3417 fgid
= format(int(0), 'x').lower();
3419 fgid
= format(int(0), 'x').lower();
3423 userinfo
= pwd
.getpwuid(os
.getuid());
3424 funame
= userinfo
.pw_name
;
3427 except AttributeError:
3435 groupinfo
= grp
.getgrgid(os
.getgid());
3436 fgname
= groupinfo
.gr_name
;
3439 except AttributeError:
3443 fcontents
= BytesIO();
3445 fcontents
.write(zipfp
.read(member
.filename
));
3446 if(not compresswholefile
):
3447 fcontents
.seek(0, 2);
3448 ucfsize
= fcontents
.tell();
3449 fcontents
.seek(0, 0);
3450 if(compression
=="auto"):
3451 ilsize
= len(compressionlistalt
);
3454 while(ilmin
< ilsize
):
3455 cfcontents
= BytesIO();
3456 shutil
.copyfileobj(fcontents
, cfcontents
);
3457 fcontents
.seek(0, 0);
3458 cfcontents
.seek(0, 0);
3459 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3460 cfcontents
.seek(0, 2);
3461 ilcsize
.append(cfcontents
.tell());
3464 ilcmin
= ilcsize
.index(min(ilcsize
));
3465 compression
= compressionlistalt
[ilcmin
];
3466 fcontents
.seek(0, 0);
3467 cfcontents
= BytesIO();
3468 shutil
.copyfileobj(fcontents
, cfcontents
);
3469 cfcontents
.seek(0, 0);
3470 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3471 cfcontents
.seek(0, 2);
3472 cfsize
= cfcontents
.tell();
3473 if(ucfsize
> cfsize
):
3474 fcsize
= format(int(cfsize
), 'x').lower();
3475 fcompression
= compression
;
3477 fcontents
= cfcontents
;
3478 if(fcompression
=="none"):
3480 fcontents
.seek(0, 0);
3481 ftypehex
= format(ftype
, 'x').lower();
3482 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3483 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3486 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3487 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3488 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3491 os
.fsync(catfp
.fileno());
3492 except io
.UnsupportedOperation
:
3494 except AttributeError:
3496 except OSError as e
:
3500 if(hasattr(sys
.stdout
, "buffer")):
3501 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3503 shutil
.copyfileobj(catfp
, sys
.stdout
);
3504 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3505 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3507 upload_file_to_internet_file(catfp
, outfile
);
3515 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3517 if(not rarfile_support
):
3518 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3521 if(rarfile_support
):
3522 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3523 formatspecs
= FormatSpecsListToDict(formatspecs
);
3524 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3525 outfile
= RemoveWindowsPath(outfile
);
3526 checksumtype
= checksumtype
.lower();
3527 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3528 checksumtype
="crc32";
3529 if(checksumtype
=="none"):
3531 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3532 compression
= "auto";
3533 if(compression
not in compressionlist
and compression
is None):
3534 compression
= "auto";
3536 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3537 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3538 if(os
.path
.exists(outfile
)):
3541 except OSError as e
:
3546 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3548 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3551 fbasename
= os
.path
.splitext(outfile
)[0];
3552 fextname
= os
.path
.splitext(outfile
)[1];
3553 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3554 catver
= formatspecs
['format_ver'];
3555 fileheaderver
= str(int(catver
.replace(".", "")));
3561 inodetocatinode
= {};
3562 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3564 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3566 rarfp
= rarfile
.RarFile(infile
, "r");
3567 rartest
= rarfp
.testrar();
3569 VerbosePrintOut("Bad file found!");
3570 numfiles
= int(len(rarfp
.infolist()));
3571 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3574 os
.fsync(catfp
.fileno());
3575 except io
.UnsupportedOperation
:
3577 except AttributeError:
3579 except OSError as e
:
3581 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3584 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3587 member
.external_attr
3589 except AttributeError:
3591 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3594 member
.external_attr
3596 except AttributeError:
3601 if(re
.findall("^[.|/]", member
.filename
)):
3602 fname
= member
.filename
;
3604 fname
= "./"+member
.filename
;
3605 rarinfo
= rarfp
.getinfo(member
.filename
);
3607 VerbosePrintOut(fname
);
3608 if(is_unix
and member
.external_attr
!=0):
3609 fpremode
= int(member
.external_attr
);
3610 elif(member
.is_file()):
3611 fpremode
= int(stat
.S_IFREG
+ 438);
3612 elif(member
.is_symlink()):
3613 fpremode
= int(stat
.S_IFLNK
+ 438);
3614 elif(member
.is_dir()):
3615 fpremode
= int(stat
.S_IFDIR
+ 511);
3616 if(is_windows
and member
.external_attr
!=0):
3617 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3619 fwinattributes
= format(int(0), 'x').lower();
3621 fcsize
= format(int(0), 'x').lower();
3624 if(member
.is_file()):
3626 elif(member
.is_symlink()):
3628 elif(member
.is_dir()):
3632 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3633 fcurfid
= format(int(curfid
), 'x').lower();
3634 fcurinode
= format(int(curfid
), 'x').lower();
3635 curfid
= curfid
+ 1;
3636 fdev_minor
= format(int(0), 'x').lower();
3637 fdev_major
= format(int(0), 'x').lower();
3638 frdev_minor
= format(int(0), 'x').lower();
3639 frdev_major
= format(int(0), 'x').lower();
3641 fsize
= format(int("0"), 'x').lower();
3643 fsize
= format(int(member
.file_size
), 'x').lower();
3645 fsize
= format(int(member
.file_size
), 'x').lower();
3648 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3650 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3651 except AttributeError:
3652 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3653 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3656 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3658 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3659 except AttributeError:
3660 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3661 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3662 if(is_unix
and member
.external_attr
!=0):
3663 fmode
= format(int(member
.external_attr
), 'x').lower();
3664 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3665 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3666 elif(member
.is_file()):
3667 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3668 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3669 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3670 elif(member
.is_symlink()):
3671 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3672 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3673 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3674 elif(member
.is_dir()):
3675 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3676 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3677 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3679 fuid
= format(int(os
.getuid()), 'x').lower();
3680 except AttributeError:
3681 fuid
= format(int(0), 'x').lower();
3683 fuid
= format(int(0), 'x').lower();
3685 fgid
= format(int(os
.getgid()), 'x').lower();
3686 except AttributeError:
3687 fgid
= format(int(0), 'x').lower();
3689 fgid
= format(int(0), 'x').lower();
3693 userinfo
= pwd
.getpwuid(os
.getuid());
3694 funame
= userinfo
.pw_name
;
3697 except AttributeError:
3705 groupinfo
= grp
.getgrgid(os
.getgid());
3706 fgname
= groupinfo
.gr_name
;
3709 except AttributeError:
3713 fcontents
= BytesIO();
3715 fcontents
.write(rarfp
.read(member
.filename
));
3716 if(not compresswholefile
):
3717 fcontents
.seek(0, 2);
3718 ucfsize
= fcontents
.tell();
3719 fcontents
.seek(0, 0);
3720 if(compression
=="auto"):
3721 ilsize
= len(compressionlistalt
);
3724 while(ilmin
< ilsize
):
3725 cfcontents
= BytesIO();
3726 shutil
.copyfileobj(fcontents
, cfcontents
);
3727 fcontents
.seek(0, 0);
3728 cfcontents
.seek(0, 0);
3729 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3731 cfcontents
.seek(0, 2);
3732 ilcsize
.append(cfcontents
.tell());
3736 ilcsize
.append(sys
.maxint
);
3737 except AttributeError:
3738 ilcsize
.append(sys
.maxsize
);
3740 ilcmin
= ilcsize
.index(min(ilcsize
));
3741 compression
= compressionlistalt
[ilcmin
];
3742 fcontents
.seek(0, 0);
3743 cfcontents
= BytesIO();
3744 shutil
.copyfileobj(fcontents
, cfcontents
);
3745 cfcontents
.seek(0, 0);
3746 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3747 cfcontents
.seek(0, 2);
3748 cfsize
= cfcontents
.tell();
3749 if(ucfsize
> cfsize
):
3750 fcsize
= format(int(cfsize
), 'x').lower();
3751 fcompression
= compression
;
3753 fcontents
= cfcontents
;
3754 if(fcompression
=="none"):
3756 fcontents
.seek(0, 0);
3757 ftypehex
= format(ftype
, 'x').lower();
3758 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3759 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3762 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3763 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3764 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3767 os
.fsync(catfp
.fileno());
3768 except io
.UnsupportedOperation
:
3770 except AttributeError:
3772 except OSError as e
:
3776 if(hasattr(sys
.stdout
, "buffer")):
3777 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3779 shutil
.copyfileobj(catfp
, sys
.stdout
);
3780 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3781 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3783 upload_file_to_internet_file(catfp
, outfile
);
3791 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3793 if(not py7zr_support
):
3794 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3798 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3799 formatspecs
= FormatSpecsListToDict(formatspecs
);
3800 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3801 outfile
= RemoveWindowsPath(outfile
);
3802 checksumtype
= checksumtype
.lower();
3803 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3804 checksumtype
="crc32";
3805 if(checksumtype
=="none"):
3807 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3808 compression
= "auto";
3809 if(compression
not in compressionlist
and compression
is None):
3810 compression
= "auto";
3812 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3813 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3814 if(os
.path
.exists(outfile
)):
3817 except OSError as e
:
3822 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3824 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3827 fbasename
= os
.path
.splitext(outfile
)[0];
3828 fextname
= os
.path
.splitext(outfile
)[1];
3829 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3830 catver
= formatspecs
['format_ver'];
3831 fileheaderver
= str(int(catver
.replace(".", "")));
3837 inodetocatinode
= {};
3838 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3840 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3841 file_content
= szpfp
.readall();
3842 #sztest = szpfp.testzip();
3843 sztestalt
= szpfp
.test();
3845 VerbosePrintOut("Bad file found!");
3846 numfiles
= int(len(szpfp
.list()));
3847 AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3848 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3849 if(re
.findall("^[.|/]", member
.filename
)):
3850 fname
= member
.filename
;
3852 fname
= "./"+member
.filename
;
3854 VerbosePrintOut(fname
);
3855 if(not member
.is_directory
):
3856 fpremode
= int(stat
.S_IFREG
+ 438);
3857 elif(member
.is_directory
):
3858 fpremode
= int(stat
.S_IFDIR
+ 511);
3859 fwinattributes
= format(int(0), 'x').lower();
3861 fcsize
= format(int(0), 'x').lower();
3864 if(member
.is_directory
):
3869 fcurfid
= format(int(curfid
), 'x').lower();
3870 fcurinode
= format(int(curfid
), 'x').lower();
3871 curfid
= curfid
+ 1;
3872 fdev_minor
= format(int(0), 'x').lower();
3873 fdev_major
= format(int(0), 'x').lower();
3874 frdev_minor
= format(int(0), 'x').lower();
3875 frdev_major
= format(int(0), 'x').lower();
3877 fsize
= format(int("0"), 'x').lower();
3878 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3879 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3880 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3881 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3882 if(member
.is_directory
):
3883 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3884 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3885 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3887 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3888 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3889 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3891 fuid
= format(int(os
.getuid()), 'x').lower();
3892 except AttributeError:
3893 fuid
= format(int(0), 'x').lower();
3895 fuid
= format(int(0), 'x').lower();
3897 fgid
= format(int(os
.getgid()), 'x').lower();
3898 except AttributeError:
3899 fgid
= format(int(0), 'x').lower();
3901 fgid
= format(int(0), 'x').lower();
3905 userinfo
= pwd
.getpwuid(os
.getuid());
3906 funame
= userinfo
.pw_name
;
3909 except AttributeError:
3917 groupinfo
= grp
.getgrgid(os
.getgid());
3918 fgname
= groupinfo
.gr_name
;
3921 except AttributeError:
3925 fcontents
= BytesIO();
3927 fcontents
.write(file_content
[member
.filename
].read());
3928 fsize
= format(fcontents
.tell(), 'x').lower();
3929 file_content
[member
.filename
].close();
3930 if(not compresswholefile
):
3931 fcontents
.seek(0, 2);
3932 ucfsize
= fcontents
.tell();
3933 fcontents
.seek(0, 0);
3934 if(compression
=="auto"):
3935 ilsize
= len(compressionlistalt
);
3938 while(ilmin
< ilsize
):
3939 cfcontents
= BytesIO();
3940 shutil
.copyfileobj(fcontents
, cfcontents
);
3941 fcontents
.seek(0, 0);
3942 cfcontents
.seek(0, 0);
3943 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3945 cfcontents
.seek(0, 2);
3946 ilcsize
.append(cfcontents
.tell());
3950 ilcsize
.append(sys
.maxint
);
3951 except AttributeError:
3952 ilcsize
.append(sys
.maxsize
);
3954 ilcmin
= ilcsize
.index(min(ilcsize
));
3955 compression
= compressionlistalt
[ilcmin
];
3956 fcontents
.seek(0, 0);
3957 cfcontents
= BytesIO();
3958 shutil
.copyfileobj(fcontents
, cfcontents
);
3959 cfcontents
.seek(0, 0);
3960 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3961 cfcontents
.seek(0, 2);
3962 cfsize
= cfcontents
.tell();
3963 if(ucfsize
> cfsize
):
3964 fcsize
= format(int(cfsize
), 'x').lower();
3965 fcompression
= compression
;
3967 fcontents
= cfcontents
;
3968 if(fcompression
=="none"):
3970 fcontents
.seek(0, 0);
3971 ftypehex
= format(ftype
, 'x').lower();
3972 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3973 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3976 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3977 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3978 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3981 os
.fsync(catfp
.fileno());
3982 except io
.UnsupportedOperation
:
3984 except AttributeError:
3986 except OSError as e
:
3990 if(hasattr(sys
.stdout
, "buffer")):
3991 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3993 shutil
.copyfileobj(catfp
, sys
.stdout
);
3994 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3995 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3997 upload_file_to_internet_file(catfp
, outfile
);
4005 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
4007 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4008 formatspecs
= FormatSpecsListToDict(formatspecs
);
4009 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4011 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4012 if(checkcompressfile
=="tarfile"):
4013 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4014 elif(checkcompressfile
=="zipfile"):
4015 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4016 elif(checkcompressfile
=="catfile"):
4017 return RePackArchiveFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
4018 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4019 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4020 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4021 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4026 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
4028 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4029 formatspecs
= FormatSpecsListToDict(formatspecs
);
4030 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4033 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4034 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4035 if(checkcompressfile
=="tarfile"):
4036 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4037 if(checkcompressfile
=="zipfile"):
4038 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4039 if(rarfile_support
and checkcompressfile
=="rarfile"):
4040 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4041 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4042 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4043 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4050 if(hasattr(sys
.stdin
, "buffer")):
4051 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4053 shutil
.copyfileobj(sys
.stdin
, catfp
);
4055 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4059 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4060 catfp
= download_file_from_internet_file(infile
);
4062 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4067 infile
= RemoveWindowsPath(infile
);
4068 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4069 if(checkcompressfile
=="tarfile"):
4070 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4071 if(checkcompressfile
=="zipfile"):
4072 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4073 if(rarfile_support
and checkcompressfile
=="rarfile"):
4074 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4075 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4076 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4077 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4079 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4080 if(not compresscheck
):
4081 fextname
= os
.path
.splitext(infile
)[1];
4082 if(fextname
==".gz"):
4083 compresscheck
= "gzip";
4084 elif(fextname
==".bz2"):
4085 compresscheck
= "bzip2";
4086 elif(fextname
==".zst"):
4087 compresscheck
= "zstd";
4088 elif(fextname
==".lz4" or fextname
==".clz4"):
4089 compresscheck
= "lz4";
4090 elif(fextname
==".lzo" or fextname
==".lzop"):
4091 compresscheck
= "lzo";
4092 elif(fextname
==".lzma" or fextname
==".xz"):
4093 compresscheck
= "lzma";
4096 if(not compresscheck
):
4098 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4103 SeekToEndOfFile(catfp);
4105 SeekToEndOfFile(catfp);
4106 CatSize = catfp.tell();
4107 CatSizeEnd = CatSize;
4115 curloc
= catfp
.tell();
4118 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4120 catfp
.seek(curloc
, 0);
4121 catstring
= catheader
[0];
4122 catversion
= re
.findall(r
"([\d]+)", catstring
);
4123 fprenumfiles
= catheader
[1];
4124 fnumfiles
= int(fprenumfiles
, 16);
4125 fprechecksumtype
= catheader
[2];
4126 fprechecksum
= catheader
[3];
4127 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4128 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4129 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4130 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4131 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4132 fheadtell
= len(fileheader
);
4133 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4134 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4135 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4137 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4138 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4139 if(seekto
>=fnumfiles
):
4140 seekto
= fnumfiles
- 1;
4146 prefhstart
= catfp
.tell();
4147 if(formatspecs
['new_style']):
4148 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4150 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4151 if(len(preheaderdata
)==0):
4153 prefheadsize
= int(preheaderdata
[0], 16);
4154 prefnumfields
= int(preheaderdata
[1], 16);
4155 preftype
= int(preheaderdata
[2], 16);
4156 if(re
.findall("^[.|/]", preheaderdata
[3])):
4157 prefname
= preheaderdata
[3];
4159 prefname
= "./"+preheaderdata
[3];
4160 prefbasedir
= os
.path
.dirname(prefname
);
4161 preflinkname
= preheaderdata
[4];
4162 prefsize
= int(preheaderdata
[5], 16);
4163 prefatime
= int(preheaderdata
[6], 16);
4164 prefmtime
= int(preheaderdata
[7], 16);
4165 prefctime
= int(preheaderdata
[8], 16);
4166 prefbtime
= int(preheaderdata
[9], 16);
4167 prefmode
= int(preheaderdata
[10], 16);
4168 prefchmode
= stat
.S_IMODE(prefmode
);
4169 preftypemod
= stat
.S_IFMT(prefmode
);
4170 prefwinattributes
= int(preheaderdata
[11], 16);
4171 prefcompression
= preheaderdata
[12];
4172 prefcsize
= int(preheaderdata
[13], 16);
4173 prefuid
= int(preheaderdata
[14], 16);
4174 prefuname
= preheaderdata
[15];
4175 prefgid
= int(preheaderdata
[16], 16);
4176 prefgname
= preheaderdata
[17];
4177 fid
= int(preheaderdata
[18], 16);
4178 finode
= int(preheaderdata
[19], 16);
4179 flinkcount
= int(preheaderdata
[20], 16);
4180 prefdev_minor
= int(preheaderdata
[21], 16);
4181 prefdev_major
= int(preheaderdata
[22], 16);
4182 prefrdev_minor
= int(preheaderdata
[23], 16);
4183 prefrdev_major
= int(preheaderdata
[24], 16);
4184 prefextrasize
= int(preheaderdata
[25], 16);
4185 prefextrafields
= int(preheaderdata
[26], 16);
4186 extrafieldslist
= [];
4188 extraend
= extrastart
+ prefextrafields
;
4189 extrafieldslist
= [];
4190 if(extrastart
<extraend
):
4191 extrafieldslist
.append(preheaderdata
[extrastart
]);
4192 extrastart
= extrastart
+ 1;
4193 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4194 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4195 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4197 hcmax
= len(preheaderdata
) - 2;
4200 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4202 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4203 if(prefcs
!=prenewfcs
and not skipchecksum
):
4204 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4205 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4207 valid_archive
= False;
4208 invalid_archive
= True;
4209 prefhend
= catfp
.tell() - 1;
4210 prefcontentstart
= catfp
.tell();
4212 pyhascontents
= False;
4214 if(prefcompression
):
4215 prefcontents
= catfp
.read(prefsize
);
4217 prefcontents
= catfp
.read(prefcsize
);
4218 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4219 pyhascontents
= True;
4220 if(prefccs
!=prenewfccs
and not skipchecksum
):
4221 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4222 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4226 catfp
.seek(seekstart
, 0);
4228 catfheadsize
= int(preheaderdata
[0], 16);
4229 catfnumfields
= int(preheaderdata
[1], 16);
4230 catftype
= int(preheaderdata
[2], 16);
4231 if(re
.findall("^[.|/]", preheaderdata
[3])):
4232 catfname
= preheaderdata
[3];
4234 catfname
= "./"+preheaderdata
[3];
4235 catflinkname
= preheaderdata
[4];
4236 catfsize
= int(preheaderdata
[5], 16);
4237 catfbasedir
= os
.path
.dirname(catfname
);
4238 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4240 catlist
.update({'catfp': catfp
});
4245 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
4247 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4248 formatspecs
= FormatSpecsListToDict(formatspecs
);
4249 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4252 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4253 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4254 if(checkcompressfile
=="tarfile"):
4255 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4256 if(checkcompressfile
=="zipfile"):
4257 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4258 if(rarfile_support
and checkcompressfile
=="rarfile"):
4259 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4260 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4261 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4262 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4269 if(hasattr(sys
.stdin
, "buffer")):
4270 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4272 shutil
.copyfileobj(sys
.stdin
, catfp
);
4274 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4278 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4279 catfp
= download_file_from_internet_file(infile
);
4280 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4286 infile
= RemoveWindowsPath(infile
);
4287 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4288 if(checkcompressfile
=="tarfile"):
4289 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4290 if(checkcompressfile
=="zipfile"):
4291 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4292 if(rarfile_support
and checkcompressfile
=="rarfile"):
4293 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4294 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4295 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4296 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4298 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4299 if(not compresscheck
):
4300 fextname
= os
.path
.splitext(infile
)[1];
4301 if(fextname
==".gz"):
4302 compresscheck
= "gzip";
4303 elif(fextname
==".bz2"):
4304 compresscheck
= "bzip2";
4305 elif(fextname
==".zst"):
4306 compresscheck
= "zstd";
4307 elif(fextname
==".lz4" or fextname
==".clz4"):
4308 compresscheck
= "lz4";
4309 elif(fextname
==".lzo" or fextname
==".lzop"):
4310 compresscheck
= "lzo";
4311 elif(fextname
==".lzma" or fextname
==".xz"):
4312 compresscheck
= "lzma";
4315 if(not compresscheck
):
4317 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4322 SeekToEndOfFile(catfp);
4324 SeekToEndOfFile(catfp);
4325 CatSize = catfp.tell();
4326 CatSizeEnd = CatSize;
4334 curloc
= catfp
.tell();
4337 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4339 catfp
.seek(curloc
, 0);
4340 catstring
= catheader
[0];
4341 catversion
= re
.findall(r
"([\d]+)", catstring
);
4342 fprenumfiles
= catheader
[1];
4343 fnumfiles
= int(fprenumfiles
, 16);
4344 fprechecksumtype
= catheader
[2];
4345 fprechecksum
= catheader
[3];
4346 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4347 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4348 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4349 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4350 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4351 fheadtell
= len(fileheader
);
4352 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4353 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4354 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4356 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4357 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4358 seekto
= fnumfiles
- 1
4363 prefhstart
= catfp
.tell();
4364 if(formatspecs
['new_style']):
4365 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4367 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4368 if(len(preheaderdata
)==0):
4370 prefheadsize
= int(preheaderdata
[0], 16);
4371 prefnumfields
= int(preheaderdata
[1], 16);
4372 preftype
= int(preheaderdata
[2], 16);
4373 if(re
.findall("^[.|/]", preheaderdata
[3])):
4374 prefname
= preheaderdata
[3];
4376 prefname
= "./"+preheaderdata
[3];
4377 prefbasedir
= os
.path
.dirname(prefname
);
4378 preflinkname
= preheaderdata
[4];
4379 prefsize
= int(preheaderdata
[5], 16);
4380 prefatime
= int(preheaderdata
[6], 16);
4381 prefmtime
= int(preheaderdata
[7], 16);
4382 prefctime
= int(preheaderdata
[8], 16);
4383 prefbtime
= int(preheaderdata
[9], 16);
4384 prefmode
= int(preheaderdata
[10], 16);
4385 prefchmode
= stat
.S_IMODE(prefmode
);
4386 preftypemod
= stat
.S_IFMT(prefmode
);
4387 prefwinattributes
= int(preheaderdata
[11], 16);
4388 prefcompression
= preheaderdata
[12];
4389 prefcsize
= int(preheaderdata
[13], 16);
4390 prefuid
= int(preheaderdata
[14], 16);
4391 prefuname
= preheaderdata
[15];
4392 prefgid
= int(preheaderdata
[16], 16);
4393 prefgname
= preheaderdata
[17];
4394 fid
= int(preheaderdata
[18], 16);
4395 finode
= int(preheaderdata
[19], 16);
4396 flinkcount
= int(preheaderdata
[20], 16);
4397 prefdev_minor
= int(preheaderdata
[21], 16);
4398 prefdev_major
= int(preheaderdata
[22], 16);
4399 prefrdev_minor
= int(preheaderdata
[23], 16);
4400 prefrdev_major
= int(preheaderdata
[24], 16);
4401 prefextrasize
= int(preheaderdata
[25], 16);
4402 prefextrafields
= int(preheaderdata
[26], 16);
4403 extrafieldslist
= [];
4405 extraend
= extrastart
+ prefextrafields
;
4406 extrafieldslist
= [];
4407 if(extrastart
<extraend
):
4408 extrafieldslist
.append(preheaderdata
[extrastart
]);
4409 extrastart
= extrastart
+ 1;
4410 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4411 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4412 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4414 hcmax
= len(preheaderdata
) - 2;
4417 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4419 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4420 if(prefcs
!=prenewfcs
and not skipchecksum
):
4421 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4422 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4424 valid_archive
= False;
4425 invalid_archive
= True;
4426 prefhend
= catfp
.tell() - 1;
4427 prefcontentstart
= catfp
.tell();
4429 pyhascontents
= False;
4431 if(prefcompression
):
4432 prefcontents
= catfp
.read(prefsize
);
4434 prefcontents
= catfp
.read(prefcsize
);
4435 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4436 pyhascontents
= True;
4437 if(prefccs
!=prenewfccs
and not skipchecksum
):
4438 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4439 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4444 prefname
= preheaderdata
[2];
4445 if(re
.findall("^[.|/]", preheaderdata
[2])):
4446 prefname
= preheaderdata
[2];
4448 prefname
= "./"+preheaderdata
[2];
4449 if(prefname
==seekfile
):
4452 catfp
.seek(seekstart
, 0);
4454 catfheadsize
= int(preheaderdata
[0], 16);
4455 catfnumfields
= int(preheaderdata
[1], 16);
4456 catftype
= int(preheaderdata
[2], 16);
4457 if(re
.findall("^[.|/]", preheaderdata
[3])):
4458 catfname
= preheaderdata
[3];
4460 catfname
= "./"+preheaderdata
[3];
4461 catflinkname
= preheaderdata
[4];
4462 catfsize
= int(preheaderdata
[5], 16);
4463 catfbasedir
= os
.path
.dirname(catfname
);
4465 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4469 catlist
.update({'catfp': catfp
});
4474 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4476 def ArchiveFileValidate(infile
, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4477 formatspecs
= FormatSpecsListToDict(formatspecs
);
4479 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4480 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4483 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4484 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4485 if(checkcompressfile
=="tarfile"):
4486 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4487 if(checkcompressfile
=="zipfile"):
4488 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4489 if(rarfile_support
and checkcompressfile
=="rarfile"):
4490 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4491 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4492 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4493 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4500 if(hasattr(sys
.stdin
, "buffer")):
4501 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4503 shutil
.copyfileobj(sys
.stdin
, catfp
);
4505 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4509 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4510 catfp
= download_file_from_internet_file(infile
);
4511 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4517 infile
= RemoveWindowsPath(infile
);
4518 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4519 if(checkcompressfile
=="tarfile"):
4520 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4521 if(checkcompressfile
=="zipfile"):
4522 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4523 if(rarfile_support
and checkcompressfile
=="rarfile"):
4524 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4525 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4526 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4527 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4529 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4530 if(not compresscheck
):
4531 fextname
= os
.path
.splitext(infile
)[1];
4532 if(fextname
==".gz"):
4533 compresscheck
= "gzip";
4534 elif(fextname
==".bz2"):
4535 compresscheck
= "bzip2";
4536 elif(fextname
==".zst"):
4537 compresscheck
= "zstd";
4538 elif(fextname
==".lz4" or fextname
==".clz4"):
4539 compresscheck
= "lz4";
4540 elif(fextname
==".lzo" or fextname
==".lzop"):
4541 compresscheck
= "lzo";
4542 elif(fextname
==".lzma" or fextname
==".xz"):
4543 compresscheck
= "lzma";
4546 if(not compresscheck
):
4548 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4553 SeekToEndOfFile(catfp);
4555 SeekToEndOfFile(catfp);
4556 CatSize = catfp.tell();
4557 CatSizeEnd = CatSize;
4565 curloc
= catfp
.tell();
4568 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4570 catfp
.seek(curloc
, 0);
4571 catstring
= catheader
[0];
4572 catversion
= re
.findall(r
"([\d]+)", catstring
);
4573 fprenumfiles
= catheader
[1];
4574 fnumfiles
= int(fprenumfiles
, 16);
4575 fprechecksumtype
= catheader
[2];
4576 fprechecksum
= catheader
[3];
4578 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4579 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4580 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4581 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4582 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4583 valid_archive
= True;
4584 invalid_archive
= False;
4586 VerbosePrintOut(infile
);
4587 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4588 if(fprechecksum
==catfileheadercshex
):
4590 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4591 VerbosePrintOut("'" + str(fprechecksum
) + "' == " + "'" + str(catfileheadercshex
) + "'");
4594 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4595 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4596 valid_archive
= False;
4597 invalid_archive
= True;
4599 VerbosePrintOut("");
4600 while(il
<fnumfiles
):
4601 catfhstart
= catfp
.tell();
4602 if(formatspecs
['new_style']):
4603 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4605 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4606 if(len(catheaderdata
)==0):
4608 catfheadsize
= int(catheaderdata
[0], 16);
4609 catfnumfields
= int(catheaderdata
[1], 16);
4610 catftype
= int(catheaderdata
[2], 16);
4611 if(re
.findall("^[.|/]", catheaderdata
[3])):
4612 catfname
= catheaderdata
[3];
4614 catfname
= "./"+catheaderdata
[3];
4615 catfbasedir
= os
.path
.dirname(catfname
);
4616 catflinkname
= catheaderdata
[4];
4617 catfsize
= int(catheaderdata
[5], 16);
4618 catfatime
= int(catheaderdata
[6], 16);
4619 catfmtime
= int(catheaderdata
[7], 16);
4620 catfctime
= int(catheaderdata
[8], 16);
4621 catfbtime
= int(catheaderdata
[9], 16);
4622 catfmode
= int(catheaderdata
[10], 16);
4623 catfchmode
= stat
.S_IMODE(catfmode
);
4624 catftypemod
= stat
.S_IFMT(catfmode
);
4625 catfwinattributes
= int(catheaderdata
[11], 16);
4626 catfcompression
= catheaderdata
[12];
4627 catfcsize
= int(catheaderdata
[13], 16);
4628 catfuid
= int(catheaderdata
[14], 16);
4629 catfuname
= catheaderdata
[15];
4630 catfgid
= int(catheaderdata
[16], 16);
4631 catfgname
= catheaderdata
[17];
4632 fid
= int(catheaderdata
[18], 16);
4633 finode
= int(catheaderdata
[19], 16);
4634 flinkcount
= int(catheaderdata
[20], 16);
4635 catfdev_minor
= int(catheaderdata
[21], 16);
4636 catfdev_major
= int(catheaderdata
[22], 16);
4637 catfrdev_minor
= int(catheaderdata
[23], 16);
4638 catfrdev_major
= int(catheaderdata
[24], 16);
4639 catfextrasize
= int(catheaderdata
[25], 16);
4640 catfextrafields
= int(catheaderdata
[26], 16);
4641 extrafieldslist
= [];
4643 extraend
= extrastart
+ catfextrafields
;
4644 extrafieldslist
= [];
4645 if(extrastart
<extraend
):
4646 extrafieldslist
.append(catheaderdata
[extrastart
]);
4647 extrastart
= extrastart
+ 1;
4648 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4649 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4650 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4652 hcmax
= len(catheaderdata
) - 2;
4655 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4657 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4659 VerbosePrintOut(catfname
);
4660 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4661 if(catfcs
==catnewfcs
):
4663 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4664 VerbosePrintOut("'" + str(catfcs
) + "' == " + "'" + str(catnewfcs
) + "'");
4667 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4668 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4669 valid_archive
= False;
4670 invalid_archive
= True;
4671 catfhend
= catfp
.tell() - 1;
4672 catfcontentstart
= catfp
.tell();
4674 pyhascontents
= False;
4676 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4677 catfcontents
= catfp
.read(catfsize
);
4679 catfcontents
= catfp
.read(catfcsize
);
4680 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4681 pyhascontents
= True;
4682 if(catfccs
==catnewfccs
):
4684 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4685 VerbosePrintOut("'" + str(catfccs
) + "' == " + "'" + str(catnewfccs
) + "'");
4688 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4689 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4690 valid_archive
= False;
4691 invalid_archive
= True;
4693 VerbosePrintOut("");
4706 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4708 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4709 formatspecs
= FormatSpecsListToDict(formatspecs
);
4710 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4713 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4714 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4715 if(checkcompressfile
=="tarfile"):
4716 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4717 if(checkcompressfile
=="zipfile"):
4718 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4719 if(rarfile_support
and checkcompressfile
=="rarfile"):
4720 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4721 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4722 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4723 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4730 if(hasattr(sys
.stdin
, "buffer")):
4731 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4733 shutil
.copyfileobj(sys
.stdin
, catfp
);
4735 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4739 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4740 catfp
= download_file_from_internet_file(infile
);
4741 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4747 infile
= RemoveWindowsPath(infile
);
4748 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4749 if(checkcompressfile
=="tarfile"):
4750 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4751 if(checkcompressfile
=="zipfile"):
4752 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4753 if(rarfile_support
and checkcompressfile
=="rarfile"):
4754 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4755 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4756 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4757 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4759 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4760 if(not compresscheck
):
4761 fextname
= os
.path
.splitext(infile
)[1];
4762 if(fextname
==".gz"):
4763 compresscheck
= "gzip";
4764 elif(fextname
==".bz2"):
4765 compresscheck
= "bzip2";
4766 elif(fextname
==".zst"):
4767 compresscheck
= "zstd";
4768 elif(fextname
==".lz4" or fextname
==".clz4"):
4769 compresscheck
= "lz4";
4770 elif(fextname
==".lzo" or fextname
==".lzop"):
4771 compresscheck
= "lzo";
4772 elif(fextname
==".lzma" or fextname
==".xz"):
4773 compresscheck
= "lzma";
4776 if(not compresscheck
):
4778 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4783 SeekToEndOfFile(catfp);
4785 SeekToEndOfFile(catfp);
4786 CatSize = catfp.tell();
4787 CatSizeEnd = CatSize;
4795 curloc
= catfp
.tell();
4798 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4800 catfp
.seek(curloc
, 0);
4801 catstring
= catheader
[0];
4802 catversion
= re
.findall(r
"([\d]+)", catstring
);
4803 fprenumfiles
= catheader
[1];
4804 fnumfiles
= int(fprenumfiles
, 16);
4805 fprechecksumtype
= catheader
[2];
4806 fprechecksum
= catheader
[3];
4807 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4808 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4809 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4810 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4811 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4812 fheadtell
= len(fileheader
);
4813 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4814 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4815 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4817 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4818 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4819 if(seekstart
<0 and seekstart
>fnumfiles
):
4821 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4822 seekend
= fnumfiles
;
4823 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4824 seekend
= fnumfiles
- abs(seekend
);
4827 while(il
< seekstart
):
4828 prefhstart
= catfp
.tell();
4829 if(formatspecs
['new_style']):
4830 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4832 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4833 if(len(preheaderdata
)==0):
4835 prefheadsize
= int(preheaderdata
[0], 16);
4836 prefnumfields
= int(preheaderdata
[1], 16);
4837 if(re
.findall("^[.|/]", preheaderdata
[3])):
4838 prefname
= preheaderdata
[3];
4840 prefname
= "./"+preheaderdata
[3];
4841 prefsize
= int(preheaderdata
[5], 16);
4842 prefcompression
= preheaderdata
[12];
4843 prefcsize
= int(preheaderdata
[13], 16);
4844 prefextrasize
= int(preheaderdata
[24], 16);
4845 prefextrafields
= int(preheaderdata
[25], 16);
4846 extrafieldslist
= [];
4848 extraend
= extrastart
+ prefextrafields
;
4849 extrafieldslist
= [];
4850 if(extrastart
<extraend
):
4851 extrafieldslist
.append(preheaderdata
[extrastart
]);
4852 extrastart
= extrastart
+ 1;
4853 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4854 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4855 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4857 hcmax
= len(preheaderdata
) - 2;
4860 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4862 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4863 if(prefcs
!=prenewfcs
and not skipchecksum
):
4864 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4865 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4867 valid_archive
= False;
4868 invalid_archive
= True;
4869 prefhend
= catfp
.tell() - 1;
4870 prefcontentstart
= catfp
.tell();
4872 pyhascontents
= False;
4874 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
4875 prefcontents
= catfp
.read(prefsize
);
4877 prefcontents
= catfp
.read(prefcsize
);
4878 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4879 pyhascontents
= True;
4880 if(prefccs
!=prenewfccs
and not skipchecksum
):
4881 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4882 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4886 fileidnum
= seekstart
;
4888 while(fileidnum
<seekend
):
4889 catfhstart
= catfp
.tell();
4890 if(formatspecs
['new_style']):
4891 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4893 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4894 if(len(catheaderdata
)==0):
4896 catfheadsize
= int(catheaderdata
[0], 16);
4897 catfnumfields
= int(catheaderdata
[1], 16);
4898 catftype
= int(catheaderdata
[2], 16);
4899 if(re
.findall("^[.|/]", catheaderdata
[3])):
4900 catfname
= catheaderdata
[3];
4902 catfname
= "./"+catheaderdata
[3];
4903 catfbasedir
= os
.path
.dirname(catfname
);
4904 catflinkname
= catheaderdata
[4];
4905 catfsize
= int(catheaderdata
[5], 16);
4906 catfatime
= int(catheaderdata
[6], 16);
4907 catfmtime
= int(catheaderdata
[7], 16);
4908 catfctime
= int(catheaderdata
[8], 16);
4909 catfbtime
= int(catheaderdata
[9], 16);
4910 catfmode
= int(catheaderdata
[10], 16);
4911 catfchmode
= stat
.S_IMODE(catfmode
);
4912 catftypemod
= stat
.S_IFMT(catfmode
);
4913 catfwinattributes
= int(catheaderdata
[11], 16);
4914 catfcompression
= catheaderdata
[12];
4915 catfcsize
= int(catheaderdata
[13], 16);
4916 catfuid
= int(catheaderdata
[14], 16);
4917 catfuname
= catheaderdata
[15];
4918 catfgid
= int(catheaderdata
[16], 16);
4919 catfgname
= catheaderdata
[17];
4920 catfid
= int(catheaderdata
[18], 16);
4921 catfinode
= int(catheaderdata
[19], 16);
4922 catflinkcount
= int(catheaderdata
[20], 16);
4923 catfdev_minor
= int(catheaderdata
[21], 16);
4924 catfdev_major
= int(catheaderdata
[22], 16);
4925 catfrdev_minor
= int(catheaderdata
[23], 16);
4926 catfrdev_major
= int(catheaderdata
[24], 16);
4927 catfextrasize
= int(catheaderdata
[25], 16);
4928 catfextrafields
= int(catheaderdata
[26], 16);
4929 extrafieldslist
= [];
4931 extraend
= extrastart
+ catfextrafields
;
4932 extrafieldslist
= [];
4933 if(extrastart
<extraend
):
4934 extrafieldslist
.append(catheaderdata
[extrastart
]);
4935 extrastart
= extrastart
+ 1;
4936 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4937 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4938 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4940 hcmax
= len(catheaderdata
) - 2;
4943 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4945 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4946 if(catfcs
!=catnewfcs
and not skipchecksum
):
4947 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
4948 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4950 catfhend
= catfp
.tell() - 1;
4951 catfcontentstart
= catfp
.tell();
4952 catfcontents
= BytesIO();
4953 pyhascontents
= False;
4954 if(catfsize
>0 and not listonly
):
4955 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4956 catfcontents
.write(catfp
.read(catfsize
));
4958 catfcontents
.write(catfp
.read(catfcsize
));
4959 catfcontents
.seek(0, 0);
4960 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
4961 pyhascontents
= True;
4962 if(catfccs
!=catnewfccs
and skipchecksum
):
4963 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
4964 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4966 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4969 catfcontents
.seek(0, 0);
4971 catfcontents
= UncompressArchiveFile(catfcontents
, formatspecs
);
4972 catfcontents
.seek(0, 0);
4973 catfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
4974 if(catfsize
>0 and listonly
):
4975 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4976 catfp
.seek(catfsize
, 1);
4978 catfp
.seek(catfcsize
, 1);
4979 pyhascontents
= False;
4981 catfcontentend
= catfp
.tell() - 1;
4982 catfcontents
.seek(0, 0);
4983 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fcompression': catfcompression
, 'fcsize': catfcsize
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': catfinode
, 'flinkcount': catflinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fchecksumtype': catfchecksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
} });
4984 fileidnum
= fileidnum
+ 1;
4985 realidnum
= realidnum
+ 1;
4987 catlist
.update({'catfp': catfp
});
4992 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
4994 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4995 formatspecs
= FormatSpecsListToDict(formatspecs
);
4996 catfp
= BytesIO(catstr
);
4997 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
4998 return listcatfiles
;
5000 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
5002 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5003 formatspecs
= FormatSpecsListToDict(formatspecs
);
5005 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5006 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5007 return listcatfiles
;
5009 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5010 formatspecs
= FormatSpecsListToDict(formatspecs
);
5012 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5013 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5014 return listcatfiles
;
5016 if(not rarfile_support
):
5017 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5020 if(rarfile_support
):
5021 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5022 formatspecs
= FormatSpecsListToDict(formatspecs
);
5024 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5025 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5026 return listcatfiles
;
5028 if(not py7zr_support
):
5029 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5033 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5034 formatspecs
= FormatSpecsListToDict(formatspecs
);
5036 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5037 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5038 return listcatfiles
;
5040 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5041 formatspecs
= FormatSpecsListToDict(formatspecs
);
5042 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
5043 if(checkcompressfile
=="tarfile"):
5044 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5045 elif(checkcompressfile
=="zipfile"):
5046 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5047 elif(checkcompressfile
=="catfile"):
5048 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5049 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5050 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5051 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5052 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5057 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5058 formatspecs
= FormatSpecsListToDict(formatspecs
);
5059 catver
= formatspecs
['format_ver'];
5060 fileheaderver
= str(int(catver
.replace(".", "")));
5061 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5062 advancedlist
= formatspecs
['use_advanced_list'];
5063 altinode
= formatspecs
['use_alt_inode'];
5066 for line
in sys
.stdin
:
5067 infilelist
.append(line
.strip());
5068 infilelist
= list(filter(None, infilelist
));
5069 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
5070 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5072 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
5073 for line
in finfile
:
5074 infilelist
.append(line
.strip());
5075 infilelist
= list(filter(None, infilelist
));
5077 if(isinstance(infiles
, (list, tuple, ))):
5078 infilelist
= list(filter(None, infiles
));
5079 elif(isinstance(infiles
, (str, ))):
5080 infilelist
= list(filter(None, [infiles
]));
5082 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
5084 GetDirList
= ListDir(infilelist
, followlink
, False);
5092 inodetocatinode
= {};
5094 fnumfiles
= int(len(GetDirList
));
5095 catver
= formatspecs
['format_ver'];
5096 fileheaderver
= str(int(catver
.replace(".", "")));
5097 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5098 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5099 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5100 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5101 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5102 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5103 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5104 fheadtell
= len(fileheader
);
5105 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5106 for curfname
in GetDirList
:
5107 catfhstart
= fheadtell
;
5108 if(re
.findall("^[.|/]", curfname
)):
5111 fname
= "./"+curfname
;
5113 VerbosePrintOut(fname
);
5114 if(not followlink
or followlink
is None):
5115 fstatinfo
= os
.lstat(fname
);
5117 fstatinfo
= os
.stat(fname
);
5118 fpremode
= fstatinfo
.st_mode
;
5119 finode
= fstatinfo
.st_ino
;
5120 flinkcount
= fstatinfo
.st_nlink
;
5122 if(stat
.S_ISREG(fpremode
)):
5124 elif(stat
.S_ISLNK(fpremode
)):
5126 elif(stat
.S_ISCHR(fpremode
)):
5128 elif(stat
.S_ISBLK(fpremode
)):
5130 elif(stat
.S_ISDIR(fpremode
)):
5132 elif(stat
.S_ISFIFO(fpremode
)):
5134 elif(stat
.S_ISSOCK(fpremode
)):
5136 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
5138 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
5140 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
5145 fbasedir
= os
.path
.dirname(fname
);
5147 if(not followlink
and finode
!=0):
5149 if(finode
in inodelist
):
5151 flinkname
= inodetofile
[finode
];
5155 fcurinode
= inodetocatinode
[finode
];
5156 if(finode
not in inodelist
):
5157 inodelist
.append(finode
);
5158 inodetofile
.update({finode
: fname
});
5159 inodetocatinode
.update({finode
: curinode
});
5163 fcurinode
= curinode
;
5164 curinode
= curinode
+ 1;
5166 fcurinode
= curinode
;
5167 curinode
= curinode
+ 1;
5168 curfid
= curfid
+ 1;
5170 flinkname
= os
.readlink(fname
);
5171 fdev
= fstatinfo
.st_dev
;
5172 getfdev
= GetDevMajorMinor(fdev
);
5173 fdev_minor
= getfdev
[0];
5174 fdev_major
= getfdev
[1];
5175 frdev
= fstatinfo
.st_dev
;
5176 if(hasattr(fstatinfo
, "st_rdev")):
5177 frdev
= fstatinfo
.st_rdev
;
5179 frdev
= fstatinfo
.st_dev
;
5180 getfrdev
= GetDevMajorMinor(frdev
);
5181 frdev_minor
= getfrdev
[0];
5182 frdev_major
= getfrdev
[1];
5183 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5185 if(ftype
==0 or ftype
==7):
5186 fsize
= fstatinfo
.st_size
;
5187 fatime
= fstatinfo
.st_atime
;
5188 fmtime
= fstatinfo
.st_mtime
;
5189 fctime
= fstatinfo
.st_ctime
;
5190 if(hasattr(fstatinfo
, "st_birthtime")):
5191 fbtime
= fstatinfo
.st_birthtime
;
5193 fbtime
= fstatinfo
.st_ctime
;
5194 fmode
= fstatinfo
.st_mode
;
5195 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
5196 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
5197 fuid
= fstatinfo
.st_uid
;
5198 fgid
= fstatinfo
.st_gid
;
5203 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
5204 funame
= userinfo
.pw_name
;
5213 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
5214 fgname
= groupinfo
.gr_name
;
5219 fdev_minor
= fdev_minor
;
5220 fdev_major
= fdev_major
;
5221 frdev_minor
= frdev_minor
;
5222 frdev_major
= frdev_major
;
5223 flinkcount
= flinkcount
;
5224 if(hasattr(fstatinfo
, "st_file_attributes")):
5225 fwinattributes
= fstatinfo
.st_file_attributes
;
5230 fcontents
= BytesIO();
5231 if(ftype
==0 or ftype
==7):
5232 with
open(fname
, "rb") as fpc
:
5233 shutil
.copyfileobj(fpc
, fcontents
);
5234 if(followlink
and (ftype
==1 or ftype
==2)):
5235 flstatinfo
= os
.stat(flinkname
);
5236 with
open(flinkname
, "rb") as fpc
:
5237 shutil
.copyfileobj(fpc
, fcontents
);
5238 fcontents
.seek(0, 0);
5239 ftypehex
= format(ftype
, 'x').lower();
5240 extrafields
= len(extradata
);
5241 extrafieldslist
= extradata
;
5242 catfextrafields
= extrafields
;
5243 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5244 if(len(extradata
)>0):
5245 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5246 extrasizelen
= len(extrasizestr
);
5247 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5248 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5249 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5250 catoutlenhex
= format(catoutlen
, 'x').lower();
5251 catoutlist
.insert(0, catoutlenhex
);
5252 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5253 catheaderdata
= catoutlist
;
5254 if(len(extradata
)>0):
5255 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5256 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5257 catfnumfields
= catoutlen
;
5258 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5259 fcontents
.seek(0, 0);
5260 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5261 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5262 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5263 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5264 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5265 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5266 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5267 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5268 catfcontentstart
= fheadtell
;
5269 fheadtell
+= len(catfileoutstr
) + 1;
5270 catfcontentend
= fheadtell
- 1;
5271 catfhend
= catfcontentend
;
5272 fcontents
.seek(0, 0);
5273 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5274 pyhascontents
= False;
5275 if(int(fsize
)>0 and not listonly
):
5276 pyhascontents
= True;
5277 if(int(fsize
)>0 and listonly
):
5278 fcontents
= BytesIO();
5279 pyhascontents
= False;
5280 fcontents
.seek(0, 0);
5281 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5282 fileidnum
= fileidnum
+ 1;
5285 def TarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5286 formatspecs
= FormatSpecsListToDict(formatspecs
);
5292 inodetocatinode
= {};
5296 if(hasattr(sys
.stdin
, "buffer")):
5297 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5299 shutil
.copyfileobj(sys
.stdin
, infile
);
5304 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5305 infile
= download_file_from_internet_file(infile
);
5310 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5312 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
5314 if(not tarfile
.is_tarfile(infile
)):
5316 except AttributeError:
5317 if(not is_tarfile(infile
)):
5322 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
5323 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
5325 tarfp
= tarfile
.open(infile
, "r");
5326 except FileNotFoundError
:
5328 fnumfiles
= int(len(tarfp
.getmembers()));
5329 catver
= formatspecs
['format_ver'];
5330 fileheaderver
= str(int(catver
.replace(".", "")));
5331 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5332 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5333 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5334 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5335 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5336 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5337 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5338 fheadtell
= len(fileheader
);
5339 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5340 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
5341 catfhstart
= fheadtell
;
5342 if(re
.findall("^[.|/]", member
.name
)):
5343 fname
= member
.name
;
5345 fname
= "./"+member
.name
;
5347 VerbosePrintOut(fname
);
5348 fpremode
= member
.mode
;
5349 ffullmode
= member
.mode
;
5353 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5355 elif(member
.isdev()):
5356 ffullmode
= member
.mode
;
5358 elif(member
.islnk()):
5359 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5361 elif(member
.issym()):
5362 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
5364 elif(member
.ischr()):
5365 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
5367 elif(member
.isblk()):
5368 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
5370 elif(member
.isdir()):
5371 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
5373 elif(member
.isfifo()):
5374 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
5376 elif(member
.issparse()):
5377 ffullmode
= member
.mode
;
5380 ffullmode
= member
.mode
;
5383 fbasedir
= os
.path
.dirname(fname
);
5387 curfid
= curfid
+ 1;
5389 flinkname
= member
.linkname
;
5390 fdev_minor
= member
.devminor
;
5391 fdev_major
= member
.devmajor
;
5392 frdev_minor
= member
.devminor
;
5393 frdev_major
= member
.devmajor
;
5394 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5396 elif(ftype
==0 or ftype
==7):
5397 fsize
= member
.size
;
5399 fsize
= member
.size
;
5400 fatime
= member
.mtime
;
5401 fmtime
= member
.mtime
;
5402 fctime
= member
.mtime
;
5403 fbtime
= member
.mtime
;
5405 fchmode
= stat
.S_IMODE(ffullmode
);
5406 ftypemod
= stat
.S_IFMT(ffullmode
);
5409 funame
= member
.uname
;
5410 fgname
= member
.gname
;
5411 flinkcount
= flinkcount
;
5412 fwinattributes
= int(0);
5415 fcontents
= BytesIO();
5416 if(ftype
==0 or ftype
==7):
5417 with tarfp
.extractfile(member
) as fpc
:
5418 shutil
.copyfileobj(fpc
, fcontents
);
5419 fcontents
.seek(0, 0);
5420 ftypehex
= format(ftype
, 'x').lower();
5421 extrafields
= len(extradata
);
5422 extrafieldslist
= extradata
;
5423 catfextrafields
= extrafields
;
5424 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5425 if(len(extradata
)>0):
5426 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5427 extrasizelen
= len(extrasizestr
);
5428 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5429 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5430 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5431 catoutlenhex
= format(catoutlen
, 'x').lower();
5432 catoutlist
.insert(0, catoutlenhex
);
5433 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5434 catheaderdata
= catoutlist
;
5435 if(len(extradata
)>0):
5436 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5437 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5438 catfnumfields
= catoutlen
;
5439 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5440 fcontents
.seek(0, 0);
5441 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5442 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5443 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5444 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5445 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5446 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5447 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5448 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5449 catfcontentstart
= fheadtell
;
5450 fheadtell
+= len(catfileoutstr
) + 1;
5451 catfcontentend
= fheadtell
- 1;
5452 catfhend
= catfcontentend
;
5453 fcontents
.seek(0, 0);
5454 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5455 pyhascontents
= False;
5456 if(int(fsize
)>0 and not listonly
):
5457 pyhascontents
= True;
5458 if(int(fsize
)>0 and listonly
):
5459 fcontents
= BytesIO();
5460 pyhascontents
= False;
5461 fcontents
.seek(0, 0);
5462 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5463 fileidnum
= fileidnum
+ 1;
5466 def ZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5467 formatspecs
= FormatSpecsListToDict(formatspecs
);
5473 inodetocatinode
= {};
5477 if(hasattr(sys
.stdin
, "buffer")):
5478 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5480 shutil
.copyfileobj(sys
.stdin
, infile
);
5485 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5486 infile
= download_file_from_internet_file(infile
);
5491 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5495 if(not zipfile
.is_zipfile(infile
)):
5498 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
5499 except FileNotFoundError
:
5501 ziptest
= zipfp
.testzip();
5503 VerbosePrintOut("Bad file found!");
5504 fnumfiles
= int(len(zipfp
.infolist()));
5505 catver
= formatspecs
['format_ver'];
5506 fileheaderver
= str(int(catver
.replace(".", "")));
5507 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5508 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5509 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5510 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5511 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5512 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5513 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5514 fheadtell
= len(fileheader
);
5515 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5516 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5517 catfhstart
= fheadtell
;
5518 if(re
.findall("^[.|/]", member
.filename
)):
5519 fname
= member
.filename
;
5521 fname
= "./"+member
.filename
;
5522 zipinfo
= zipfp
.getinfo(member
.filename
);
5524 VerbosePrintOut(fname
);
5525 if(not member
.is_dir()):
5526 fpremode
= stat
.S_IFREG
+ 438;
5527 elif(member
.is_dir()):
5528 fpremode
= stat
.S_IFDIR
+ 511;
5531 if(not member
.is_dir()):
5533 elif(member
.is_dir()):
5536 fbasedir
= os
.path
.dirname(fname
);
5540 curfid
= curfid
+ 1;
5548 fsize
= member
.file_size
;
5550 fsize
= member
.file_size
;
5551 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5552 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5553 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5554 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5555 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5556 fwinattributes
= int(zipinfo
.external_attr
);
5557 if(not member
.is_dir()):
5558 fmode
= int(stat
.S_IFREG
+ 438);
5559 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5560 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5561 elif(member
.is_dir()):
5562 fmode
= int(stat
.S_IFDIR
+ 511);
5563 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5564 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5565 elif(zipinfo
.create_system
==3):
5566 fwinattributes
= int(0);
5567 fmode
= int(zipinfo
.external_attr
);
5569 fwinattributes
= int(0);
5570 if(not member
.is_dir()):
5571 fmode
= int(stat
.S_IFREG
+ 438);
5572 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5573 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5574 elif(member
.is_dir()):
5575 fmode
= int(stat
.S_IFDIR
+ 511);
5576 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5577 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5582 except AttributeError:
5588 except AttributeError:
5595 userinfo
= pwd
.getpwuid(os
.getuid());
5596 funame
= userinfo
.pw_name
;
5599 except AttributeError:
5607 groupinfo
= grp
.getgrgid(os
.getgid());
5608 fgname
= groupinfo
.gr_name
;
5611 except AttributeError:
5615 fcontents
= BytesIO();
5617 fcontents
.write(zipfp
.read(member
.filename
));
5618 fcontents
.seek(0, 0);
5619 ftypehex
= format(ftype
, 'x').lower();
5620 extrafields
= len(extradata
);
5621 extrafieldslist
= extradata
;
5622 catfextrafields
= extrafields
;
5623 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5624 if(len(extradata
)>0):
5625 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5626 extrasizelen
= len(extrasizestr
);
5627 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5628 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5629 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5630 catoutlenhex
= format(catoutlen
, 'x').lower();
5631 catoutlist
.insert(0, catoutlenhex
);
5632 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5633 catheaderdata
= catoutlist
;
5634 if(len(extradata
)>0):
5635 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5636 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5637 catfnumfields
= catoutlen
;
5638 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5639 fcontents
.seek(0, 0);
5640 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5641 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5642 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5643 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5644 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5645 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5646 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5647 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5648 catfcontentstart
= fheadtell
;
5649 fheadtell
+= len(catfileoutstr
) + 1;
5650 catfcontentend
= fheadtell
- 1;
5651 catfhend
= catfcontentend
;
5652 fcontents
.seek(0, 0);
5653 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5654 pyhascontents
= False;
5655 if(int(fsize
)>0 and not listonly
):
5656 pyhascontents
= True;
5657 if(int(fsize
)>0 and listonly
):
5658 fcontents
= BytesIO();
5659 pyhascontents
= False;
5660 fcontents
.seek(0, 0);
5661 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5662 fileidnum
= fileidnum
+ 1;
5665 if(not rarfile_support
):
5666 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5669 if(rarfile_support
):
5670 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5671 formatspecs
= FormatSpecsListToDict(formatspecs
);
5677 inodetocatinode
= {};
5679 if(not os
.path
.exists(infile
,) or not os
.path
.isfile(infile
,)):
5681 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5683 rarfp
= rarfile
.RarFile(infile
, "r");
5684 rartest
= rarfp
.testrar();
5686 VerbosePrintOut("Bad file found!");
5687 fnumfiles
= int(len(rarfp
.infolist()));
5688 catver
= formatspecs
['format_ver'];
5689 fileheaderver
= str(int(catver
.replace(".", "")));
5690 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5691 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5692 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5693 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5694 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5695 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5696 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5697 fheadtell
= len(fileheader
);
5698 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5699 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5700 catfhstart
= fheadtell
;
5703 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5706 member
.external_attr
5708 except AttributeError:
5710 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5713 member
.external_attr
5715 except AttributeError:
5720 if(re
.findall("^[.|/]", member
.filename
)):
5721 fname
= member
.filename
;
5723 fname
= "./"+member
.filename
;
5724 rarinfo
= rarfp
.getinfo(member
.filename
);
5726 VerbosePrintOut(fname
);
5727 if(is_unix
and member
.external_attr
!=0):
5728 fpremode
= int(member
.external_attr
);
5729 elif(member
.is_file()):
5730 fpremode
= stat
.S_IFREG
+ 438;
5731 elif(member
.is_symlink()):
5732 fpremode
= stat
.S_IFLNK
+ 438;
5733 elif(member
.is_dir()):
5734 fpremode
= stat
.S_IFDIR
+ 511;
5735 if(is_windows
and member
.external_attr
!=0):
5736 fwinattributes
= int(member
.external_attr
);
5738 fwinattributes
= int(0);
5743 if(member
.is_file()):
5745 elif(member
.is_symlink()):
5747 elif(member
.is_dir()):
5751 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5752 fbasedir
= os
.path
.dirname(fname
);
5756 curfid
= curfid
+ 1;
5764 fsize
= member
.file_size
;
5767 fatime
= int(member
.atime
.timestamp());
5769 fatime
= int(member
.mtime
.timestamp());
5770 except AttributeError:
5771 fatime
= int(member
.mtime
.timestamp());
5772 fmtime
= int(member
.mtime
.timestamp());
5775 fctime
= int(member
.ctime
.timestamp());
5777 fctime
= int(member
.mtime
.timestamp());
5778 except AttributeError:
5779 fctime
= int(member
.mtime
.timestamp());
5780 fbtime
= int(member
.mtime
.timestamp());
5781 if(is_unix
and member
.external_attr
!=0):
5782 fmode
= int(member
.external_attr
);
5783 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5784 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5785 elif(member
.is_file()):
5786 fmode
= int(stat
.S_IFREG
+ 438)
5787 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5788 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5789 elif(member
.is_symlink()):
5790 fmode
= int(stat
.S_IFLNK
+ 438)
5791 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5792 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5793 elif(member
.is_dir()):
5794 fmode
= int(stat
.S_IFDIR
+ 511)
5795 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5796 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5799 except AttributeError:
5805 except AttributeError:
5812 userinfo
= pwd
.getpwuid(os
.getuid());
5813 funame
= userinfo
.pw_name
;
5816 except AttributeError:
5824 groupinfo
= grp
.getgrgid(os
.getgid());
5825 fgname
= groupinfo
.gr_name
;
5828 except AttributeError:
5832 fcontents
= BytesIO();
5834 fcontents
.write(rarfp
.read(member
.filename
));
5835 fcontents
.seek(0, 0);
5836 ftypehex
= format(ftype
, 'x').lower();
5837 extrafields
= len(extradata
);
5838 extrafieldslist
= extradata
;
5839 catfextrafields
= extrafields
;
5840 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5841 if(len(extradata
)>0):
5842 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5843 extrasizelen
= len(extrasizestr
);
5844 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5845 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5846 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5847 catoutlenhex
= format(catoutlen
, 'x').lower();
5848 catoutlist
.insert(0, catoutlenhex
);
5849 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5850 if(len(extradata
)>0):
5851 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5852 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5853 catfnumfields
= 24 + catfextrafields
;
5854 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5855 fcontents
.seek(0, 0);
5856 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5857 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5858 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5859 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5860 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5861 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5862 catheaderdata
= catoutlist
;
5863 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5864 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5865 catfcontentstart
= fheadtell
;
5866 fheadtell
+= len(catfileoutstr
) + 1;
5867 catfcontentend
= fheadtell
- 1;
5868 catfhend
= catfcontentend
;
5869 fcontents
.seek(0, 0);
5870 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5871 pyhascontents
= False;
5872 if(int(fsize
)>0 and not listonly
):
5873 pyhascontents
= True;
5874 if(int(fsize
)>0 and listonly
):
5875 fcontents
= BytesIO();
5876 pyhascontents
= False;
5877 fcontents
.seek(0, 0);
5878 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5879 fileidnum
= fileidnum
+ 1;
5882 if(not py7zr_support
):
5883 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5887 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5888 formatspecs
= FormatSpecsListToDict(formatspecs
);
5894 inodetocatinode
= {};
5896 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5897 file_content
= szpfp
.readall();
5898 #sztest = szpfp.testzip();
5899 sztestalt
= szpfp
.test();
5901 VerbosePrintOut("Bad file found!");
5902 numfiles
= int(len(szpfp
.list()));
5903 catver
= formatspecs
['format_ver'];
5904 fileheaderver
= str(int(catver
.replace(".", "")));
5905 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5906 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5907 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5908 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5909 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5910 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5911 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5912 fheadtell
= len(fileheader
);
5913 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5914 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5915 catfhstart
= fheadtell
;
5916 if(re
.findall("^[.|/]", member
.filename
)):
5917 fname
= member
.filename
;
5919 fname
= "./"+member
.filename
;
5920 if(not member
.is_directory
):
5921 fpremode
= int(stat
.S_IFREG
+ 438);
5922 elif(member
.is_directory
):
5923 fpremode
= int(stat
.S_IFDIR
+ 511);
5924 fwinattributes
= int(0);
5929 if(member
.is_directory
):
5934 fbasedir
= os
.path
.dirname(fname
);
5938 curfid
= curfid
+ 1;
5945 fatime
= int(member
.creationtime
.timestamp());
5946 fmtime
= int(member
.creationtime
.timestamp());
5947 fctime
= int(member
.creationtime
.timestamp());
5948 fbtime
= int(member
.creationtime
.timestamp());
5949 if(member
.is_directory
):
5950 fmode
= int(stat
.S_IFDIR
+ 511)
5951 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5952 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5954 fmode
= int(stat
.S_IFLNK
+ 438)
5955 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5956 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5959 except AttributeError:
5965 except AttributeError:
5972 userinfo
= pwd
.getpwuid(os
.getuid());
5973 funame
= userinfo
.pw_name
;
5976 except AttributeError:
5984 groupinfo
= grp
.getgrgid(os
.getgid());
5985 fgname
= groupinfo
.gr_name
;
5988 except AttributeError:
5992 fcontents
= BytesIO();
5994 fcontents
.write(file_content
[member
.filename
].read());
5995 fsize
= format(fcontents
.tell(), 'x').lower();
5997 fcontents
.seek(0, 0);
5998 ftypehex
= format(ftype
, 'x').lower();
5999 extrafields
= len(extradata
);
6000 extrafieldslist
= extradata
;
6001 catfextrafields
= extrafields
;
6002 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
6003 if(len(extradata
)>0):
6004 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6005 extrasizelen
= len(extrasizestr
);
6006 extrasizelenhex
= format(extrasizelen
, 'x').lower();
6007 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
6008 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
6009 catoutlenhex
= format(catoutlen
, 'x').lower();
6010 catoutlist
.insert(0, catoutlenhex
);
6011 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
6012 catheaderdata
= catoutlist
;
6013 if(len(extradata
)>0):
6014 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6015 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
6016 catfnumfields
= 24 + catfextrafields
;
6017 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6018 fcontents
.seek(0, 0);
6019 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
6020 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6021 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
6022 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
6023 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6024 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6025 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
6026 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
6027 catfcontentstart
= fheadtell
;
6028 fheadtell
+= len(catfileoutstr
) + 1;
6029 catfcontentend
= fheadtell
- 1;
6030 catfhend
= catfcontentend
;
6031 fcontents
.seek(0, 0);
6032 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
6033 pyhascontents
= False;
6034 if(int(fsize
)>0 and not listonly
):
6035 pyhascontents
= True;
6036 if(int(fsize
)>0 and listonly
):
6037 fcontents
= BytesIO();
6038 pyhascontents
= False;
6039 fcontents
.seek(0, 0);
6040 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
6041 fileidnum
= fileidnum
+ 1;
6044 def InFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6045 formatspecs
= FormatSpecsListToDict(formatspecs
);
6046 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
6047 if(checkcompressfile
=="tarfile"):
6048 return TarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6049 elif(checkcompressfile
=="zipfile"):
6050 return ZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6051 elif(checkcompressfile
=="catfile"):
6052 return ArchiveFileToArray(infile
, 0, 0, listonly
, True, False, formatspecs
, False);
6053 elif(rarfile_support
and checkcompressfile
=="rarfile"):
6054 return RarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6055 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
6056 return SevenZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6061 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6062 formatspecs
= FormatSpecsListToDict(formatspecs
);
6063 outarray
= BytesIO();
6064 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6065 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6066 return listcatfiles
;
6068 def ArchiveFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6069 formatspecs
= FormatSpecsListToDict(formatspecs
);
6070 if(isinstance(infile
, dict)):
6071 listcatfiles
= infile
;
6073 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6074 infile
= RemoveWindowsPath(infile
);
6075 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
, returnfp
);
6076 if(not listcatfiles
):
6078 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6080 catarray
.update({'catfp': listcatfiles
['catfp']});
6081 lenlist
= len(listcatfiles
['ffilelist']);
6083 lcfx
= int(listcatfiles
['fnumfiles']);
6084 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6085 lcfx
= int(lenlist
);
6087 lcfx
= int(listcatfiles
['fnumfiles']);
6089 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6090 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6091 catarray
['filetoid'].update(filetoidarray
);
6092 catarray
['idtofile'].update(idtofilearray
);
6093 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6094 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6095 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6096 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6097 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6098 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6099 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6100 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6101 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6102 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6103 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6104 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6105 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6106 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6107 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6108 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6109 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6110 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6111 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6112 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6113 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6114 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6115 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6116 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6117 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6118 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6119 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6120 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6121 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6122 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6123 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6127 create_alias_function("", __file_format_name__
, "ToArrayIndex", ArchiveFileToArrayIndex
);
6129 def ListDirToArrayIndexAlt(infiles
, dirlistfromtxt
=False, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6130 formatspecs
= FormatSpecsListToDict(formatspecs
);
6131 listcatfiles
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6132 if(not listcatfiles
):
6134 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6135 lenlist
= len(listcatfiles
['ffilelist']);
6137 lcfx
= int(listcatfiles
['fnumfiles']);
6138 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6139 lcfx
= int(lenlist
);
6141 lcfx
= int(listcatfiles
['fnumfiles']);
6143 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6144 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6145 catarray
['filetoid'].update(filetoidarray
);
6146 catarray
['idtofile'].update(idtofilearray
);
6147 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6148 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6149 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6150 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6151 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6152 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6153 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6154 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6155 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6156 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6157 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6158 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6159 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6160 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6161 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6162 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6163 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6164 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6165 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6166 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6167 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6168 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6169 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6170 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6171 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6172 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6173 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6174 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6175 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6176 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6177 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6181 def TarFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6182 formatspecs
= FormatSpecsListToDict(formatspecs
);
6183 listcatfiles
= TarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6184 if(not listcatfiles
):
6186 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6187 lenlist
= len(listcatfiles
['ffilelist']);
6189 lcfx
= int(listcatfiles
['fnumfiles']);
6190 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6191 lcfx
= int(lenlist
);
6193 lcfx
= int(listcatfiles
['fnumfiles']);
6195 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6196 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6197 catarray
['filetoid'].update(filetoidarray
);
6198 catarray
['idtofile'].update(idtofilearray
);
6199 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6200 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6201 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6202 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6203 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6204 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6205 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6206 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6207 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6208 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6209 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6210 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6211 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6212 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6213 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6214 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6215 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6216 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6217 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6218 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6219 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6220 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6221 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6222 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6223 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6224 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6225 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6226 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6227 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6228 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6229 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6233 def ZipFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6234 formatspecs
= FormatSpecsListToDict(formatspecs
);
6235 listcatfiles
= ZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6236 if(not listcatfiles
):
6238 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6239 lenlist
= len(listcatfiles
['ffilelist']);
6241 lcfx
= int(listcatfiles
['fnumfiles']);
6242 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6243 lcfx
= int(lenlist
);
6245 lcfx
= int(listcatfiles
['fnumfiles']);
6247 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6248 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6249 catarray
['filetoid'].update(filetoidarray
);
6250 catarray
['idtofile'].update(idtofilearray
);
6251 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6252 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6253 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6254 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6255 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6256 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6257 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6258 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6259 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6260 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6261 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6262 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6263 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6264 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6265 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6266 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6267 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6268 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6269 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6270 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6271 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6272 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6273 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6274 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6275 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6276 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6277 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6278 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6279 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6280 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6281 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6285 if(not rarfile_support
):
6286 def RarFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6289 if(rarfile_support
):
6290 def RarFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6291 formatspecs
= FormatSpecsListToDict(formatspecs
);
6292 listcatfiles
= RarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6293 if(not listcatfiles
):
6295 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6296 lenlist
= len(listcatfiles
['ffilelist']);
6298 lcfx
= int(listcatfiles
['fnumfiles']);
6299 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6300 lcfx
= int(lenlist
);
6302 lcfx
= int(listcatfiles
['fnumfiles']);
6304 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6305 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6306 catarray
['filetoid'].update(filetoidarray
);
6307 catarray
['idtofile'].update(idtofilearray
);
6308 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6309 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6310 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6311 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6312 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6313 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6314 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6315 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6316 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6317 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6318 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6319 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6320 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6321 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6322 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6323 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6324 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6325 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6326 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6327 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6328 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6329 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6330 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6331 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6332 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6333 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6334 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6335 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6336 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6337 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6338 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6342 if(not py7zr_support
):
6343 def SevenZipFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6347 def SevenZipFileToArrayIndexAlt(infile
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6348 formatspecs
= FormatSpecsListToDict(formatspecs
);
6349 listcatfiles
= SevenZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6350 if(not listcatfiles
):
6352 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6353 lenlist
= len(listcatfiles
['ffilelist']);
6355 lcfx
= int(listcatfiles
['fnumfiles']);
6356 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6357 lcfx
= int(lenlist
);
6359 lcfx
= int(listcatfiles
['fnumfiles']);
6361 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6362 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6363 catarray
['filetoid'].update(filetoidarray
);
6364 catarray
['idtofile'].update(idtofilearray
);
6365 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6366 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6367 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6368 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6369 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6370 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6371 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6372 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6373 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6374 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6375 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6376 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6377 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6378 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6379 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6380 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6381 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6382 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6383 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6384 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6385 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6386 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6387 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6388 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6389 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6390 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6391 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6392 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6393 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6394 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6395 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6399 def ArchiveFileStringToArrayIndex(catstr
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6400 formatspecs
= FormatSpecsListToDict(formatspecs
);
6401 catfp
= BytesIO(catstr
);
6402 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
, returnfp
);
6403 return listcatfiles
;
6405 create_alias_function("", __file_format_name__
, "StringToArrayIndex", ArchiveFileStringToArrayIndex
);
6407 def TarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6408 formatspecs
= FormatSpecsListToDict(formatspecs
);
6410 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
6411 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6412 return listcatfiles
;
6414 def ZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6415 formatspecs
= FormatSpecsListToDict(formatspecs
);
6417 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
6418 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6419 return listcatfiles
;
6421 if(not rarfile_support
):
6422 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6425 if(rarfile_support
):
6426 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6427 formatspecs
= FormatSpecsListToDict(formatspecs
);
6429 catfp
= PackArchiveFileFromRarFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
6430 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6431 return listcatfiles
;
6433 if(not py7zr_support
):
6434 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6438 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6439 formatspecs
= FormatSpecsListToDict(formatspecs
);
6441 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
6442 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6443 return listcatfiles
;
6445 def ListDirToArrayIndex(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6446 formatspecs
= FormatSpecsListToDict(formatspecs
);
6447 outarray
= BytesIO();
6448 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
, True);
6449 listcatfiles
= ArchiveFileToArrayIndex(outarray
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
)
6450 return listcatfiles
;
6452 def RePackArchiveFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6453 formatspecs
= FormatSpecsListToDict(formatspecs
);
6454 if(isinstance(infile
, dict)):
6455 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6456 listcatfiles
= prelistcatfiles
['list'];
6458 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6459 infile
= RemoveWindowsPath(infile
);
6461 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6462 listcatfiles
= prelistcatfiles
['list'];
6464 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6465 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
6466 outfile
= RemoveWindowsPath(outfile
);
6467 checksumtype
= checksumtype
.lower();
6468 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
6469 checksumtype
="crc32";
6470 if(checksumtype
=="none"):
6472 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
6473 compression
= "auto";
6474 if(compression
not in compressionlist
and compression
is None):
6475 compression
= "auto";
6477 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6478 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
6479 if(os
.path
.exists(outfile
)):
6482 except OSError as e
:
6484 if(not listcatfiles
):
6489 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6491 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6494 fbasename
= os
.path
.splitext(outfile
)[0];
6495 fextname
= os
.path
.splitext(outfile
)[1];
6496 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
6497 catver
= formatspecs
['format_ver'];
6498 fileheaderver
= str(int(catver
.replace(".", "")));
6499 lenlist
= len(listcatfiles
['ffilelist']);
6500 fnumfiles
= int(listcatfiles
['fnumfiles']);
6501 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
6502 fnumfiles
= lenlist
;
6503 AppendFileHeader(catfp
, fnumfiles
, checksumtype
, formatspecs
);
6504 lenlist
= len(listcatfiles
['ffilelist']);
6505 fnumfiles
= int(listcatfiles
['fnumfiles']);
6507 lcfx
= int(listcatfiles
['fnumfiles']);
6508 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6509 lcfx
= int(lenlist
);
6511 lcfx
= int(listcatfiles
['fnumfiles']);
6519 if(re
.findall("^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
6520 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
6522 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
6524 VerbosePrintOut(fname
);
6525 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
6526 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
6527 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6528 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
6529 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
6530 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
6531 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
6532 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
6533 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
6534 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
6535 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6536 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6537 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6538 finode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['finode']), 'x').lower();
6539 flinkcount
= format(int(listcatfiles
['ffilelist'][reallcfi
]['flinkcount']), 'x').lower();
6540 fwinattributes
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fwinattributes']), 'x').lower();
6541 fcompression
= listcatfiles
['ffilelist'][reallcfi
]['fcompression'];
6542 fcsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fcsize']), 'x').lower();
6543 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6544 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6545 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6546 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6547 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6548 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6549 if(not followlink
and len(extradata
)<0):
6550 extradata
= listcatfiles
['ffilelist'][reallcfi
]['fextralist'];
6551 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6553 fcsize
= format(int(0), 'x').lower();
6554 if(not compresswholefile
):
6555 fcontents
.seek(0, 2);
6556 ucfsize
= fcontents
.tell();
6557 fcontents
.seek(0, 0);
6558 if(compression
=="auto"):
6559 ilsize
= len(compressionlistalt
);
6562 while(ilmin
< ilsize
):
6563 cfcontents
= BytesIO();
6564 shutil
.copyfileobj(fcontents
, cfcontents
);
6565 fcontents
.seek(0, 0);
6566 cfcontents
.seek(0, 0);
6567 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
6569 cfcontents
.seek(0, 2);
6570 ilcsize
.append(cfcontents
.tell());
6574 ilcsize
.append(sys
.maxint
);
6575 except AttributeError:
6576 ilcsize
.append(sys
.maxsize
);
6578 ilcmin
= ilcsize
.index(min(ilcsize
));
6579 compression
= compressionlistalt
[ilcmin
];
6580 fcontents
.seek(0, 0);
6581 cfcontents
= BytesIO();
6582 shutil
.copyfileobj(fcontents
, cfcontents
);
6583 cfcontents
.seek(0, 0);
6584 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
6585 cfcontents
.seek(0, 2);
6586 cfsize
= cfcontents
.tell();
6587 if(ucfsize
> cfsize
):
6588 fcsize
= format(int(cfsize
), 'x').lower();
6589 fcompression
= compression
;
6591 fcontents
= cfcontents
;
6593 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6594 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6595 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6596 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6597 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6598 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6599 flinkname
= flinkinfo
['flinkname'];
6600 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6601 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6602 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6603 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6604 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6605 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6606 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6607 funame
= flinkinfo
['funame'];
6608 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6609 fgname
= flinkinfo
['fgname'];
6610 finode
= format(int(flinkinfo
['finode']), 'x').lower();
6611 flinkcount
= format(int(flinkinfo
['flinkcount']), 'x').lower();
6612 fwinattributes
= format(int(flinkinfo
['fwinattributes']), 'x').lower();
6613 fcompression
= flinkinfo
['fcompression'];
6614 fcsize
= format(int(flinkinfo
['fcsize']), 'x').lower();
6615 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6616 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6617 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6618 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6619 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6620 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6621 if(len(extradata
)<0):
6622 extradata
= flinkinfo
['fextralist'];
6623 fcontents
= flinkinfo
['fcontents'];
6624 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6626 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6627 fcurfid
= format(curfid
, 'x').lower();
6628 if(not followlink
and finode
!=0):
6629 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6630 fcurinode
= format(int(curinode
), 'x').lower();
6631 inodetofile
.update({curinode
: fname
});
6632 filetoinode
.update({fname
: curinode
});
6633 curinode
= curinode
+ 1;
6635 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6637 fcurinode
= format(int(curinode
), 'x').lower();
6638 curinode
= curinode
+ 1;
6639 curfid
= curfid
+ 1;
6640 if(fcompression
=="none"):
6642 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
6643 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
6646 reallcfi
= reallcfi
+ 1;
6648 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
6649 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6650 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6653 os
.fsync(catfp
.fileno());
6654 except io
.UnsupportedOperation
:
6656 except AttributeError:
6658 except OSError as e
:
6662 if(hasattr(sys
.stdout
, "buffer")):
6663 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6665 shutil
.copyfileobj(catfp
, sys
.stdout
);
6666 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6667 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6669 upload_file_to_internet_file(catfp
, outfile
);
6677 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6679 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6680 formatspecs
= FormatSpecsListToDict(formatspecs
);
6681 catfp
= BytesIO(catstr
);
6682 listcatfiles
= RePackArchiveFile(catfp
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6683 return listcatfiles
;
6685 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6687 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6688 formatspecs
= FormatSpecsListToDict(formatspecs
);
6689 outarray
= BytesIO();
6690 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6691 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6692 return listcatfiles
;
6694 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6696 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6697 formatspecs
= FormatSpecsListToDict(formatspecs
);
6698 if(outdir
is not None):
6699 outdir
= RemoveWindowsPath(outdir
);
6701 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6702 if(isinstance(infile
, dict)):
6703 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6704 listcatfiles
= prelistcatfiles
['list'];
6706 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6707 infile
= RemoveWindowsPath(infile
);
6709 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6710 listcatfiles
= prelistcatfiles
['list'];
6712 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6713 if(not listcatfiles
):
6715 lenlist
= len(listcatfiles
['ffilelist']);
6716 fnumfiles
= int(listcatfiles
['fnumfiles']);
6718 lcfx
= int(listcatfiles
['fnumfiles']);
6719 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6720 lcfx
= int(lenlist
);
6722 lcfx
= int(listcatfiles
['fnumfiles']);
6728 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6729 funame
= userinfo
.pw_name
;
6738 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6739 fgname
= groupinfo
.gr_name
;
6745 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6746 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6747 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6748 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6749 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6752 os
.fsync(fpc
.fileno());
6753 except io
.UnsupportedOperation
:
6755 except AttributeError:
6757 except OSError as e
:
6759 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6760 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6761 if(preservepermissions
):
6762 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6764 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6765 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6767 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6768 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6769 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6774 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6775 funame
= userinfo
.pw_name
;
6784 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6785 fgname
= groupinfo
.gr_name
;
6790 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6791 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6792 flinkinfo
['fcontents'].seek(0, 0);
6793 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6796 os
.fsync(fpc
.fileno());
6797 except io
.UnsupportedOperation
:
6799 except AttributeError:
6801 except OSError as e
:
6803 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6804 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6805 if(preservepermissions
):
6806 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6808 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6809 if(flinkinfo
['ftype']==1):
6810 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6811 if(flinkinfo
['ftype']==2):
6812 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6813 if(flinkinfo
['ftype']==5):
6814 if(preservepermissions
):
6815 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6817 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6818 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6819 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6820 if(preservepermissions
):
6821 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6823 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6824 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6825 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6827 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6828 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6830 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6831 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6832 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6837 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6838 funame
= userinfo
.pw_name
;
6847 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6848 fgname
= groupinfo
.gr_name
;
6853 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6854 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6855 flinkinfo
['fcontents'].seek(0, 0);
6856 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6859 os
.fsync(fpc
.fileno());
6860 except io
.UnsupportedOperation
:
6862 except AttributeError:
6864 except OSError as e
:
6866 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6867 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6868 if(preservepermissions
):
6869 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6871 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6872 if(flinkinfo
['ftype']==1):
6873 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6874 if(flinkinfo
['ftype']==2):
6875 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6876 if(flinkinfo
['ftype']==5):
6877 if(preservepermissions
):
6878 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6880 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6881 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6882 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6883 if(preservepermissions
):
6884 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6886 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6887 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6888 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6890 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6891 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6892 if(preservepermissions
):
6893 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6895 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6896 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6897 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6898 if(preservepermissions
):
6899 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6901 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6902 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6903 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6906 return listcatfiles
['ffilelist']['catfp'];
6910 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6912 if(hasattr(shutil
, "register_unpack_format")):
6913 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6914 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_dict__
['format_delimiter'], False, False);
6915 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6917 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6918 formatspecs
= FormatSpecsListToDict(formatspecs
);
6919 catfp
= BytesIO(catstr
);
6920 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6921 return listcatfiles
;
6923 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6925 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6926 formatspecs
= FormatSpecsListToDict(formatspecs
);
6927 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6928 if(isinstance(infile
, dict)):
6929 listcatfiles
= infile
;
6931 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6932 infile
= RemoveWindowsPath(infile
);
6933 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, False, skipchecksum
, formatspecs
, returnfp
);
6934 if(not listcatfiles
):
6936 lenlist
= len(listcatfiles
['ffilelist']);
6937 fnumfiles
= int(listcatfiles
['fnumfiles']);
6939 lcfx
= int(listcatfiles
['fnumfiles']);
6940 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6941 lcfx
= int(lenlist
);
6943 lcfx
= int(listcatfiles
['fnumfiles']);
6946 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6948 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6950 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6951 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6952 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6953 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6954 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6955 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6956 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6957 if(len(fuprint
)<=0):
6958 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6959 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6960 if(len(fgprint
)<=0):
6961 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6962 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6965 return listcatfiles
['catfp'];
6969 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6971 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6972 formatspecs
= FormatSpecsListToDict(formatspecs
);
6973 catfp
= BytesIO(catstr
);
6974 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6975 return listcatfiles
;
6977 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6979 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6980 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6983 if(hasattr(sys
.stdin
, "buffer")):
6984 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6986 shutil
.copyfileobj(sys
.stdin
, infile
);
6991 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6992 infile
= download_file_from_internet_file(infile
);
6997 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6999 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
7001 if(not tarfile
.is_tarfile(infile
)):
7003 except AttributeError:
7004 if(not is_tarfile(infile
)):
7009 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
7010 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
7012 tarfp
= tarfile
.open(infile
, "r");
7013 except FileNotFoundError
:
7017 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
7018 returnval
.update({lcfi
: member
.name
});
7019 fpremode
= member
.mode
;
7020 ffullmode
= member
.mode
;
7024 ffullmode
= member
.mode
+ stat
.S_IFREG
;
7026 elif(member
.isdev()):
7027 ffullmode
= member
.mode
;
7029 elif(member
.islnk()):
7030 ffullmode
= member
.mode
+ stat
.S_IFREG
;
7032 elif(member
.issym()):
7033 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
7035 elif(member
.ischr()):
7036 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
7038 elif(member
.isblk()):
7039 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
7041 elif(member
.isdir()):
7042 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
7044 elif(member
.isfifo()):
7045 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
7047 elif(member
.issparse()):
7048 ffullmode
= member
.mode
;
7051 VerbosePrintOut(member
.name
);
7053 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7054 printfname
= member
.name
;
7056 printfname
= member
.name
+ " link to " + member
.linkname
;
7057 elif(member
.issym()):
7058 printfname
= member
.name
+ " -> " + member
.linkname
;
7059 fuprint
= member
.uname
;
7060 if(len(fuprint
)<=0):
7061 fuprint
= member
.uid
;
7062 fgprint
= member
.gname
;
7063 if(len(fgprint
)<=0):
7064 fgprint
= member
.gid
;
7065 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7068 return listcatfiles
['catfp'];
7072 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7073 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7076 if(hasattr(sys
.stdin
, "buffer")):
7077 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
7079 shutil
.copyfileobj(sys
.stdin
, infile
);
7084 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
7085 infile
= download_file_from_internet_file(infile
);
7090 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7094 if(not zipfile
.is_zipfile(infile
)):
7097 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
7098 except FileNotFoundError
:
7102 ziptest
= zipfp
.testzip();
7104 VerbosePrintOut("Bad file found!");
7105 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
7106 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
7107 fwinattributes
= int(zipinfo
.external_attr
);
7108 if(not member
.is_dir()):
7109 fmode
= int(stat
.S_IFREG
+ 438);
7110 fchmode
= int(stat
.S_IMODE(fmode
));
7111 ftypemod
= int(stat
.S_IFMT(fmode
));
7112 elif(member
.is_dir()):
7113 fmode
= int(stat
.S_IFDIR
+ 511);
7114 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7115 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7116 elif(zipinfo
.create_system
==3):
7117 fwinattributes
=int(0);
7118 fmode
= int(zipinfo
.external_attr
);
7119 fchmode
= int(stat
.S_IMODE(fmode
));
7120 ftypemod
= int(stat
.S_IFMT(fmode
));
7122 fwinattributes
= int(0);
7123 if(not member
.is_dir()):
7124 fmode
= int(stat
.S_IFREG
+ 438);
7125 fchmode
= int(stat
.S_IMODE(fmode
));
7126 ftypemod
= int(stat
.S_IFMT(fmode
));
7127 elif(member
.is_dir()):
7128 fmode
= int(stat
.S_IFDIR
+ 511);
7129 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7130 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7131 returnval
.update({lcfi
: member
.filename
});
7133 VerbosePrintOut(member
.filename
);
7135 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7137 for fmodval
in str(oct(fmode
))[-3:]:
7138 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7139 if(not member
.is_dir()):
7141 permissionstr
= "-" + permissionstr
;
7142 elif(member
.is_dir()):
7144 permissionstr
= "d" + permissionstr
;
7145 printfname
= member
.filename
;
7147 fuid
= int(os
.getuid());
7148 except AttributeError:
7153 fgid
= int(os
.getgid());
7154 except AttributeError:
7161 userinfo
= pwd
.getpwuid(os
.getuid());
7162 funame
= userinfo
.pw_name
;
7165 except AttributeError:
7173 groupinfo
= grp
.getgrgid(os
.getgid());
7174 fgname
= groupinfo
.gr_name
;
7177 except AttributeError:
7182 if(len(fuprint
)<=0):
7183 fuprint
= str(fuid
);
7185 if(len(fgprint
)<=0):
7186 fgprint
= str(fgid
);
7187 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7190 return listcatfiles
['catfp'];
7194 if(not rarfile_support
):
7195 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
7196 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7197 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7200 if(rarfile_support
):
7201 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
7202 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7203 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7205 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
7209 rarfp
= rarfile
.RarFile(infile
, "r");
7210 rartest
= rarfp
.testrar();
7212 VerbosePrintOut("Bad file found!");
7213 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
7216 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
7219 member
.external_attr
7221 except AttributeError:
7223 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
7226 member
.external_attr
7228 except AttributeError:
7233 if(is_unix
and member
.external_attr
!=0):
7234 fpremode
= int(member
.external_attr
);
7235 elif(member
.is_file()):
7236 fpremode
= int(stat
.S_IFREG
+ 438);
7237 elif(member
.is_symlink()):
7238 fpremode
= int(stat
.S_IFLNK
+ 438);
7239 elif(member
.is_dir()):
7240 fpremode
= int(stat
.S_IFDIR
+ 511);
7241 if(is_windows
and member
.external_attr
!=0):
7242 fwinattributes
= int(member
.external_attr
);
7244 fwinattributes
= int(0);
7245 if(is_unix
and member
.external_attr
!=0):
7246 fmode
= int(member
.external_attr
);
7247 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
7248 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
7249 elif(member
.is_file()):
7250 fmode
= int(stat
.S_IFREG
+ 438);
7251 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
7252 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
7253 elif(member
.is_symlink()):
7254 fmode
= int(stat
.S_IFLNK
+ 438);
7255 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7256 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7257 elif(member
.is_dir()):
7258 fmode
= int(stat
.S_IFDIR
+ 511);
7259 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7260 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7261 returnval
.update({lcfi
: member
.filename
});
7263 VerbosePrintOut(member
.filename
);
7265 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7267 for fmodval
in str(oct(fmode
))[-3:]:
7268 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7269 if(member
.is_file()):
7271 permissionstr
= "-" + permissionstr
;
7272 printfname
= member
.filename
;
7273 elif(member
.is_symlink()):
7275 permissionstr
= "l" + permissionstr
;
7276 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
7277 elif(member
.is_dir()):
7279 permissionstr
= "d" + permissionstr
;
7280 printfname
= member
.filename
;
7282 fuid
= int(os
.getuid());
7283 except AttributeError:
7288 fgid
= int(os
.getgid());
7289 except AttributeError:
7296 userinfo
= pwd
.getpwuid(os
.getuid());
7297 funame
= userinfo
.pw_name
;
7300 except AttributeError:
7308 groupinfo
= grp
.getgrgid(os
.getgid());
7309 fgname
= groupinfo
.gr_name
;
7312 except AttributeError:
7317 if(len(fuprint
)<=0):
7318 fuprint
= str(fuid
);
7320 if(len(fgprint
)<=0):
7321 fgprint
= str(fgid
);
7322 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7325 return listcatfiles
['catfp'];
7329 if(not py7zr_support
):
7330 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7331 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7332 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7336 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7337 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7338 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7342 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
7343 file_content
= szpfp
.readall();
7344 #sztest = szpfp.testzip();
7345 sztestalt
= szpfp
.test();
7347 VerbosePrintOut("Bad file found!");
7348 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
7349 if(re
.findall("^[.|/]", member
.filename
)):
7350 fname
= member
.filename
;
7352 fname
= "./"+member
.filename
;
7353 if(not member
.is_directory
):
7354 fpremode
= int(stat
.S_IFREG
+ 438);
7355 elif(member
.is_directory
):
7356 fpremode
= int(stat
.S_IFDIR
+ 511);
7357 fwinattributes
= int(0);
7358 if(member
.is_directory
):
7359 fmode
= int(stat
.S_IFDIR
+ 511);
7360 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7361 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7363 fmode
= int(stat
.S_IFLNK
+ 438);
7364 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7365 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7366 returnval
.update({lcfi
: member
.filename
});
7368 VerbosePrintOut(member
.filename
);
7370 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7372 for fmodval
in str(oct(fmode
))[-3:]:
7373 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7375 if(not member
.is_directory
):
7377 permissionstr
= "-" + permissionstr
;
7378 printfname
= member
.filename
;
7379 elif(member
.is_directory
):
7381 permissionstr
= "d" + permissionstr
;
7382 printfname
= member
.filename
;
7384 fsize
= len(file_content
[member
.filename
].read());
7385 file_content
[member
.filename
].close();
7387 fuid
= int(os
.getuid());
7388 except AttributeError:
7393 fgid
= int(os
.getgid());
7394 except AttributeError:
7401 userinfo
= pwd
.getpwuid(os
.getuid());
7402 funame
= userinfo
.pw_name
;
7405 except AttributeError:
7413 groupinfo
= grp
.getgrgid(os
.getgid());
7414 fgname
= groupinfo
.gr_name
;
7417 except AttributeError:
7422 if(len(fuprint
)<=0):
7423 fuprint
= str(fuid
);
7425 if(len(fgprint
)<=0):
7426 fgprint
= str(fgid
);
7427 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7430 return listcatfiles
['catfp'];
7434 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
7435 formatspecs
= FormatSpecsListToDict(formatspecs
);
7436 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7437 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
7438 if(checkcompressfile
=="tarfile"):
7439 return TarFileListFiles(infile
, verbose
, returnfp
);
7440 elif(checkcompressfile
=="zipfile"):
7441 return ZipFileListFiles(infile
, verbose
, returnfp
);
7442 elif(checkcompressfile
=="catfile"):
7443 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
7444 elif(rarfile_support
and checkcompressfile
=="rarfile"):
7445 return RarFileListFiles(infile
, verbose
, returnfp
);
7446 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
7447 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
7452 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7453 formatspecs
= FormatSpecsListToDict(formatspecs
);
7454 outarray
= BytesIO();
7455 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
7456 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7457 return listcatfiles
;
7459 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7460 formatspecs
= FormatSpecsListToDict(formatspecs
);
7461 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
7462 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7463 return listcatfiles
;
7465 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7466 formatspecs
= FormatSpecsListToDict(formatspecs
);
7467 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
7468 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
7469 return listcatfiles
;
7471 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
7473 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7474 formatspecs
= FormatSpecsListToDict(formatspecs
);
7475 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7476 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7477 return listcatfiles
;
7479 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
7481 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7482 formatspecs
= FormatSpecsListToDict(formatspecs
);
7483 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7484 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7485 return listcatfiles
;
7487 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
7489 if(not rarfile_support
):
7490 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7493 if(rarfile_support
):
7494 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7495 formatspecs
= FormatSpecsListToDict(formatspecs
);
7496 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7497 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7498 return listcatfiles
;
7500 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
7502 if(not py7zr_support
):
7503 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7507 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7508 formatspecs
= FormatSpecsListToDict(formatspecs
);
7509 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7510 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7511 return listcatfiles
;
7513 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
7515 def download_file_from_ftp_file(url
):
7516 urlparts
= urlparse(url
);
7517 file_name
= os
.path
.basename(urlparts
.path
);
7518 file_dir
= os
.path
.dirname(urlparts
.path
);
7519 if(urlparts
.username
is not None):
7520 ftp_username
= urlparts
.username
;
7522 ftp_username
= "anonymous";
7523 if(urlparts
.password
is not None):
7524 ftp_password
= urlparts
.password
;
7525 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7526 ftp_password
= "anonymous";
7529 if(urlparts
.scheme
=="ftp"):
7531 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7535 if(urlparts
.scheme
=="sftp"):
7537 return download_file_from_pysftp_file(url
);
7539 return download_file_from_sftp_file(url
);
7540 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7541 return download_file_from_http_file(url
);
7542 ftp_port
= urlparts
.port
;
7543 if(urlparts
.port
is None):
7546 ftp
.connect(urlparts
.hostname
, ftp_port
);
7547 except socket
.gaierror
:
7548 log
.info("Error With URL "+url
);
7550 except socket
.timeout
:
7551 log
.info("Error With URL "+url
);
7553 ftp
.login(urlparts
.username
, urlparts
.password
);
7554 if(urlparts
.scheme
=="ftps"):
7556 ftpfile
= BytesIO();
7557 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7558 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7563 def download_file_from_ftp_string(url
):
7564 ftpfile
= download_file_from_ftp_file(url
);
7565 return ftpfile
.read();
7567 def upload_file_to_ftp_file(ftpfile
, url
):
7568 urlparts
= urlparse(url
);
7569 file_name
= os
.path
.basename(urlparts
.path
);
7570 file_dir
= os
.path
.dirname(urlparts
.path
);
7571 if(urlparts
.username
is not None):
7572 ftp_username
= urlparts
.username
;
7574 ftp_username
= "anonymous";
7575 if(urlparts
.password
is not None):
7576 ftp_password
= urlparts
.password
;
7577 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7578 ftp_password
= "anonymous";
7581 if(urlparts
.scheme
=="ftp"):
7583 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7587 if(urlparts
.scheme
=="sftp"):
7589 return upload_file_to_pysftp_file(url
);
7591 return upload_file_to_sftp_file(url
);
7592 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7594 ftp_port
= urlparts
.port
;
7595 if(urlparts
.port
is None):
7598 ftp
.connect(urlparts
.hostname
, ftp_port
);
7599 except socket
.gaierror
:
7600 log
.info("Error With URL "+url
);
7602 except socket
.timeout
:
7603 log
.info("Error With URL "+url
);
7605 ftp
.login(urlparts
.username
, urlparts
.password
);
7606 if(urlparts
.scheme
=="ftps"):
7608 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7613 def upload_file_to_ftp_string(ftpstring
, url
):
7614 ftpfileo
= BytesIO(ftpstring
);
7615 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7619 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7620 # Parse the URL to extract username and password if present
7621 urlparts
= urlparse(url
);
7622 username
= urlparts
.username
;
7623 password
= urlparts
.password
;
7624 # Rebuild the URL without the username and password
7625 netloc
= urlparts
.hostname
;
7626 if(urlparts
.scheme
=="sftp"):
7628 return download_file_from_pysftp_file(url
);
7630 return download_file_from_sftp_file(url
);
7631 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7632 return download_file_from_ftp_file(url
);
7634 netloc
+= ':' + str(urlparts
.port
);
7635 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7636 # Create a temporary file object
7637 httpfile
= BytesIO();
7639 # Use the requests library if available
7640 if username
and password
:
7641 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7643 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7644 response
.raw
.decode_content
= True
7645 shutil
.copyfileobj(response
.raw
, httpfile
);
7647 # Build a Request object for urllib
7648 request
= Request(rebuilt_url
, headers
=headers
);
7649 # Create an opener object for handling URLs
7650 if username
and password
:
7651 # Create a password manager
7652 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7653 # Add the username and password
7654 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7655 # Create an authentication handler using the password manager
7656 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7657 # Build the opener with the authentication handler
7658 opener
= build_opener(auth_handler
);
7660 opener
= build_opener();
7661 with opener
.open(request
) as response
:
7662 shutil
.copyfileobj(response
, httpfile
);
7663 # Reset file pointer to the start
7664 httpfile
.seek(0, 0);
7665 # Return the temporary file object
7668 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7669 httpfile
= download_file_from_http_file(url
, headers
);
7670 return ftpfile
.read();
7673 def download_file_from_sftp_file(url
):
7674 urlparts
= urlparse(url
);
7675 file_name
= os
.path
.basename(urlparts
.path
);
7676 file_dir
= os
.path
.dirname(urlparts
.path
);
7677 sftp_port
= urlparts
.port
;
7678 if(urlparts
.port
is None):
7681 sftp_port
= urlparts
.port
;
7682 if(urlparts
.username
is not None):
7683 sftp_username
= urlparts
.username
;
7685 sftp_username
= "anonymous";
7686 if(urlparts
.password
is not None):
7687 sftp_password
= urlparts
.password
;
7688 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7689 sftp_password
= "anonymous";
7692 if(urlparts
.scheme
=="ftp"):
7693 return download_file_from_ftp_file(url
);
7694 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7695 return download_file_from_http_file(url
);
7696 if(urlparts
.scheme
!="sftp"):
7698 ssh
= paramiko
.SSHClient();
7699 ssh
.load_system_host_keys();
7700 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7702 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7703 except paramiko
.ssh_exception
.SSHException
:
7705 except socket
.gaierror
:
7706 log
.info("Error With URL "+url
);
7708 except socket
.timeout
:
7709 log
.info("Error With URL "+url
);
7711 sftp
= ssh
.open_sftp();
7712 sftpfile
= BytesIO();
7713 sftp
.getfo(urlparts
.path
, sftpfile
);
7716 sftpfile
.seek(0, 0);
7719 def download_file_from_sftp_file(url
):
7723 def download_file_from_sftp_string(url
):
7724 sftpfile
= download_file_from_sftp_file(url
);
7725 return sftpfile
.read();
7727 def download_file_from_ftp_string(url
):
7731 def upload_file_to_sftp_file(sftpfile
, url
):
7732 urlparts
= urlparse(url
);
7733 file_name
= os
.path
.basename(urlparts
.path
);
7734 file_dir
= os
.path
.dirname(urlparts
.path
);
7735 sftp_port
= urlparts
.port
;
7736 if(urlparts
.port
is None):
7739 sftp_port
= urlparts
.port
;
7740 if(urlparts
.username
is not None):
7741 sftp_username
= urlparts
.username
;
7743 sftp_username
= "anonymous";
7744 if(urlparts
.password
is not None):
7745 sftp_password
= urlparts
.password
;
7746 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7747 sftp_password
= "anonymous";
7750 if(urlparts
.scheme
=="ftp"):
7751 return upload_file_to_ftp_file(url
);
7752 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7754 if(urlparts
.scheme
!="sftp"):
7756 ssh
= paramiko
.SSHClient();
7757 ssh
.load_system_host_keys();
7758 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7760 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7761 except paramiko
.ssh_exception
.SSHException
:
7763 except socket
.gaierror
:
7764 log
.info("Error With URL "+url
);
7766 except socket
.timeout
:
7767 log
.info("Error With URL "+url
);
7769 sftp
= ssh
.open_sftp();
7770 sftp
.putfo(sftpfile
, urlparts
.path
);
7773 sftpfile
.seek(0, 0);
7776 def upload_file_to_sftp_file(sftpfile
, url
):
7780 def upload_file_to_sftp_string(sftpstring
, url
):
7781 sftpfileo
= BytesIO(sftpstring
);
7782 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7786 def upload_file_to_sftp_string(url
):
7790 def download_file_from_pysftp_file(url
):
7791 urlparts
= urlparse(url
);
7792 file_name
= os
.path
.basename(urlparts
.path
);
7793 file_dir
= os
.path
.dirname(urlparts
.path
);
7794 sftp_port
= urlparts
.port
;
7795 if(urlparts
.port
is None):
7798 sftp_port
= urlparts
.port
;
7799 if(urlparts
.username
is not None):
7800 sftp_username
= urlparts
.username
;
7802 sftp_username
= "anonymous";
7803 if(urlparts
.password
is not None):
7804 sftp_password
= urlparts
.password
;
7805 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7806 sftp_password
= "anonymous";
7809 if(urlparts
.scheme
=="ftp"):
7810 return download_file_from_ftp_file(url
);
7811 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7812 return download_file_from_http_file(url
);
7813 if(urlparts
.scheme
!="sftp"):
7816 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7817 except paramiko
.ssh_exception
.SSHException
:
7819 except socket
.gaierror
:
7820 log
.info("Error With URL "+url
);
7822 except socket
.timeout
:
7823 log
.info("Error With URL "+url
);
7825 sftp
= ssh
.open_sftp();
7826 sftpfile
= BytesIO();
7827 sftp
.getfo(urlparts
.path
, sftpfile
);
7830 sftpfile
.seek(0, 0);
7833 def download_file_from_pysftp_file(url
):
7837 def download_file_from_pysftp_string(url
):
7838 sftpfile
= download_file_from_pysftp_file(url
);
7839 return sftpfile
.read();
7841 def download_file_from_ftp_string(url
):
7845 def upload_file_to_pysftp_file(sftpfile
, url
):
7846 urlparts
= urlparse(url
);
7847 file_name
= os
.path
.basename(urlparts
.path
);
7848 file_dir
= os
.path
.dirname(urlparts
.path
);
7849 sftp_port
= urlparts
.port
;
7850 if(urlparts
.port
is None):
7853 sftp_port
= urlparts
.port
;
7854 if(urlparts
.username
is not None):
7855 sftp_username
= urlparts
.username
;
7857 sftp_username
= "anonymous";
7858 if(urlparts
.password
is not None):
7859 sftp_password
= urlparts
.password
;
7860 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7861 sftp_password
= "anonymous";
7864 if(urlparts
.scheme
=="ftp"):
7865 return upload_file_to_ftp_file(url
);
7866 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7868 if(urlparts
.scheme
!="sftp"):
7871 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7872 except paramiko
.ssh_exception
.SSHException
:
7874 except socket
.gaierror
:
7875 log
.info("Error With URL "+url
);
7877 except socket
.timeout
:
7878 log
.info("Error With URL "+url
);
7880 sftp
= ssh
.open_sftp();
7881 sftp
.putfo(sftpfile
, urlparts
.path
);
7884 sftpfile
.seek(0, 0);
7887 def upload_file_to_pysftp_file(sftpfile
, url
):
7891 def upload_file_to_pysftp_string(sftpstring
, url
):
7892 sftpfileo
= BytesIO(sftpstring
);
7893 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7897 def upload_file_to_pysftp_string(url
):
7900 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7901 urlparts
= urlparse(url
);
7902 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7903 return download_file_from_http_file(url
, headers
);
7904 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7905 return download_file_from_ftp_file(url
);
7906 elif(urlparts
.scheme
=="sftp"):
7907 if(__use_pysftp__
and havepysftp
):
7908 return download_file_from_pysftp_file(url
);
7910 return download_file_from_sftp_file(url
);
7915 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7916 formatspecs
= FormatSpecsListToDict(formatspecs
);
7917 fp
= download_file_from_internet_file(url
);
7918 fp
= UncompressArchiveFile(fp
, formatspecs
);
7924 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7925 urlparts
= urlparse(url
);
7926 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7927 return download_file_from_http_string(url
, headers
);
7928 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7929 return download_file_from_ftp_string(url
);
7930 elif(urlparts
.scheme
=="sftp"):
7931 if(__use_pysftp__
and havepysftp
):
7932 return download_file_from_pysftp_string(url
);
7934 return download_file_from_sftp_string(url
);
7939 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7940 formatspecs
= FormatSpecsListToDict(formatspecs
);
7941 fp
= download_file_from_internet_string(url
);
7942 fp
= UncompressArchiveFile(fp
, formatspecs
);
7948 def upload_file_to_internet_file(ifp
, url
):
7949 urlparts
= urlparse(url
);
7950 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7952 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7953 return upload_file_to_ftp_file(ifp
, url
);
7954 elif(urlparts
.scheme
=="sftp"):
7955 if(__use_pysftp__
and havepysftp
):
7956 return upload_file_to_pysftp_file(ifp
, url
);
7958 return upload_file_to_sftp_file(ifp
, url
);
7963 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_dict__
):
7964 formatspecs
= FormatSpecsListToDict(formatspecs
);
7965 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
7969 upload_file_to_internet_file(catfp
, outfile
);
7972 def upload_file_to_internet_string(ifp
, url
):
7973 urlparts
= urlparse(url
);
7974 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7976 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7977 return upload_file_to_ftp_string(ifp
, url
);
7978 elif(urlparts
.scheme
=="sftp"):
7979 if(__use_pysftp__
and havepysftp
):
7980 return upload_file_to_pysftp_string(ifp
, url
);
7982 return upload_file_to_sftp_string(ifp
, url
);
7987 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_dict__
):
7988 formatspecs
= FormatSpecsListToDict(formatspecs
);
7989 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, compressionlevel
, formatspecs
);
7993 upload_file_to_internet_file(catfp
, outfile
);
7997 if(hasattr(shutil
, "register_archive_format")):
7998 # Register the packing format
7999 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
8000 except shutil
.RegistryError
:
8004 if(hasattr(shutil
, "register_unpack_format")):
8005 # Register the unpacking format
8006 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
8007 except shutil
.RegistryError
: