4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
15 $FileInfo: pywwwgetold.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
18 from __future__
import division
, absolute_import
, print_function
;
19 import re
, os
, sys
, hashlib
, shutil
, platform
, tempfile
, urllib
, zlib
, time
, argparse
, cgi
, subprocess
, socket
, email
.utils
, datetime
, time
;
20 import logging
as log
;
21 from ftplib
import FTP
, FTP_TLS
;
22 from base64
import b64encode
;
29 havemechanize
= False;
34 havemechanize
= False;
62 from httplib2
import HTTPConnectionWithTimeout
, HTTPSConnectionWithTimeout
;
90 if(sys
.version
[0]=="2"):
92 from io
import StringIO
, BytesIO
;
95 from cStringIO
import StringIO
;
96 from cStringIO
import StringIO
as BytesIO
;
98 from StringIO
import StringIO
;
99 from StringIO
import StringIO
as BytesIO
;
100 # From http://python-future.org/compatible_idioms.html
101 from urlparse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
;
102 from urllib
import urlencode
;
103 from urllib
import urlopen
as urlopenalt
;
104 from urllib2
import urlopen
, Request
, install_opener
, HTTPError
, URLError
, build_opener
, HTTPCookieProcessor
;
105 import urlparse
, cookielib
;
106 from httplib
import HTTPConnection
, HTTPSConnection
;
107 if(sys
.version
[0]>="3"):
108 from io
import StringIO
, BytesIO
;
109 # From http://python-future.org/compatible_idioms.html
110 from urllib
.parse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
, urlencode
;
111 from urllib
.request
import urlopen
, Request
, install_opener
, build_opener
, HTTPCookieProcessor
;
112 from urllib
.error
import HTTPError
, URLError
;
113 import urllib
.parse
as urlparse
;
114 import http
.cookiejar
as cookielib
;
115 from http
.client
import HTTPConnection
, HTTPSConnection
;
117 __program_name__
= "PyWWW-Get";
118 __program_alt_name__
= "PyWWWGet";
119 __program_small_name__
= "wwwget";
120 __project__
= __program_name__
;
121 __project_url__
= "https://github.com/GameMaker2k/PyWWW-Get";
122 __version_info__
= (2, 0, 2, "RC 1", 1);
123 __version_date_info__
= (2023, 10, 5, "RC 1", 1);
124 __version_date__
= str(__version_date_info__
[0])+"."+str(__version_date_info__
[1]).zfill(2)+"."+str(__version_date_info__
[2]).zfill(2);
125 __revision__
= __version_info__
[3];
126 __revision_id__
= "$Id$";
127 if(__version_info__
[4] is not None):
128 __version_date_plusrc__
= __version_date__
+"-"+str(__version_date_info__
[4]);
129 if(__version_info__
[4] is None):
130 __version_date_plusrc__
= __version_date__
;
131 if(__version_info__
[3] is not None):
132 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2])+" "+str(__version_info__
[3]);
133 if(__version_info__
[3] is None):
134 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2]);
136 tmpfileprefix
= "py"+str(sys
.version_info
[0])+__program_small_name__
+str(__version_info__
[0])+"-";
138 pytempdir
= tempfile
.gettempdir();
140 PyBitness
= platform
.architecture();
141 if(PyBitness
=="32bit" or PyBitness
=="32"):
143 elif(PyBitness
=="64bit" or PyBitness
=="64"):
148 compression_supported
= "gzip, deflate";
149 if(havebrotli
and not havezstd
):
150 compression_supported
= "gzip, deflate, br";
151 elif(not havebrotli
and havezstd
):
152 compression_supported
= "gzip, deflate, zstd";
153 elif(havebrotli
and havezstd
):
154 compression_supported
= "gzip, deflate, zstd, br";
156 compression_supported
= "gzip, deflate";
158 geturls_cj
= cookielib
.CookieJar();
159 windowsNT4_ua_string
= "Windows NT 4.0";
160 windowsNT4_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
161 windows2k_ua_string
= "Windows NT 5.0";
162 windows2k_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
163 windowsXP_ua_string
= "Windows NT 5.1";
164 windowsXP_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
165 windowsXP64_ua_string
= "Windows NT 5.2; Win64; x64";
166 windowsXP64_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
167 windows7_ua_string
= "Windows NT 6.1; Win64; x64";
168 windows7_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
169 windows8_ua_string
= "Windows NT 6.2; Win64; x64";
170 windows8_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
171 windows81_ua_string
= "Windows NT 6.3; Win64; x64";
172 windows81_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
173 windows10_ua_string
= "Windows NT 10.0; Win64; x64";
174 windows10_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
175 windows11_ua_string
= "Windows NT 11.0; Win64; x64";
176 windows11_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
177 geturls_ua_firefox_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:109.0) Gecko/20100101 Firefox/117.0";
178 geturls_ua_seamonkey_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
179 geturls_ua_chrome_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
180 geturls_ua_chromium_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
181 geturls_ua_palemoon_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
182 geturls_ua_opera_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
183 geturls_ua_vivaldi_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
184 geturls_ua_internet_explorer_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; Trident/7.0; rv:11.0) like Gecko";
185 geturls_ua_microsoft_edge_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
186 geturls_ua_pywwwget_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
187 if(platform
.python_implementation()!=""):
188 py_implementation
= platform
.python_implementation();
189 if(platform
.python_implementation()==""):
190 py_implementation
= "Python";
191 geturls_ua_pywwwget_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
192 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
193 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
194 geturls_ua
= geturls_ua_firefox_windows7
;
195 geturls_headers_firefox_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
196 geturls_headers_seamonkey_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
197 geturls_headers_chrome_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
198 geturls_headers_chrome_windows7
.update(windows7_ua_addon
);
199 geturls_headers_chromium_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
200 geturls_headers_chromium_windows7
.update(windows7_ua_addon
);
201 geturls_headers_palemoon_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
202 geturls_headers_opera_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
203 geturls_headers_opera_windows7
.update(windows7_ua_addon
);
204 geturls_headers_vivaldi_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
205 geturls_headers_vivaldi_windows7
.update(windows7_ua_addon
);
206 geturls_headers_internet_explorer_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
207 geturls_headers_microsoft_edge_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
208 geturls_headers_microsoft_edge_windows7
.update(windows7_ua_addon
);
209 geturls_headers_pywwwget_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
210 geturls_headers_pywwwget_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
211 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
212 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers
= geturls_headers_firefox_windows7
;
214 geturls_download_sleep
= 0;
216 def verbose_printout(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
217 if(outtype
=="print" and dbgenable
):
220 elif(outtype
=="log" and dbgenable
):
221 logging
.info(dbgtxt
);
223 elif(outtype
=="warning" and dbgenable
):
224 logging
.warning(dbgtxt
);
226 elif(outtype
=="error" and dbgenable
):
227 logging
.error(dbgtxt
);
229 elif(outtype
=="critical" and dbgenable
):
230 logging
.critical(dbgtxt
);
232 elif(outtype
=="exception" and dbgenable
):
233 logging
.exception(dbgtxt
);
235 elif(outtype
=="logalt" and dbgenable
):
236 logging
.log(dgblevel
, dbgtxt
);
238 elif(outtype
=="debug" and dbgenable
):
239 logging
.debug(dbgtxt
);
247 def verbose_printout_return(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
248 dbgout
= verbose_printout(dbgtxt
, outtype
, dbgenable
, dgblevel
);
253 def add_url_param(url
, **params
):
255 parts
= list(urlparse
.urlsplit(url
));
256 d
= dict(cgi
.parse_qsl(parts
[n
])); # use cgi.parse_qs for list values
258 parts
[n
]=urlencode(d
);
259 return urlparse
.urlunsplit(parts
);
261 os
.environ
["PATH"] = os
.environ
["PATH"] + os
.pathsep
+ os
.path
.dirname(os
.path
.realpath(__file__
)) + os
.pathsep
+ os
.getcwd();
262 def which_exec(execfile):
263 for path
in os
.environ
["PATH"].split(":"):
264 if os
.path
.exists(path
+ "/" + execfile):
265 return path
+ "/" + execfile;
267 def listize(varlist
):
275 newlistreg
.update({ilx
: varlist
[il
]});
276 newlistrev
.update({varlist
[il
]: ilx
});
279 newlistfull
= {1: newlistreg
, 2: newlistrev
, 'reg': newlistreg
, 'rev': newlistrev
};
282 def twolistize(varlist
):
292 newlistnamereg
.update({ilx
: varlist
[il
][0].strip()});
293 newlistnamerev
.update({varlist
[il
][0].strip(): ilx
});
294 newlistdescreg
.update({ilx
: varlist
[il
][1].strip()});
295 newlistdescrev
.update({varlist
[il
][1].strip(): ilx
});
298 newlistnametmp
= {1: newlistnamereg
, 2: newlistnamerev
, 'reg': newlistnamereg
, 'rev': newlistnamerev
};
299 newlistdesctmp
= {1: newlistdescreg
, 2: newlistdescrev
, 'reg': newlistdescreg
, 'rev': newlistdescrev
};
300 newlistfull
= {1: newlistnametmp
, 2: newlistdesctmp
, 'name': newlistnametmp
, 'desc': newlistdesctmp
}
303 def arglistize(proexec
, *varlist
):
307 newarglist
= [proexec
];
309 if varlist
[il
][0] is not None:
310 newarglist
.append(varlist
[il
][0]);
311 if varlist
[il
][1] is not None:
312 newarglist
.append(varlist
[il
][1]);
316 def fix_header_names(header_dict
):
317 if(sys
.version
[0]=="2"):
318 header_dict
= {k
.title(): v
for k
, v
in header_dict
.iteritems()};
319 if(sys
.version
[0]>="3"):
320 header_dict
= {k
.title(): v
for k
, v
in header_dict
.items()};
323 # hms_string by ArcGIS Python Recipes
324 # https://arcpy.wordpress.com/2012/04/20/146/
325 def hms_string(sec_elapsed
):
326 h
= int(sec_elapsed
/ (60 * 60));
327 m
= int((sec_elapsed
% (60 * 60)) / 60);
328 s
= sec_elapsed
% 60.0;
329 return "{}:{:>02}:{:>05.2f}".format(h
, m
, s
);
331 # get_readable_size by Lipis
332 # http://stackoverflow.com/posts/14998888/revisions
333 def get_readable_size(bytes
, precision
=1, unit
="IEC"):
335 if(unit
!="IEC" and unit
!="SI"):
338 units
= [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
339 unitswos
= ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
342 units
= [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
343 unitswos
= ["B","kB","MB","GB","TB","PB","EB","ZB"];
348 if abs(bytes
) < unitsize
:
349 strformat
= "%3."+str(precision
)+"f%s";
350 pre_return_val
= (strformat
% (bytes
, unit
));
351 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
352 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
353 alt_return_val
= pre_return_val
.split();
354 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
357 strformat
= "%."+str(precision
)+"f%s";
358 pre_return_val
= (strformat
% (bytes
, "YiB"));
359 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
360 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
361 alt_return_val
= pre_return_val
.split();
362 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
365 def get_readable_size_from_file(infile
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
367 usehashtypes
= usehashtypes
.lower();
368 getfilesize
= os
.path
.getsize(infile
);
369 return_val
= get_readable_size(getfilesize
, precision
, unit
);
371 hashtypelist
= usehashtypes
.split(",");
372 openfile
= open(infile
, "rb");
373 filecontents
= openfile
.read();
376 listnumend
= len(hashtypelist
);
377 while(listnumcount
< listnumend
):
378 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
379 hashtypelistup
= hashtypelistlow
.upper();
380 filehash
= hashlib
.new(hashtypelistup
);
381 filehash
.update(filecontents
);
382 filegethash
= filehash
.hexdigest();
383 return_val
.update({hashtypelistup
: filegethash
});
387 def get_readable_size_from_string(instring
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
389 usehashtypes
= usehashtypes
.lower();
390 getfilesize
= len(instring
);
391 return_val
= get_readable_size(getfilesize
, precision
, unit
);
393 hashtypelist
= usehashtypes
.split(",");
395 listnumend
= len(hashtypelist
);
396 while(listnumcount
< listnumend
):
397 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
398 hashtypelistup
= hashtypelistlow
.upper();
399 filehash
= hashlib
.new(hashtypelistup
);
400 if(sys
.version
[0]=="2"):
401 filehash
.update(instring
);
402 if(sys
.version
[0]>="3"):
403 filehash
.update(instring
.encode('utf-8'));
404 filegethash
= filehash
.hexdigest();
405 return_val
.update({hashtypelistup
: filegethash
});
409 def http_status_to_reason(code
):
412 101: 'Switching Protocols',
417 203: 'Non-Authoritative Information',
419 205: 'Reset Content',
420 206: 'Partial Content',
422 208: 'Already Reported',
424 300: 'Multiple Choices',
425 301: 'Moved Permanently',
430 307: 'Temporary Redirect',
431 308: 'Permanent Redirect',
434 402: 'Payment Required',
437 405: 'Method Not Allowed',
438 406: 'Not Acceptable',
439 407: 'Proxy Authentication Required',
440 408: 'Request Timeout',
443 411: 'Length Required',
444 412: 'Precondition Failed',
445 413: 'Payload Too Large',
447 415: 'Unsupported Media Type',
448 416: 'Range Not Satisfiable',
449 417: 'Expectation Failed',
450 421: 'Misdirected Request',
451 422: 'Unprocessable Entity',
453 424: 'Failed Dependency',
454 426: 'Upgrade Required',
455 428: 'Precondition Required',
456 429: 'Too Many Requests',
457 431: 'Request Header Fields Too Large',
458 451: 'Unavailable For Legal Reasons',
459 500: 'Internal Server Error',
460 501: 'Not Implemented',
462 503: 'Service Unavailable',
463 504: 'Gateway Timeout',
464 505: 'HTTP Version Not Supported',
465 506: 'Variant Also Negotiates',
466 507: 'Insufficient Storage',
467 508: 'Loop Detected',
469 511: 'Network Authentication Required'
471 return reasons
.get(code
, 'Unknown Status Code');
473 def ftp_status_to_reason(code
):
475 110: 'Restart marker reply',
476 120: 'Service ready in nnn minutes',
477 125: 'Data connection already open; transfer starting',
478 150: 'File status okay; about to open data connection',
480 202: 'Command not implemented, superfluous at this site',
481 211: 'System status, or system help reply',
482 212: 'Directory status',
485 215: 'NAME system type',
486 220: 'Service ready for new user',
487 221: 'Service closing control connection',
488 225: 'Data connection open; no transfer in progress',
489 226: 'Closing data connection',
490 227: 'Entering Passive Mode',
491 230: 'User logged in, proceed',
492 250: 'Requested file action okay, completed',
493 257: '"PATHNAME" created',
494 331: 'User name okay, need password',
495 332: 'Need account for login',
496 350: 'Requested file action pending further information',
497 421: 'Service not available, closing control connection',
498 425: 'Can\'t open data connection',
499 426: 'Connection closed; transfer aborted',
500 450: 'Requested file action not taken',
501 451: 'Requested action aborted. Local error in processing',
502 452: 'Requested action not taken. Insufficient storage space in system',
503 500: 'Syntax error, command unrecognized',
504 501: 'Syntax error in parameters or arguments',
505 502: 'Command not implemented',
506 503: 'Bad sequence of commands',
507 504: 'Command not implemented for that parameter',
508 530: 'Not logged in',
509 532: 'Need account for storing files',
510 550: 'Requested action not taken. File unavailable',
511 551: 'Requested action aborted. Page type unknown',
512 552: 'Requested file action aborted. Exceeded storage allocation',
513 553: 'Requested action not taken. File name not allowed'
515 return reasons
.get(code
, 'Unknown Status Code');
517 def sftp_status_to_reason(code
):
521 2: 'SSH_FX_NO_SUCH_FILE',
522 3: 'SSH_FX_PERMISSION_DENIED',
524 5: 'SSH_FX_BAD_MESSAGE',
525 6: 'SSH_FX_NO_CONNECTION',
526 7: 'SSH_FX_CONNECTION_LOST',
527 8: 'SSH_FX_OP_UNSUPPORTED'
529 return reasons
.get(code
, 'Unknown Status Code');
531 def make_http_headers_from_dict_to_list(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
532 if isinstance(headers
, dict):
534 if(sys
.version
[0]=="2"):
535 for headkey
, headvalue
in headers
.iteritems():
536 returnval
.append((headkey
, headvalue
));
537 if(sys
.version
[0]>="3"):
538 for headkey
, headvalue
in headers
.items():
539 returnval
.append((headkey
, headvalue
));
540 elif isinstance(headers
, list):
546 def make_http_headers_from_dict_to_pycurl(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
547 if isinstance(headers
, dict):
549 if(sys
.version
[0]=="2"):
550 for headkey
, headvalue
in headers
.iteritems():
551 returnval
.append(headkey
+": "+headvalue
);
552 if(sys
.version
[0]>="3"):
553 for headkey
, headvalue
in headers
.items():
554 returnval
.append(headkey
+": "+headvalue
);
555 elif isinstance(headers
, list):
561 def make_http_headers_from_pycurl_to_dict(headers
):
563 headers
= headers
.strip().split('\r\n');
564 for header
in headers
:
565 parts
= header
.split(': ', 1)
568 header_dict
[key
.title()] = value
;
571 def make_http_headers_from_list_to_dict(headers
=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua
), ("Accept-Encoding", compression_supported
), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
572 if isinstance(headers
, list):
577 returnval
.update({headers
[mli
][0]: headers
[mli
][1]});
579 elif isinstance(headers
, dict):
585 def get_httplib_support(checkvalue
=None):
586 global haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
588 returnval
.append("ftp");
589 returnval
.append("httplib");
591 returnval
.append("httplib2");
592 returnval
.append("urllib");
594 returnval
.append("urllib3");
595 returnval
.append("request3");
596 returnval
.append("request");
598 returnval
.append("requests");
600 returnval
.append("httpx");
601 returnval
.append("httpx2");
603 returnval
.append("mechanize");
605 returnval
.append("pycurl");
606 returnval
.append("pycurl2");
607 returnval
.append("pycurl3");
609 returnval
.append("sftp");
611 returnval
.append("pysftp");
612 if(not checkvalue
is None):
613 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
614 checkvalue
= "urllib";
615 if(checkvalue
=="httplib1"):
616 checkvalue
= "httplib";
617 if(checkvalue
in returnval
):
623 def check_httplib_support(checkvalue
="urllib"):
624 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
625 checkvalue
= "urllib";
626 if(checkvalue
=="httplib1"):
627 checkvalue
= "httplib";
628 returnval
= get_httplib_support(checkvalue
);
631 def get_httplib_support_list():
632 returnval
= get_httplib_support(None);
635 def download_from_url(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", buffersize
=524288, sleep
=-1, timeout
=10):
636 global geturls_download_sleep
, havezstd
, havebrotli
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
638 sleep
= geturls_download_sleep
;
641 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
642 httplibuse
= "urllib";
643 if(httplibuse
=="httplib1"):
644 httplibuse
= "httplib";
645 if(not haverequests
and httplibuse
=="requests"):
646 httplibuse
= "urllib";
647 if(not havehttpx
and httplibuse
=="httpx"):
648 httplibuse
= "urllib";
649 if(not havehttpx
and httplibuse
=="httpx2"):
650 httplibuse
= "urllib";
651 if(not havehttpcore
and httplibuse
=="httpcore"):
652 httplibuse
= "urllib";
653 if(not havehttpcore
and httplibuse
=="httpcore2"):
654 httplibuse
= "urllib";
655 if(not havemechanize
and httplibuse
=="mechanize"):
656 httplibuse
= "urllib";
657 if(not havepycurl
and httplibuse
=="pycurl"):
658 httplibuse
= "urllib";
659 if(not havepycurl
and httplibuse
=="pycurl2"):
660 httplibuse
= "urllib";
661 if(not havepycurl
and httplibuse
=="pycurl3"):
662 httplibuse
= "urllib";
663 if(not havehttplib2
and httplibuse
=="httplib2"):
664 httplibuse
= "httplib";
665 if(not haveparamiko
and httplibuse
=="sftp"):
667 if(not havepysftp
and httplibuse
=="pysftp"):
669 if(httplibuse
=="urllib" or httplibuse
=="request"):
670 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
671 elif(httplibuse
=="request"):
672 returnval
= download_from_url_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
673 elif(httplibuse
=="request3"):
674 returnval
= download_from_url_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
675 elif(httplibuse
=="httplib"):
676 returnval
= download_from_url_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
677 elif(httplibuse
=="httplib2"):
678 returnval
= download_from_url_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
679 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
680 returnval
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
681 elif(httplibuse
=="requests"):
682 returnval
= download_from_url_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
683 elif(httplibuse
=="httpx"):
684 returnval
= download_from_url_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
685 elif(httplibuse
=="httpx2"):
686 returnval
= download_from_url_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
687 elif(httplibuse
=="httpcore"):
688 returnval
= download_from_url_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
689 elif(httplibuse
=="httpcore2"):
690 returnval
= download_from_url_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
691 elif(httplibuse
=="mechanize"):
692 returnval
= download_from_url_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
693 elif(httplibuse
=="pycurl"):
694 returnval
= download_from_url_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
695 elif(httplibuse
=="pycurl2"):
696 returnval
= download_from_url_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
697 elif(httplibuse
=="pycurl3"):
698 returnval
= download_from_url_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
699 elif(httplibuse
=="ftp"):
700 returnval
= download_from_url_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
701 elif(httplibuse
=="sftp"):
702 returnval
= download_from_url_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
703 elif(httplibuse
=="pysftp"):
704 returnval
= download_from_url_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
709 def download_from_url_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
710 global geturls_download_sleep
, havezstd
, havebrotli
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
712 sleep
= geturls_download_sleep
;
715 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
716 httplibuse
= "urllib";
717 if(httplibuse
=="httplib1"):
718 httplibuse
= "httplib";
719 if(not haverequests
and httplibuse
=="requests"):
720 httplibuse
= "urllib";
721 if(not havehttpx
and httplibuse
=="httpx"):
722 httplibuse
= "urllib";
723 if(not havehttpx
and httplibuse
=="httpx2"):
724 httplibuse
= "urllib";
725 if(not havehttpcore
and httplibuse
=="httpcore"):
726 httplibuse
= "urllib";
727 if(not havehttpcore
and httplibuse
=="httpcore2"):
728 httplibuse
= "urllib";
729 if(not havemechanize
and httplibuse
=="mechanize"):
730 httplibuse
= "urllib";
731 if(not havepycurl
and httplibuse
=="pycurl"):
732 httplibuse
= "urllib";
733 if(not havepycurl
and httplibuse
=="pycurl2"):
734 httplibuse
= "urllib";
735 if(not havepycurl
and httplibuse
=="pycurl3"):
736 httplibuse
= "urllib";
737 if(not havehttplib2
and httplibuse
=="httplib2"):
738 httplibuse
= "httplib";
739 if(not haveparamiko
and httplibuse
=="sftp"):
741 if(not haveparamiko
and httplibuse
=="pysftp"):
743 if(httplibuse
=="urllib" or httplibuse
=="request"):
744 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
745 elif(httplibuse
=="request"):
746 returnval
= download_from_url_file_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
747 elif(httplibuse
=="request3"):
748 returnval
= download_from_url_file_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
749 elif(httplibuse
=="httplib"):
750 returnval
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
751 elif(httplibuse
=="httplib2"):
752 returnval
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
753 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
754 returnval
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
755 elif(httplibuse
=="requests"):
756 returnval
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
757 elif(httplibuse
=="httpx"):
758 returnval
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
759 elif(httplibuse
=="httpx2"):
760 returnval
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
761 elif(httplibuse
=="httpcore"):
762 returnval
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
763 elif(httplibuse
=="httpcore2"):
764 returnval
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
765 elif(httplibuse
=="mechanize"):
766 returnval
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
767 elif(httplibuse
=="pycurl"):
768 returnval
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
769 elif(httplibuse
=="pycurl2"):
770 returnval
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
771 elif(httplibuse
=="pycurl3"):
772 returnval
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
773 elif(httplibuse
=="ftp"):
774 returnval
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
775 elif(httplibuse
=="sftp"):
776 returnval
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
777 elif(httplibuse
=="pysftp"):
778 returnval
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
783 def download_from_url_to_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
784 global geturls_download_sleep
, havezstd
, havebrotli
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
786 sleep
= geturls_download_sleep
;
789 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
790 httplibuse
= "urllib";
791 if(httplibuse
=="httplib1"):
792 httplibuse
= "httplib";
793 if(not haverequests
and httplibuse
=="requests"):
794 httplibuse
= "urllib";
795 if(not havehttpx
and httplibuse
=="httpx"):
796 httplibuse
= "urllib";
797 if(not havehttpx
and httplibuse
=="httpx2"):
798 httplibuse
= "urllib";
799 if(not havehttpcore
and httplibuse
=="httpcore"):
800 httplibuse
= "urllib";
801 if(not havehttpcore
and httplibuse
=="httpcore2"):
802 httplibuse
= "urllib";
803 if(not havemechanize
and httplibuse
=="mechanize"):
804 httplibuse
= "urllib";
805 if(not havepycurl
and httplibuse
=="pycurl"):
806 httplibuse
= "urllib";
807 if(not havepycurl
and httplibuse
=="pycurl2"):
808 httplibuse
= "urllib";
809 if(not havepycurl
and httplibuse
=="pycurl3"):
810 httplibuse
= "urllib";
811 if(not havehttplib2
and httplibuse
=="httplib2"):
812 httplibuse
= "httplib";
813 if(not haveparamiko
and httplibuse
=="sftp"):
815 if(not havepysftp
and httplibuse
=="pysftp"):
817 if(httplibuse
=="urllib" or httplibuse
=="request"):
818 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
819 elif(httplibuse
=="request"):
820 returnval
= download_from_url_to_file_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
821 elif(httplibuse
=="request3"):
822 returnval
= download_from_url_to_file_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
823 elif(httplibuse
=="httplib"):
824 returnval
= download_from_url_to_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
825 elif(httplibuse
=="httplib2"):
826 returnval
= download_from_url_to_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
827 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
828 returnval
= download_from_url_to_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
829 elif(httplibuse
=="requests"):
830 returnval
= download_from_url_to_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
831 elif(httplibuse
=="httpx"):
832 returnval
= download_from_url_to_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
833 elif(httplibuse
=="httpx2"):
834 returnval
= download_from_url_to_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
835 elif(httplibuse
=="httpcore"):
836 returnval
= download_from_url_to_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
837 elif(httplibuse
=="httpcore2"):
838 returnval
= download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
839 elif(httplibuse
=="mechanize"):
840 returnval
= download_from_url_to_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
841 elif(httplibuse
=="pycurl"):
842 returnval
= download_from_url_to_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
843 elif(httplibuse
=="pycurl2"):
844 returnval
= download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
845 elif(httplibuse
=="pycurl3"):
846 returnval
= download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
847 elif(httplibuse
=="ftp"):
848 returnval
= download_from_url_to_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
849 elif(httplibuse
=="sftp"):
850 returnval
= download_from_url_to_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
851 elif(httplibuse
=="pysftp"):
852 returnval
= download_from_url_to_file_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
857 def download_from_url_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
858 global geturls_download_sleep
, havezstd
, havebrotli
;
860 sleep
= geturls_download_sleep
;
863 urlparts
= urlparse
.urlparse(httpurl
);
864 if(isinstance(httpheaders
, list)):
865 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
866 httpheaders
= fix_header_names(httpheaders
);
867 if(httpuseragent
is not None):
868 if('User-Agent' in httpheaders
):
869 httpheaders
['User-Agent'] = httpuseragent
;
871 httpuseragent
.update({'User-Agent': httpuseragent
});
872 if(httpreferer
is not None):
873 if('Referer' in httpheaders
):
874 httpheaders
['Referer'] = httpreferer
;
876 httpuseragent
.update({'Referer': httpreferer
});
877 if(urlparts
.username
is not None or urlparts
.password
is not None):
878 if(sys
.version
[0]=="2"):
879 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
880 if(sys
.version
[0]>="3"):
881 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
882 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
883 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
884 if(isinstance(httpheaders
, dict)):
885 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
886 geturls_opener
.addheaders
= httpheaders
;
888 if(postdata
is not None and not isinstance(postdata
, dict)):
889 postdata
= urlencode(postdata
);
891 geturls_request
= Request(httpurl
);
892 if(httpmethod
=="GET"):
893 geturls_text
= geturls_opener
.open(geturls_request
);
894 elif(httpmethod
=="POST"):
895 geturls_text
= geturls_opener
.open(geturls_request
, data
=postdata
);
897 geturls_text
= geturls_opener
.open(geturls_request
);
898 except HTTPError
as geturls_text_error
:
899 geturls_text
= geturls_text_error
;
900 log
.info("Error With URL "+httpurl
);
902 log
.info("Error With URL "+httpurl
);
904 except socket
.timeout
:
905 log
.info("Error With URL "+httpurl
);
907 httpcodeout
= geturls_text
.getcode();
909 httpcodereason
= geturls_text
.reason
;
910 except AttributeError:
911 httpcodereason
= http_status_to_reason(geturls_text
.getcode());
913 httpversionout
= geturls_text
.version
;
914 except AttributeError:
915 httpversionout
= "1.1";
916 httpmethodout
= geturls_request
.get_method();
917 httpurlout
= geturls_text
.geturl();
918 httpheaderout
= geturls_text
.info();
919 httpheadersentout
= httpheaders
;
920 if(isinstance(httpheaderout
, list)):
921 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
922 httpheaderout
= fix_header_names(httpheaderout
);
923 if(sys
.version
[0]=="2"):
925 prehttpheaderout
= httpheaderout
;
926 httpheaderkeys
= httpheaderout
.keys();
927 imax
= len(httpheaderkeys
);
931 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
933 except AttributeError:
935 if(isinstance(httpheadersentout
, list)):
936 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
937 httpheadersentout
= fix_header_names(httpheadersentout
);
938 log
.info("Downloading URL "+httpurl
);
939 downloadsize
= httpheaderout
.get('Content-Length');
940 if(downloadsize
is not None):
941 downloadsize
= int(downloadsize
);
942 if downloadsize
is None: downloadsize
= 0;
945 log
.info("Downloading URL "+httpurl
);
946 with
BytesIO() as strbuf
:
948 databytes
= geturls_text
.read(buffersize
);
949 if not databytes
: break;
950 datasize
= len(databytes
);
951 fulldatasize
= datasize
+ fulldatasize
;
954 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
955 downloaddiff
= fulldatasize
- prevdownsize
;
956 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
957 prevdownsize
= fulldatasize
;
958 strbuf
.write(databytes
);
960 returnval_content
= strbuf
.read();
961 if(httpheaderout
.get("Content-Encoding")=="gzip"):
963 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
966 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
968 returnval_content
= zlib
.decompress(returnval_content
);
971 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
973 returnval_content
= brotli
.decompress(returnval_content
);
976 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
978 returnval_content
= zstandard
.decompress(returnval_content
);
979 except zstandard
.error
:
981 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
982 geturls_text
.close();
985 def download_from_url_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
986 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
987 exec_time_start
= time
.time();
988 myhash
= hashlib
.new("sha1");
989 if(sys
.version
[0]=="2"):
990 myhash
.update(httpurl
);
991 myhash
.update(str(buffersize
));
992 myhash
.update(str(exec_time_start
));
993 if(sys
.version
[0]>="3"):
994 myhash
.update(httpurl
.encode('utf-8'));
995 myhash
.update(str(buffersize
).encode('utf-8'));
996 myhash
.update(str(exec_time_start
).encode('utf-8'));
997 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
999 sleep
= geturls_download_sleep
;
1002 pretmpfilename
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1003 if(not pretmpfilename
):
1005 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1006 tmpfilename
= f
.name
;
1008 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1009 except AttributeError:
1011 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1016 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
1017 f
.write(pretmpfilename
['Content']);
1019 exec_time_end
= time
.time();
1020 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1021 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1024 def download_from_url_to_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1025 global geturls_download_sleep
, havezstd
, havebrotli
;
1027 sleep
= geturls_download_sleep
;
1030 if(not outfile
=="-"):
1031 outpath
= outpath
.rstrip(os
.path
.sep
);
1032 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1033 if(not os
.path
.exists(outpath
)):
1034 os
.makedirs(outpath
);
1035 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1037 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1039 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1040 if(not pretmpfilename
):
1042 tmpfilename
= pretmpfilename
['Filename'];
1043 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1045 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1046 exec_time_start
= time
.time();
1047 shutil
.move(tmpfilename
, filepath
);
1049 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1050 except AttributeError:
1052 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1057 exec_time_end
= time
.time();
1058 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1059 if(os
.path
.exists(tmpfilename
)):
1060 os
.remove(tmpfilename
);
1061 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1063 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1064 tmpfilename
= pretmpfilename
['Filename'];
1065 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1068 exec_time_start
= time
.time();
1069 with
open(tmpfilename
, 'rb') as ft
:
1072 databytes
= ft
.read(buffersize
[1]);
1073 if not databytes
: break;
1074 datasize
= len(databytes
);
1075 fulldatasize
= datasize
+ fulldatasize
;
1078 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1079 downloaddiff
= fulldatasize
- prevdownsize
;
1080 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1081 prevdownsize
= fulldatasize
;
1084 fdata
= f
.getvalue();
1087 os
.remove(tmpfilename
);
1088 exec_time_end
= time
.time();
1089 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1090 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1093 def download_from_url_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1094 global geturls_download_sleep
, havezstd
, havebrotli
, havezstd
, havebrotli
;
1096 sleep
= geturls_download_sleep
;
1099 urlparts
= urlparse
.urlparse(httpurl
);
1100 if(isinstance(httpheaders
, list)):
1101 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1102 httpheaders
= fix_header_names(httpheaders
);
1103 if(httpuseragent
is not None):
1104 if('User-Agent' in httpheaders
):
1105 httpheaders
['User-Agent'] = httpuseragent
;
1107 httpuseragent
.update({'User-Agent': httpuseragent
});
1108 if(httpreferer
is not None):
1109 if('Referer' in httpheaders
):
1110 httpheaders
['Referer'] = httpreferer
;
1112 httpuseragent
.update({'Referer': httpreferer
});
1113 if(urlparts
.username
is not None or urlparts
.password
is not None):
1114 if(sys
.version
[0]=="2"):
1115 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1116 if(sys
.version
[0]>="3"):
1117 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1118 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1119 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1120 geturls_opener
.addheaders
= httpheaders
;
1122 if(urlparts
[0]=="http"):
1123 httpconn
= HTTPConnection(urlparts
[1], timeout
=timeout
);
1124 elif(urlparts
[0]=="https"):
1125 httpconn
= HTTPSConnection(urlparts
[1], timeout
=timeout
);
1128 if(postdata
is not None and not isinstance(postdata
, dict)):
1129 postdata
= urlencode(postdata
);
1131 if(httpmethod
=="GET"):
1132 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1133 elif(httpmethod
=="POST"):
1134 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1136 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1137 except socket
.timeout
:
1138 log
.info("Error With URL "+httpurl
);
1140 except socket
.gaierror
:
1141 log
.info("Error With URL "+httpurl
);
1143 except BlockingIOError
:
1144 log
.info("Error With URL "+httpurl
);
1146 geturls_text
= httpconn
.getresponse();
1147 httpcodeout
= geturls_text
.status
;
1148 httpcodereason
= geturls_text
.reason
;
1149 if(geturls_text
.version
=="10"):
1150 httpversionout
= "1.0";
1152 httpversionout
= "1.1";
1153 httpmethodout
= geturls_text
._method
;
1154 httpurlout
= httpurl
;
1155 httpheaderout
= geturls_text
.getheaders();
1156 httpheadersentout
= httpheaders
;
1157 if(isinstance(httpheaderout
, list)):
1158 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1159 if(sys
.version
[0]=="2"):
1161 prehttpheaderout
= httpheaderout
;
1162 httpheaderkeys
= httpheaderout
.keys();
1163 imax
= len(httpheaderkeys
);
1167 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1169 except AttributeError:
1171 httpheaderout
= fix_header_names(httpheaderout
);
1172 if(isinstance(httpheadersentout
, list)):
1173 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1174 httpheadersentout
= fix_header_names(httpheadersentout
);
1175 log
.info("Downloading URL "+httpurl
);
1176 downloadsize
= httpheaderout
.get('Content-Length');
1177 if(downloadsize
is not None):
1178 downloadsize
= int(downloadsize
);
1179 if downloadsize
is None: downloadsize
= 0;
1182 log
.info("Downloading URL "+httpurl
);
1183 with
BytesIO() as strbuf
:
1185 databytes
= geturls_text
.read(buffersize
);
1186 if not databytes
: break;
1187 datasize
= len(databytes
);
1188 fulldatasize
= datasize
+ fulldatasize
;
1191 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1192 downloaddiff
= fulldatasize
- prevdownsize
;
1193 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1194 prevdownsize
= fulldatasize
;
1195 strbuf
.write(databytes
);
1197 returnval_content
= strbuf
.read();
1198 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1200 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1203 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1205 returnval_content
= zlib
.decompress(returnval_content
);
1208 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1210 returnval_content
= brotli
.decompress(returnval_content
);
1211 except brotli
.error
:
1213 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1215 returnval_content
= zstandard
.decompress(returnval_content
);
1216 except zstandard
.error
:
1218 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
1219 geturls_text
.close();
1222 def download_from_url_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1223 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1224 exec_time_start
= time
.time();
1225 myhash
= hashlib
.new("sha1");
1226 if(sys
.version
[0]=="2"):
1227 myhash
.update(httpurl
);
1228 myhash
.update(str(buffersize
));
1229 myhash
.update(str(exec_time_start
));
1230 if(sys
.version
[0]>="3"):
1231 myhash
.update(httpurl
.encode('utf-8'));
1232 myhash
.update(str(buffersize
).encode('utf-8'));
1233 myhash
.update(str(exec_time_start
).encode('utf-8'));
1234 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1236 sleep
= geturls_download_sleep
;
1239 pretmpfilename
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1240 if(not pretmpfilename
):
1242 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1243 tmpfilename
= f
.name
;
1245 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1246 except AttributeError:
1248 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1253 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
1254 f
.write(pretmpfilename
['Content']);
1256 exec_time_end
= time
.time();
1257 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1258 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1261 def download_from_url_to_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1262 global geturls_download_sleep
, havezstd
, havebrotli
;
1264 sleep
= geturls_download_sleep
;
1267 if(not outfile
=="-"):
1268 outpath
= outpath
.rstrip(os
.path
.sep
);
1269 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1270 if(not os
.path
.exists(outpath
)):
1271 os
.makedirs(outpath
);
1272 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1274 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1276 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1277 if(not pretmpfilename
):
1279 tmpfilename
= pretmpfilename
['Filename'];
1280 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1282 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1283 exec_time_start
= time
.time();
1284 shutil
.move(tmpfilename
, filepath
);
1286 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1287 except AttributeError:
1289 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1294 exec_time_end
= time
.time();
1295 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1296 if(os
.path
.exists(tmpfilename
)):
1297 os
.remove(tmpfilename
);
1298 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1300 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1301 tmpfilename
= pretmpfilename
['Filename'];
1302 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1305 exec_time_start
= time
.time();
1306 with
open(tmpfilename
, 'rb') as ft
:
1309 databytes
= ft
.read(buffersize
[1]);
1310 if not databytes
: break;
1311 datasize
= len(databytes
);
1312 fulldatasize
= datasize
+ fulldatasize
;
1315 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1316 downloaddiff
= fulldatasize
- prevdownsize
;
1317 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1318 prevdownsize
= fulldatasize
;
1321 fdata
= f
.getvalue();
1324 os
.remove(tmpfilename
);
1325 exec_time_end
= time
.time();
1326 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1327 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1331 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1332 global geturls_download_sleep
, havezstd
, havebrotli
;
1334 sleep
= geturls_download_sleep
;
1337 urlparts
= urlparse
.urlparse(httpurl
);
1338 if(isinstance(httpheaders
, list)):
1339 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1340 httpheaders
= fix_header_names(httpheaders
);
1341 if(httpuseragent
is not None):
1342 if('User-Agent' in httpheaders
):
1343 httpheaders
['User-Agent'] = httpuseragent
;
1345 httpuseragent
.update({'User-Agent': httpuseragent
});
1346 if(httpreferer
is not None):
1347 if('Referer' in httpheaders
):
1348 httpheaders
['Referer'] = httpreferer
;
1350 httpuseragent
.update({'Referer': httpreferer
});
1351 if(urlparts
.username
is not None or urlparts
.password
is not None):
1352 if(sys
.version
[0]=="2"):
1353 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1354 if(sys
.version
[0]>="3"):
1355 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1356 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1357 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1358 geturls_opener
.addheaders
= httpheaders
;
1360 if(urlparts
[0]=="http"):
1361 httpconn
= HTTPConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
1362 elif(urlparts
[0]=="https"):
1363 httpconn
= HTTPSConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
1366 if(postdata
is not None and not isinstance(postdata
, dict)):
1367 postdata
= urlencode(postdata
);
1369 if(httpmethod
=="GET"):
1370 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1371 elif(httpmethod
=="POST"):
1372 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1374 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1375 except socket
.timeout
:
1376 log
.info("Error With URL "+httpurl
);
1378 except socket
.gaierror
:
1379 log
.info("Error With URL "+httpurl
);
1381 except BlockingIOError
:
1382 log
.info("Error With URL "+httpurl
);
1384 geturls_text
= httpconn
.getresponse();
1385 httpcodeout
= geturls_text
.status
;
1386 httpcodereason
= geturls_text
.reason
;
1387 if(geturls_text
.version
=="10"):
1388 httpversionout
= "1.0";
1390 httpversionout
= "1.1";
1391 httpmethodout
= httpmethod
;
1392 httpurlout
= httpurl
;
1393 httpheaderout
= geturls_text
.getheaders();
1394 httpheadersentout
= httpheaders
;
1395 if(isinstance(httpheaderout
, list)):
1396 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1397 if(sys
.version
[0]=="2"):
1399 prehttpheaderout
= httpheaderout
;
1400 httpheaderkeys
= httpheaderout
.keys();
1401 imax
= len(httpheaderkeys
);
1405 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1407 except AttributeError:
1409 httpheaderout
= fix_header_names(httpheaderout
);
1410 if(isinstance(httpheadersentout
, list)):
1411 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1412 httpheadersentout
= fix_header_names(httpheadersentout
);
1413 log
.info("Downloading URL "+httpurl
);
1414 downloadsize
= httpheaderout
.get('Content-Length');
1415 if(downloadsize
is not None):
1416 downloadsize
= int(downloadsize
);
1417 if downloadsize
is None: downloadsize
= 0;
1420 log
.info("Downloading URL "+httpurl
);
1421 with
BytesIO() as strbuf
:
1423 databytes
= geturls_text
.read(buffersize
);
1424 if not databytes
: break;
1425 datasize
= len(databytes
);
1426 fulldatasize
= datasize
+ fulldatasize
;
1429 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1430 downloaddiff
= fulldatasize
- prevdownsize
;
1431 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1432 prevdownsize
= fulldatasize
;
1433 strbuf
.write(databytes
);
1435 returnval_content
= strbuf
.read();
1436 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1438 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1441 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1443 returnval_content
= zlib
.decompress(returnval_content
);
1446 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1448 returnval_content
= brotli
.decompress(returnval_content
);
1449 except brotli
.error
:
1451 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1453 returnval_content
= zstandard
.decompress(returnval_content
);
1454 except zstandard
.error
:
1456 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
1457 geturls_text
.close();
1460 if(not havehttplib2
):
1461 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1462 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1466 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1467 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1468 exec_time_start
= time
.time();
1469 myhash
= hashlib
.new("sha1");
1470 if(sys
.version
[0]=="2"):
1471 myhash
.update(httpurl
);
1472 myhash
.update(str(buffersize
));
1473 myhash
.update(str(exec_time_start
));
1474 if(sys
.version
[0]>="3"):
1475 myhash
.update(httpurl
.encode('utf-8'));
1476 myhash
.update(str(buffersize
).encode('utf-8'));
1477 myhash
.update(str(exec_time_start
).encode('utf-8'));
1478 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1480 sleep
= geturls_download_sleep
;
1483 pretmpfilename
= download_from_url_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1484 if(not pretmpfilename
):
1486 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1487 tmpfilename
= f
.name
;
1489 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1490 except AttributeError:
1492 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1497 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
1498 f
.write(pretmpfilename
['Content']);
1500 exec_time_end
= time
.time();
1501 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1502 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1505 if(not havehttplib2
):
1506 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1507 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1511 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1512 global geturls_download_sleep
, havezstd
, havebrotli
;
1514 sleep
= geturls_download_sleep
;
1517 if(not outfile
=="-"):
1518 outpath
= outpath
.rstrip(os
.path
.sep
);
1519 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1520 if(not os
.path
.exists(outpath
)):
1521 os
.makedirs(outpath
);
1522 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1524 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1526 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1527 if(not pretmpfilename
):
1529 tmpfilename
= pretmpfilename
['Filename'];
1530 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1532 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1533 exec_time_start
= time
.time();
1534 shutil
.move(tmpfilename
, filepath
);
1536 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1537 except AttributeError:
1539 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1544 exec_time_end
= time
.time();
1545 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1546 if(os
.path
.exists(tmpfilename
)):
1547 os
.remove(tmpfilename
);
1548 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1550 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1551 tmpfilename
= pretmpfilename
['Filename'];
1552 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1555 exec_time_start
= time
.time();
1556 with
open(tmpfilename
, 'rb') as ft
:
1559 databytes
= ft
.read(buffersize
[1]);
1560 if not databytes
: break;
1561 datasize
= len(databytes
);
1562 fulldatasize
= datasize
+ fulldatasize
;
1565 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1566 downloaddiff
= fulldatasize
- prevdownsize
;
1567 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1568 prevdownsize
= fulldatasize
;
1571 fdata
= f
.getvalue();
1574 os
.remove(tmpfilename
);
1575 exec_time_end
= time
.time();
1576 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1577 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1580 if(not havehttplib2
):
1581 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1582 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1585 def download_from_url_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1586 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1589 def download_from_url_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1590 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1593 def download_from_url_to_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1594 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1598 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1599 global geturls_download_sleep
, havezstd
, havebrotli
;
1601 sleep
= geturls_download_sleep
;
1604 urlparts
= urlparse
.urlparse(httpurl
);
1605 if(isinstance(httpheaders
, list)):
1606 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1607 httpheaders
= fix_header_names(httpheaders
);
1608 if(httpuseragent
is not None):
1609 if('User-Agent' in httpheaders
):
1610 httpheaders
['User-Agent'] = httpuseragent
;
1612 httpuseragent
.update({'User-Agent': httpuseragent
});
1613 if(httpreferer
is not None):
1614 if('Referer' in httpheaders
):
1615 httpheaders
['Referer'] = httpreferer
;
1617 httpuseragent
.update({'Referer': httpreferer
});
1618 if(urlparts
.username
is not None or urlparts
.password
is not None):
1619 if(sys
.version
[0]=="2"):
1620 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1621 if(sys
.version
[0]>="3"):
1622 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1623 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1625 if(postdata
is not None and not isinstance(postdata
, dict)):
1626 postdata
= urlencode(postdata
);
1628 reqsession
= requests
.Session();
1629 if(httpmethod
=="GET"):
1630 geturls_text
= reqsession
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1631 elif(httpmethod
=="POST"):
1632 geturls_text
= reqsession
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1634 geturls_text
= reqsession
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1635 except requests
.exceptions
.ConnectTimeout
:
1636 log
.info("Error With URL "+httpurl
);
1638 except requests
.exceptions
.ConnectError
:
1639 log
.info("Error With URL "+httpurl
);
1641 except socket
.timeout
:
1642 log
.info("Error With URL "+httpurl
);
1644 httpcodeout
= geturls_text
.status_code
;
1645 httpcodereason
= geturls_text
.reason
;
1646 if(geturls_text
.raw
.version
=="10"):
1647 httpversionout
= "1.0";
1649 httpversionout
= "1.1";
1650 httpmethodout
= httpmethod
;
1651 httpurlout
= geturls_text
.url
;
1652 httpheaderout
= geturls_text
.headers
;
1653 httpheadersentout
= geturls_text
.request
.headers
;
1654 if(isinstance(httpheaderout
, list)):
1655 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1656 if(sys
.version
[0]=="2"):
1658 prehttpheaderout
= httpheaderout
;
1659 httpheaderkeys
= httpheaderout
.keys();
1660 imax
= len(httpheaderkeys
);
1664 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1666 except AttributeError:
1668 httpheaderout
= fix_header_names(httpheaderout
);
1669 if(isinstance(httpheadersentout
, list)):
1670 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1671 httpheadersentout
= fix_header_names(httpheadersentout
);
1672 log
.info("Downloading URL "+httpurl
);
1673 downloadsize
= httpheaderout
.get('Content-Length');
1674 if(downloadsize
is not None):
1675 downloadsize
= int(downloadsize
);
1676 if downloadsize
is None: downloadsize
= 0;
1679 log
.info("Downloading URL "+httpurl
);
1680 with
BytesIO() as strbuf
:
1682 databytes
= geturls_text
.raw
.read(buffersize
);
1683 if not databytes
: break;
1684 datasize
= len(databytes
);
1685 fulldatasize
= datasize
+ fulldatasize
;
1688 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1689 downloaddiff
= fulldatasize
- prevdownsize
;
1690 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1691 prevdownsize
= fulldatasize
;
1692 strbuf
.write(databytes
);
1694 returnval_content
= strbuf
.read();
1695 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1697 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1700 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1702 returnval_content
= zlib
.decompress(returnval_content
);
1705 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1707 returnval_content
= brotli
.decompress(returnval_content
);
1708 except brotli
.error
:
1710 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1712 returnval_content
= zstandard
.decompress(returnval_content
);
1713 except zstandard
.error
:
1715 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
1716 geturls_text
.close();
1719 if(not haverequests
):
1720 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1721 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1725 def download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1726 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1727 exec_time_start
= time
.time();
1728 myhash
= hashlib
.new("sha1");
1729 if(sys
.version
[0]=="2"):
1730 myhash
.update(httpurl
);
1731 myhash
.update(str(buffersize
));
1732 myhash
.update(str(exec_time_start
));
1733 if(sys
.version
[0]>="3"):
1734 myhash
.update(httpurl
.encode('utf-8'));
1735 myhash
.update(str(buffersize
).encode('utf-8'));
1736 myhash
.update(str(exec_time_start
).encode('utf-8'));
1737 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1739 sleep
= geturls_download_sleep
;
1742 pretmpfilename
= download_from_url_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1743 if(not pretmpfilename
):
1745 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1746 tmpfilename
= f
.name
;
1748 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1749 except AttributeError:
1751 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1756 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
1757 f
.write(pretmpfilename
['Content']);
1759 exec_time_end
= time
.time();
1760 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1761 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1764 if(not haverequests
):
1765 def download_from_url_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1766 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1770 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1771 global geturls_download_sleep
, havezstd
, havebrotli
;
1773 sleep
= geturls_download_sleep
;
1776 if(not outfile
=="-"):
1777 outpath
= outpath
.rstrip(os
.path
.sep
);
1778 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1779 if(not os
.path
.exists(outpath
)):
1780 os
.makedirs(outpath
);
1781 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1783 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1785 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1786 if(not pretmpfilename
):
1788 tmpfilename
= pretmpfilename
['Filename'];
1789 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1791 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1792 exec_time_start
= time
.time();
1793 shutil
.move(tmpfilename
, filepath
);
1795 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1796 except AttributeError:
1798 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1803 exec_time_end
= time
.time();
1804 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1805 if(os
.path
.exists(tmpfilename
)):
1806 os
.remove(tmpfilename
);
1807 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1809 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1810 tmpfilename
= pretmpfilename
['Filename'];
1811 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1814 exec_time_start
= time
.time();
1815 with
open(tmpfilename
, 'rb') as ft
:
1818 databytes
= ft
.read(buffersize
[1]);
1819 if not databytes
: break;
1820 datasize
= len(databytes
);
1821 fulldatasize
= datasize
+ fulldatasize
;
1824 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1825 downloaddiff
= fulldatasize
- prevdownsize
;
1826 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1827 prevdownsize
= fulldatasize
;
1830 fdata
= f
.getvalue();
1833 os
.remove(tmpfilename
);
1834 exec_time_end
= time
.time();
1835 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1836 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
1839 if(not haverequests
):
1840 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1841 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1845 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1846 global geturls_download_sleep
, havezstd
, havebrotli
;
1848 sleep
= geturls_download_sleep
;
1851 urlparts
= urlparse
.urlparse(httpurl
);
1852 if(isinstance(httpheaders
, list)):
1853 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1854 httpheaders
= fix_header_names(httpheaders
);
1855 if(httpuseragent
is not None):
1856 if('User-Agent' in httpheaders
):
1857 httpheaders
['User-Agent'] = httpuseragent
;
1859 httpuseragent
.update({'User-Agent': httpuseragent
});
1860 if(httpreferer
is not None):
1861 if('Referer' in httpheaders
):
1862 httpheaders
['Referer'] = httpreferer
;
1864 httpuseragent
.update({'Referer': httpreferer
});
1865 if(urlparts
.username
is not None or urlparts
.password
is not None):
1866 if(sys
.version
[0]=="2"):
1867 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1868 if(sys
.version
[0]>="3"):
1869 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1870 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1872 if(postdata
is not None and not isinstance(postdata
, dict)):
1873 postdata
= urlencode(postdata
);
1875 if(httpmethod
=="GET"):
1876 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1877 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
1878 elif(httpmethod
=="POST"):
1879 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1880 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1882 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1883 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
1884 except httpx
.ConnectTimeout
:
1885 log
.info("Error With URL "+httpurl
);
1887 except httpx
.ConnectError
:
1888 log
.info("Error With URL "+httpurl
);
1890 except socket
.timeout
:
1891 log
.info("Error With URL "+httpurl
);
1893 httpcodeout
= geturls_text
.status_code
;
1895 httpcodereason
= geturls_text
.reason_phrase
;
1897 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
1898 httpversionout
= geturls_text
.http_version
;
1899 httpmethodout
= httpmethod
;
1900 httpurlout
= str(geturls_text
.url
);
1901 httpheaderout
= geturls_text
.headers
;
1902 httpheadersentout
= geturls_text
.request
.headers
;
1903 if(isinstance(httpheaderout
, list)):
1904 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1905 if(sys
.version
[0]=="2"):
1907 prehttpheaderout
= httpheaderout
;
1908 httpheaderkeys
= httpheaderout
.keys();
1909 imax
= len(httpheaderkeys
);
1913 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1915 except AttributeError:
1917 httpheaderout
= fix_header_names(httpheaderout
);
1918 if(isinstance(httpheadersentout
, list)):
1919 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1920 httpheadersentout
= fix_header_names(httpheadersentout
);
1921 log
.info("Downloading URL "+httpurl
);
1922 downloadsize
= httpheaderout
.get('Content-Length');
1923 if(downloadsize
is not None):
1924 downloadsize
= int(downloadsize
);
1925 if downloadsize
is None: downloadsize
= 0;
1928 log
.info("Downloading URL "+httpurl
);
1929 with
BytesIO() as strbuf
:
1931 databytes
= geturls_text
.read(buffersize
);
1932 if not databytes
: break;
1933 datasize
= len(databytes
);
1934 fulldatasize
= datasize
+ fulldatasize
;
1937 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1938 downloaddiff
= fulldatasize
- prevdownsize
;
1939 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1940 prevdownsize
= fulldatasize
;
1941 strbuf
.write(databytes
);
1943 returnval_content
= strbuf
.read();
1944 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1946 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1949 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1951 returnval_content
= zlib
.decompress(returnval_content
);
1954 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1956 returnval_content
= brotli
.decompress(returnval_content
);
1957 except brotli
.error
:
1959 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1961 returnval_content
= zstandard
.decompress(returnval_content
);
1962 except zstandard
.error
:
1964 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
1965 geturls_text
.close();
1969 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1970 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1974 def download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1975 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1976 exec_time_start
= time
.time();
1977 myhash
= hashlib
.new("sha1");
1978 if(sys
.version
[0]=="2"):
1979 myhash
.update(httpurl
);
1980 myhash
.update(str(buffersize
));
1981 myhash
.update(str(exec_time_start
));
1982 if(sys
.version
[0]>="3"):
1983 myhash
.update(httpurl
.encode('utf-8'));
1984 myhash
.update(str(buffersize
).encode('utf-8'));
1985 myhash
.update(str(exec_time_start
).encode('utf-8'));
1986 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1988 sleep
= geturls_download_sleep
;
1991 pretmpfilename
= download_from_url_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1992 if(not pretmpfilename
):
1994 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1995 tmpfilename
= f
.name
;
1997 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1998 except AttributeError:
2000 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2005 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
2006 f
.write(pretmpfilename
['Content']);
2008 exec_time_end
= time
.time();
2009 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2010 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2014 def download_from_url_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2015 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2019 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2020 global geturls_download_sleep
, havezstd
, havebrotli
;
2022 sleep
= geturls_download_sleep
;
2025 if(not outfile
=="-"):
2026 outpath
= outpath
.rstrip(os
.path
.sep
);
2027 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2028 if(not os
.path
.exists(outpath
)):
2029 os
.makedirs(outpath
);
2030 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2032 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2034 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2035 if(not pretmpfilename
):
2037 tmpfilename
= pretmpfilename
['Filename'];
2038 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2040 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2041 exec_time_start
= time
.time();
2042 shutil
.move(tmpfilename
, filepath
);
2044 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2045 except AttributeError:
2047 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2052 exec_time_end
= time
.time();
2053 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2054 if(os
.path
.exists(tmpfilename
)):
2055 os
.remove(tmpfilename
);
2056 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2058 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2059 tmpfilename
= pretmpfilename
['Filename'];
2060 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2063 exec_time_start
= time
.time();
2064 with
open(tmpfilename
, 'rb') as ft
:
2067 databytes
= ft
.read(buffersize
[1]);
2068 if not databytes
: break;
2069 datasize
= len(databytes
);
2070 fulldatasize
= datasize
+ fulldatasize
;
2073 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2074 downloaddiff
= fulldatasize
- prevdownsize
;
2075 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2076 prevdownsize
= fulldatasize
;
2079 fdata
= f
.getvalue();
2082 os
.remove(tmpfilename
);
2083 exec_time_end
= time
.time();
2084 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2085 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2089 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2090 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2094 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2095 global geturls_download_sleep
, havezstd
, havebrotli
;
2097 sleep
= geturls_download_sleep
;
2100 urlparts
= urlparse
.urlparse(httpurl
);
2101 if(isinstance(httpheaders
, list)):
2102 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2103 httpheaders
= fix_header_names(httpheaders
);
2104 if(httpuseragent
is not None):
2105 if('User-Agent' in httpheaders
):
2106 httpheaders
['User-Agent'] = httpuseragent
;
2108 httpuseragent
.update({'User-Agent': httpuseragent
});
2109 if(httpreferer
is not None):
2110 if('Referer' in httpheaders
):
2111 httpheaders
['Referer'] = httpreferer
;
2113 httpuseragent
.update({'Referer': httpreferer
});
2114 if(urlparts
.username
is not None or urlparts
.password
is not None):
2115 if(sys
.version
[0]=="2"):
2116 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2117 if(sys
.version
[0]>="3"):
2118 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2119 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2121 if(postdata
is not None and not isinstance(postdata
, dict)):
2122 postdata
= urlencode(postdata
);
2124 if(httpmethod
=="GET"):
2125 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2126 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2127 elif(httpmethod
=="POST"):
2128 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2129 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
2131 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2132 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2133 except httpx
.ConnectTimeout
:
2134 log
.info("Error With URL "+httpurl
);
2136 except httpx
.ConnectError
:
2137 log
.info("Error With URL "+httpurl
);
2139 except socket
.timeout
:
2140 log
.info("Error With URL "+httpurl
);
2142 httpcodeout
= geturls_text
.status_code
;
2144 httpcodereason
= geturls_text
.reason_phrase
;
2146 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
2147 httpversionout
= geturls_text
.http_version
;
2148 httpmethodout
= httpmethod
;
2149 httpurlout
= str(geturls_text
.url
);
2150 httpheaderout
= geturls_text
.headers
;
2151 httpheadersentout
= geturls_text
.request
.headers
;
2152 if(isinstance(httpheaderout
, list)):
2153 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2154 if(sys
.version
[0]=="2"):
2156 prehttpheaderout
= httpheaderout
;
2157 httpheaderkeys
= httpheaderout
.keys();
2158 imax
= len(httpheaderkeys
);
2162 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2164 except AttributeError:
2166 httpheaderout
= fix_header_names(httpheaderout
);
2167 if(isinstance(httpheadersentout
, list)):
2168 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2169 httpheadersentout
= fix_header_names(httpheadersentout
);
2170 log
.info("Downloading URL "+httpurl
);
2171 downloadsize
= httpheaderout
.get('Content-Length');
2172 if(downloadsize
is not None):
2173 downloadsize
= int(downloadsize
);
2174 if downloadsize
is None: downloadsize
= 0;
2177 log
.info("Downloading URL "+httpurl
);
2178 with
BytesIO() as strbuf
:
2180 databytes
= geturls_text
.read(buffersize
);
2181 if not databytes
: break;
2182 datasize
= len(databytes
);
2183 fulldatasize
= datasize
+ fulldatasize
;
2186 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2187 downloaddiff
= fulldatasize
- prevdownsize
;
2188 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2189 prevdownsize
= fulldatasize
;
2190 strbuf
.write(databytes
);
2192 returnval_content
= strbuf
.read();
2193 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2195 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2198 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2200 returnval_content
= zlib
.decompress(returnval_content
);
2203 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2205 returnval_content
= brotli
.decompress(returnval_content
);
2206 except brotli
.error
:
2208 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2210 returnval_content
= zstandard
.decompress(returnval_content
);
2211 except zstandard
.error
:
2213 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
2214 geturls_text
.close();
2218 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2219 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2223 def download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2224 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2225 exec_time_start
= time
.time();
2226 myhash
= hashlib
.new("sha1");
2227 if(sys
.version
[0]=="2"):
2228 myhash
.update(httpurl
);
2229 myhash
.update(str(buffersize
));
2230 myhash
.update(str(exec_time_start
));
2231 if(sys
.version
[0]>="3"):
2232 myhash
.update(httpurl
.encode('utf-8'));
2233 myhash
.update(str(buffersize
).encode('utf-8'));
2234 myhash
.update(str(exec_time_start
).encode('utf-8'));
2235 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2237 sleep
= geturls_download_sleep
;
2240 pretmpfilename
= download_from_url_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2241 if(not pretmpfilename
):
2243 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2244 tmpfilename
= f
.name
;
2246 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2247 except AttributeError:
2249 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2254 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
2255 f
.write(pretmpfilename
['Content']);
2257 exec_time_end
= time
.time();
2258 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2259 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2263 def download_from_url_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2264 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2268 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2269 global geturls_download_sleep
, havezstd
, havebrotli
;
2271 sleep
= geturls_download_sleep
;
2274 if(not outfile
=="-"):
2275 outpath
= outpath
.rstrip(os
.path
.sep
);
2276 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2277 if(not os
.path
.exists(outpath
)):
2278 os
.makedirs(outpath
);
2279 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2281 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2283 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2284 if(not pretmpfilename
):
2286 tmpfilename
= pretmpfilename
['Filename'];
2287 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2289 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2290 exec_time_start
= time
.time();
2291 shutil
.move(tmpfilename
, filepath
);
2293 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2294 except AttributeError:
2296 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2301 exec_time_end
= time
.time();
2302 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2303 if(os
.path
.exists(tmpfilename
)):
2304 os
.remove(tmpfilename
);
2305 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2307 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2308 tmpfilename
= pretmpfilename
['Filename'];
2309 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2312 exec_time_start
= time
.time();
2313 with
open(tmpfilename
, 'rb') as ft
:
2316 databytes
= ft
.read(buffersize
[1]);
2317 if not databytes
: break;
2318 datasize
= len(databytes
);
2319 fulldatasize
= datasize
+ fulldatasize
;
2322 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2323 downloaddiff
= fulldatasize
- prevdownsize
;
2324 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2325 prevdownsize
= fulldatasize
;
2328 fdata
= f
.getvalue();
2331 os
.remove(tmpfilename
);
2332 exec_time_end
= time
.time();
2333 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2334 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2338 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2339 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2343 def download_from_url_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2344 global geturls_download_sleep
, havezstd
, havebrotli
;
2346 sleep
= geturls_download_sleep
;
2349 urlparts
= urlparse
.urlparse(httpurl
);
2350 if(isinstance(httpheaders
, list)):
2351 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2352 httpheaders
= fix_header_names(httpheaders
);
2353 if(httpuseragent
is not None):
2354 if('User-Agent' in httpheaders
):
2355 httpheaders
['User-Agent'] = httpuseragent
;
2357 httpuseragent
.update({'User-Agent': httpuseragent
});
2358 if(httpreferer
is not None):
2359 if('Referer' in httpheaders
):
2360 httpheaders
['Referer'] = httpreferer
;
2362 httpuseragent
.update({'Referer': httpreferer
});
2363 if(urlparts
.username
is not None or urlparts
.password
is not None):
2364 if(sys
.version
[0]=="2"):
2365 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2366 if(sys
.version
[0]>="3"):
2367 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2368 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2370 if(postdata
is not None and not isinstance(postdata
, dict)):
2371 postdata
= urlencode(postdata
);
2373 if(httpmethod
=="GET"):
2374 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2375 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2376 elif(httpmethod
=="POST"):
2377 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2378 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
2380 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2381 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2382 except httpcore
.ConnectTimeout
:
2383 log
.info("Error With URL "+httpurl
);
2385 except httpcore
.ConnectError
:
2386 log
.info("Error With URL "+httpurl
);
2388 except socket
.timeout
:
2389 log
.info("Error With URL "+httpurl
);
2391 httpcodeout
= geturls_text
.status
;
2392 httpcodereason
= http_status_to_reason(geturls_text
.status
);
2393 httpversionout
= "1.1";
2394 httpmethodout
= httpmethod
;
2395 httpurlout
= str(httpurl
);
2396 httpheaderout
= geturls_text
.headers
;
2397 httpheadersentout
= httpheaders
;
2398 if(isinstance(httpheaderout
, list)):
2399 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2400 if(sys
.version
[0]=="2"):
2402 prehttpheaderout
= httpheaderout
;
2403 httpheaderkeys
= httpheaderout
.keys();
2404 imax
= len(httpheaderkeys
);
2408 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2410 except AttributeError:
2412 httpheaderout
= fix_header_names(httpheaderout
);
2413 if(isinstance(httpheadersentout
, list)):
2414 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2415 httpheadersentout
= fix_header_names(httpheadersentout
);
2416 log
.info("Downloading URL "+httpurl
);
2417 downloadsize
= httpheaderout
.get('Content-Length');
2418 if(downloadsize
is not None):
2419 downloadsize
= int(downloadsize
);
2420 if downloadsize
is None: downloadsize
= 0;
2423 log
.info("Downloading URL "+httpurl
);
2424 with
BytesIO() as strbuf
:
2426 databytes
= geturls_text
.read(buffersize
);
2427 if not databytes
: break;
2428 datasize
= len(databytes
);
2429 fulldatasize
= datasize
+ fulldatasize
;
2432 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2433 downloaddiff
= fulldatasize
- prevdownsize
;
2434 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2435 prevdownsize
= fulldatasize
;
2436 strbuf
.write(databytes
);
2438 returnval_content
= strbuf
.read();
2439 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2441 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2444 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2446 returnval_content
= zlib
.decompress(returnval_content
);
2449 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2451 returnval_content
= brotli
.decompress(returnval_content
);
2452 except brotli
.error
:
2454 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2456 returnval_content
= zstandard
.decompress(returnval_content
);
2457 except zstandard
.error
:
2459 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
2460 geturls_text
.close();
2463 if(not havehttpcore
):
2464 def download_from_url_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2465 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2469 def download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2470 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2471 exec_time_start
= time
.time();
2472 myhash
= hashlib
.new("sha1");
2473 if(sys
.version
[0]=="2"):
2474 myhash
.update(httpurl
);
2475 myhash
.update(str(buffersize
));
2476 myhash
.update(str(exec_time_start
));
2477 if(sys
.version
[0]>="3"):
2478 myhash
.update(httpurl
.encode('utf-8'));
2479 myhash
.update(str(buffersize
).encode('utf-8'));
2480 myhash
.update(str(exec_time_start
).encode('utf-8'));
2481 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2483 sleep
= geturls_download_sleep
;
2486 pretmpfilename
= download_from_url_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2487 if(not pretmpfilename
):
2489 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2490 tmpfilename
= f
.name
;
2492 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2493 except AttributeError:
2495 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2500 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
2501 f
.write(pretmpfilename
['Content']);
2503 exec_time_end
= time
.time();
2504 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2505 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2508 if(not havehttpcore
):
2509 def download_from_url_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2510 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2514 def download_from_url_to_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2515 global geturls_download_sleep
, havezstd
, havebrotli
;
2517 sleep
= geturls_download_sleep
;
2520 if(not outfile
=="-"):
2521 outpath
= outpath
.rstrip(os
.path
.sep
);
2522 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2523 if(not os
.path
.exists(outpath
)):
2524 os
.makedirs(outpath
);
2525 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2527 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2529 pretmpfilename
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2530 if(not pretmpfilename
):
2532 tmpfilename
= pretmpfilename
['Filename'];
2533 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2535 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2536 exec_time_start
= time
.time();
2537 shutil
.move(tmpfilename
, filepath
);
2539 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2540 except AttributeError:
2542 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2547 exec_time_end
= time
.time();
2548 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2549 if(os
.path
.exists(tmpfilename
)):
2550 os
.remove(tmpfilename
);
2551 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2553 pretmpfilename
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2554 tmpfilename
= pretmpfilename
['Filename'];
2555 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2558 exec_time_start
= time
.time();
2559 with
open(tmpfilename
, 'rb') as ft
:
2562 databytes
= ft
.read(buffersize
[1]);
2563 if not databytes
: break;
2564 datasize
= len(databytes
);
2565 fulldatasize
= datasize
+ fulldatasize
;
2568 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2569 downloaddiff
= fulldatasize
- prevdownsize
;
2570 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2571 prevdownsize
= fulldatasize
;
2574 fdata
= f
.getvalue();
2577 os
.remove(tmpfilename
);
2578 exec_time_end
= time
.time();
2579 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2580 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2583 if(not havehttpcore
):
2584 def download_from_url_to_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2585 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2589 def download_from_url_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2590 global geturls_download_sleep
, havezstd
, havebrotli
;
2592 sleep
= geturls_download_sleep
;
2595 urlparts
= urlparse
.urlparse(httpurl
);
2596 if(isinstance(httpheaders
, list)):
2597 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2598 httpheaders
= fix_header_names(httpheaders
);
2599 if(httpuseragent
is not None):
2600 if('User-Agent' in httpheaders
):
2601 httpheaders
['User-Agent'] = httpuseragent
;
2603 httpuseragent
.update({'User-Agent': httpuseragent
});
2604 if(httpreferer
is not None):
2605 if('Referer' in httpheaders
):
2606 httpheaders
['Referer'] = httpreferer
;
2608 httpuseragent
.update({'Referer': httpreferer
});
2609 if(urlparts
.username
is not None or urlparts
.password
is not None):
2610 if(sys
.version
[0]=="2"):
2611 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2612 if(sys
.version
[0]>="3"):
2613 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2614 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2616 if(postdata
is not None and not isinstance(postdata
, dict)):
2617 postdata
= urlencode(postdata
);
2619 if(httpmethod
=="GET"):
2620 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
2621 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2622 elif(httpmethod
=="POST"):
2623 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
2624 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
2626 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
2627 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2628 except httpcore
.ConnectTimeout
:
2629 log
.info("Error With URL "+httpurl
);
2631 except httpcore
.ConnectError
:
2632 log
.info("Error With URL "+httpurl
);
2634 except socket
.timeout
:
2635 log
.info("Error With URL "+httpurl
);
2637 httpcodeout
= geturls_text
.status
;
2638 httpcodereason
= http_status_to_reason(geturls_text
.status
);
2639 httpversionout
= "1.1";
2640 httpmethodout
= httpmethod
;
2641 httpurlout
= str(httpurl
);
2642 httpheaderout
= geturls_text
.headers
;
2643 httpheadersentout
= httpheaders
;
2644 if(isinstance(httpheaderout
, list)):
2645 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2646 if(sys
.version
[0]=="2"):
2648 prehttpheaderout
= httpheaderout
;
2649 httpheaderkeys
= httpheaderout
.keys();
2650 imax
= len(httpheaderkeys
);
2654 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2656 except AttributeError:
2658 httpheaderout
= fix_header_names(httpheaderout
);
2659 if(isinstance(httpheadersentout
, list)):
2660 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2661 httpheadersentout
= fix_header_names(httpheadersentout
);
2662 log
.info("Downloading URL "+httpurl
);
2663 downloadsize
= httpheaderout
.get('Content-Length');
2664 if(downloadsize
is not None):
2665 downloadsize
= int(downloadsize
);
2666 if downloadsize
is None: downloadsize
= 0;
2669 log
.info("Downloading URL "+httpurl
);
2670 with
BytesIO() as strbuf
:
2672 databytes
= geturls_text
.read(buffersize
);
2673 if not databytes
: break;
2674 datasize
= len(databytes
);
2675 fulldatasize
= datasize
+ fulldatasize
;
2678 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2679 downloaddiff
= fulldatasize
- prevdownsize
;
2680 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2681 prevdownsize
= fulldatasize
;
2682 strbuf
.write(databytes
);
2684 returnval_content
= strbuf
.read();
2685 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2687 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2690 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2692 returnval_content
= zlib
.decompress(returnval_content
);
2695 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2697 returnval_content
= brotli
.decompress(returnval_content
);
2698 except brotli
.error
:
2700 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2702 returnval_content
= zstandard
.decompress(returnval_content
);
2703 except zstandard
.error
:
2705 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
2706 geturls_text
.close();
2709 if(not havehttpcore
):
2710 def download_from_url_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2711 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2715 def download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2716 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2717 exec_time_start
= time
.time();
2718 myhash
= hashlib
.new("sha1");
2719 if(sys
.version
[0]=="2"):
2720 myhash
.update(httpurl
);
2721 myhash
.update(str(buffersize
));
2722 myhash
.update(str(exec_time_start
));
2723 if(sys
.version
[0]>="3"):
2724 myhash
.update(httpurl
.encode('utf-8'));
2725 myhash
.update(str(buffersize
).encode('utf-8'));
2726 myhash
.update(str(exec_time_start
).encode('utf-8'));
2727 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2729 sleep
= geturls_download_sleep
;
2732 pretmpfilename
= download_from_url_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2733 if(not pretmpfilename
):
2735 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2736 tmpfilename
= f
.name
;
2738 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2739 except AttributeError:
2741 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2746 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
2747 f
.write(pretmpfilename
['Content']);
2749 exec_time_end
= time
.time();
2750 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2751 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2754 if(not havehttpcore
):
2755 def download_from_url_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2756 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2760 def download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2761 global geturls_download_sleep
, havezstd
, havebrotli
;
2763 sleep
= geturls_download_sleep
;
2766 if(not outfile
=="-"):
2767 outpath
= outpath
.rstrip(os
.path
.sep
);
2768 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2769 if(not os
.path
.exists(outpath
)):
2770 os
.makedirs(outpath
);
2771 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2773 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2775 pretmpfilename
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2776 if(not pretmpfilename
):
2778 tmpfilename
= pretmpfilename
['Filename'];
2779 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2781 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2782 exec_time_start
= time
.time();
2783 shutil
.move(tmpfilename
, filepath
);
2785 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2786 except AttributeError:
2788 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2793 exec_time_end
= time
.time();
2794 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2795 if(os
.path
.exists(tmpfilename
)):
2796 os
.remove(tmpfilename
);
2797 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2799 pretmpfilename
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2800 tmpfilename
= pretmpfilename
['Filename'];
2801 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2804 exec_time_start
= time
.time();
2805 with
open(tmpfilename
, 'rb') as ft
:
2808 databytes
= ft
.read(buffersize
[1]);
2809 if not databytes
: break;
2810 datasize
= len(databytes
);
2811 fulldatasize
= datasize
+ fulldatasize
;
2814 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2815 downloaddiff
= fulldatasize
- prevdownsize
;
2816 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2817 prevdownsize
= fulldatasize
;
2820 fdata
= f
.getvalue();
2823 os
.remove(tmpfilename
);
2824 exec_time_end
= time
.time();
2825 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2826 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
2830 def download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2831 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2835 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2836 returnval
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2839 if(not haveurllib3
):
2840 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2841 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2845 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2846 returnval
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2849 if(not haveurllib3
):
2850 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2851 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2855 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2856 returnval
= download_from_url_to_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2859 if(not haveurllib3
):
2860 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2861 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2865 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2866 global geturls_download_sleep
, havezstd
, havebrotli
;
2868 sleep
= geturls_download_sleep
;
2871 urlparts
= urlparse
.urlparse(httpurl
);
2872 if(isinstance(httpheaders
, list)):
2873 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2874 httpheaders
= fix_header_names(httpheaders
);
2875 if(httpuseragent
is not None):
2876 if('User-Agent' in httpheaders
):
2877 httpheaders
['User-Agent'] = httpuseragent
;
2879 httpuseragent
.update({'User-Agent': httpuseragent
});
2880 if(httpreferer
is not None):
2881 if('Referer' in httpheaders
):
2882 httpheaders
['Referer'] = httpreferer
;
2884 httpuseragent
.update({'Referer': httpreferer
});
2885 if(urlparts
.username
is not None or urlparts
.password
is not None):
2886 if(sys
.version
[0]=="2"):
2887 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2888 if(sys
.version
[0]>="3"):
2889 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2890 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2892 timeout
= urllib3
.util
.Timeout(connect
=timeout
, read
=timeout
);
2893 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
, timeout
=timeout
);
2894 if(postdata
is not None and not isinstance(postdata
, dict)):
2895 postdata
= urlencode(postdata
);
2897 if(httpmethod
=="GET"):
2898 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2899 elif(httpmethod
=="POST"):
2900 geturls_text
= urllib_pool
.request("POST", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
2902 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2903 except urllib3
.exceptions
.ConnectTimeoutError
:
2904 log
.info("Error With URL "+httpurl
);
2906 except urllib3
.exceptions
.ConnectError
:
2907 log
.info("Error With URL "+httpurl
);
2909 except urllib3
.exceptions
.MaxRetryError
:
2910 log
.info("Error With URL "+httpurl
);
2912 except socket
.timeout
:
2913 log
.info("Error With URL "+httpurl
);
2916 log
.info("Error With URL "+httpurl
);
2918 httpcodeout
= geturls_text
.status
;
2919 httpcodereason
= geturls_text
.reason
;
2920 if(geturls_text
.version
=="10"):
2921 httpversionout
= "1.0";
2923 httpversionout
= "1.1";
2924 httpmethodout
= httpmethod
;
2925 httpurlout
= geturls_text
.geturl();
2926 httpheaderout
= geturls_text
.info();
2927 httpheadersentout
= httpheaders
;
2928 if(isinstance(httpheaderout
, list)):
2929 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2930 if(sys
.version
[0]=="2"):
2932 prehttpheaderout
= httpheaderout
;
2933 httpheaderkeys
= httpheaderout
.keys();
2934 imax
= len(httpheaderkeys
);
2938 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2940 except AttributeError:
2942 httpheaderout
= fix_header_names(httpheaderout
);
2943 if(isinstance(httpheadersentout
, list)):
2944 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2945 httpheadersentout
= fix_header_names(httpheadersentout
);
2946 log
.info("Downloading URL "+httpurl
);
2947 downloadsize
= httpheaderout
.get('Content-Length');
2948 if(downloadsize
is not None):
2949 downloadsize
= int(downloadsize
);
2950 if downloadsize
is None: downloadsize
= 0;
2953 log
.info("Downloading URL "+httpurl
);
2954 with
BytesIO() as strbuf
:
2956 databytes
= geturls_text
.read(buffersize
);
2957 if not databytes
: break;
2958 datasize
= len(databytes
);
2959 fulldatasize
= datasize
+ fulldatasize
;
2962 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2963 downloaddiff
= fulldatasize
- prevdownsize
;
2964 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2965 prevdownsize
= fulldatasize
;
2966 strbuf
.write(databytes
);
2968 returnval_content
= strbuf
.read();
2969 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2971 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2974 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2976 returnval_content
= zlib
.decompress(returnval_content
);
2979 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2981 returnval_content
= brotli
.decompress(returnval_content
);
2982 except brotli
.error
:
2984 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2986 returnval_content
= zstandard
.decompress(returnval_content
);
2987 except zstandard
.error
:
2989 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
2990 geturls_text
.close();
2993 if(not haveurllib3
):
2994 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2995 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2999 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3000 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3001 exec_time_start
= time
.time();
3002 myhash
= hashlib
.new("sha1");
3003 if(sys
.version
[0]=="2"):
3004 myhash
.update(httpurl
);
3005 myhash
.update(str(buffersize
));
3006 myhash
.update(str(exec_time_start
));
3007 if(sys
.version
[0]>="3"):
3008 myhash
.update(httpurl
.encode('utf-8'));
3009 myhash
.update(str(buffersize
).encode('utf-8'));
3010 myhash
.update(str(exec_time_start
).encode('utf-8'));
3011 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3013 sleep
= geturls_download_sleep
;
3016 pretmpfilename
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3017 if(not pretmpfilename
):
3019 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3020 tmpfilename
= f
.name
;
3022 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3023 except AttributeError:
3025 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3030 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
3031 f
.write(pretmpfilename
['Content']);
3033 exec_time_end
= time
.time();
3034 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3035 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3038 if(not haveurllib3
):
3039 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3040 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3044 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3045 global geturls_download_sleep
, havezstd
, havebrotli
;
3047 sleep
= geturls_download_sleep
;
3050 if(not outfile
=="-"):
3051 outpath
= outpath
.rstrip(os
.path
.sep
);
3052 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3053 if(not os
.path
.exists(outpath
)):
3054 os
.makedirs(outpath
);
3055 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3057 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3059 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3060 if(not pretmpfilename
):
3062 tmpfilename
= pretmpfilename
['Filename'];
3063 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3065 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3066 exec_time_start
= time
.time();
3067 shutil
.move(tmpfilename
, filepath
);
3069 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3070 except AttributeError:
3072 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3077 exec_time_end
= time
.time();
3078 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3079 if(os
.path
.exists(tmpfilename
)):
3080 os
.remove(tmpfilename
);
3081 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3083 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3084 tmpfilename
= pretmpfilename
['Filename'];
3085 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3088 exec_time_start
= time
.time();
3089 with
open(tmpfilename
, 'rb') as ft
:
3092 databytes
= ft
.read(buffersize
[1]);
3093 if not databytes
: break;
3094 datasize
= len(databytes
);
3095 fulldatasize
= datasize
+ fulldatasize
;
3098 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3099 downloaddiff
= fulldatasize
- prevdownsize
;
3100 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3101 prevdownsize
= fulldatasize
;
3104 fdata
= f
.getvalue();
3107 os
.remove(tmpfilename
);
3108 exec_time_end
= time
.time();
3109 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3110 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3113 if(not haveurllib3
):
3114 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3115 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3119 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3120 global geturls_download_sleep
, havezstd
, havebrotli
;
3122 sleep
= geturls_download_sleep
;
3125 urlparts
= urlparse
.urlparse(httpurl
);
3126 if(isinstance(httpheaders
, list)):
3127 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3128 httpheaders
= fix_header_names(httpheaders
);
3129 if(httpuseragent
is not None):
3130 if('User-Agent' in httpheaders
):
3131 httpheaders
['User-Agent'] = httpuseragent
;
3133 httpuseragent
.update({'User-Agent': httpuseragent
});
3134 if(httpreferer
is not None):
3135 if('Referer' in httpheaders
):
3136 httpheaders
['Referer'] = httpreferer
;
3138 httpuseragent
.update({'Referer': httpreferer
});
3139 if(urlparts
.username
is not None or urlparts
.password
is not None):
3140 if(sys
.version
[0]=="2"):
3141 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3142 if(sys
.version
[0]>="3"):
3143 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3144 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3145 geturls_opener
= mechanize
.Browser();
3146 if(isinstance(httpheaders
, dict)):
3147 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
3149 geturls_opener
.addheaders
= httpheaders
;
3150 geturls_opener
.set_cookiejar(httpcookie
);
3151 geturls_opener
.set_handle_robots(False);
3152 if(postdata
is not None and not isinstance(postdata
, dict)):
3153 postdata
= urlencode(postdata
);
3155 if(httpmethod
=="GET"):
3156 geturls_text
= geturls_opener
.open(httpurl
);
3157 elif(httpmethod
=="POST"):
3158 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
3160 geturls_text
= geturls_opener
.open(httpurl
);
3161 except mechanize
.HTTPError
as geturls_text_error
:
3162 geturls_text
= geturls_text_error
;
3163 log
.info("Error With URL "+httpurl
);
3165 log
.info("Error With URL "+httpurl
);
3167 except socket
.timeout
:
3168 log
.info("Error With URL "+httpurl
);
3170 httpcodeout
= geturls_text
.code
;
3171 httpcodereason
= geturls_text
.msg
;
3172 httpversionout
= "1.1";
3173 httpmethodout
= httpmethod
;
3174 httpurlout
= geturls_text
.geturl();
3175 httpheaderout
= geturls_text
.info();
3176 reqhead
= geturls_opener
.request
;
3177 httpheadersentout
= reqhead
.header_items();
3178 if(isinstance(httpheaderout
, list)):
3179 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
3180 if(sys
.version
[0]=="2"):
3182 prehttpheaderout
= httpheaderout
;
3183 httpheaderkeys
= httpheaderout
.keys();
3184 imax
= len(httpheaderkeys
);
3188 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3190 except AttributeError:
3192 httpheaderout
= fix_header_names(httpheaderout
);
3193 if(isinstance(httpheadersentout
, list)):
3194 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
3195 httpheadersentout
= fix_header_names(httpheadersentout
);
3196 log
.info("Downloading URL "+httpurl
);
3197 downloadsize
= httpheaderout
.get('Content-Length');
3198 if(downloadsize
is not None):
3199 downloadsize
= int(downloadsize
);
3200 if downloadsize
is None: downloadsize
= 0;
3203 log
.info("Downloading URL "+httpurl
);
3204 with
BytesIO() as strbuf
:
3206 databytes
= geturls_text
.read(buffersize
);
3207 if not databytes
: break;
3208 datasize
= len(databytes
);
3209 fulldatasize
= datasize
+ fulldatasize
;
3212 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3213 downloaddiff
= fulldatasize
- prevdownsize
;
3214 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3215 prevdownsize
= fulldatasize
;
3216 strbuf
.write(databytes
);
3218 returnval_content
= strbuf
.read();
3219 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3221 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3224 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3226 returnval_content
= zlib
.decompress(returnval_content
);
3229 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3231 returnval_content
= brotli
.decompress(returnval_content
);
3232 except brotli
.error
:
3234 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3236 returnval_content
= zstandard
.decompress(returnval_content
);
3237 except zstandard
.error
:
3239 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
3240 geturls_text
.close();
3243 if(not havemechanize
):
3244 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3245 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3249 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3250 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3251 exec_time_start
= time
.time();
3252 myhash
= hashlib
.new("sha1");
3253 if(sys
.version
[0]=="2"):
3254 myhash
.update(httpurl
);
3255 myhash
.update(str(buffersize
));
3256 myhash
.update(str(exec_time_start
));
3257 if(sys
.version
[0]>="3"):
3258 myhash
.update(httpurl
.encode('utf-8'));
3259 myhash
.update(str(buffersize
).encode('utf-8'));
3260 myhash
.update(str(exec_time_start
).encode('utf-8'));
3261 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3263 sleep
= geturls_download_sleep
;
3266 pretmpfilename
= download_from_url_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3267 if(not pretmpfilename
):
3269 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3270 tmpfilename
= f
.name
;
3272 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3273 except AttributeError:
3275 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3280 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
3281 f
.write(pretmpfilename
['Content']);
3283 exec_time_end
= time
.time();
3284 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3285 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3288 if(not havemechanize
):
3289 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3290 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3294 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3295 global geturls_download_sleep
, havezstd
, havebrotli
;
3297 sleep
= geturls_download_sleep
;
3300 if(not outfile
=="-"):
3301 outpath
= outpath
.rstrip(os
.path
.sep
);
3302 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3303 if(not os
.path
.exists(outpath
)):
3304 os
.makedirs(outpath
);
3305 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3307 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3309 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3310 if(not pretmpfilename
):
3312 tmpfilename
= pretmpfilename
['Filename'];
3313 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3315 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3316 exec_time_start
= time
.time();
3317 shutil
.move(tmpfilename
, filepath
);
3319 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3320 except AttributeError:
3322 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3327 exec_time_end
= time
.time();
3328 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3329 if(os
.path
.exists(tmpfilename
)):
3330 os
.remove(tmpfilename
);
3331 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3333 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3334 tmpfilename
= pretmpfilename
['Filename'];
3335 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3338 exec_time_start
= time
.time();
3339 with
open(tmpfilename
, 'rb') as ft
:
3342 databytes
= ft
.read(buffersize
[1]);
3343 if not databytes
: break;
3344 datasize
= len(databytes
);
3345 fulldatasize
= datasize
+ fulldatasize
;
3348 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3349 downloaddiff
= fulldatasize
- prevdownsize
;
3350 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3351 prevdownsize
= fulldatasize
;
3354 fdata
= f
.getvalue();
3357 os
.remove(tmpfilename
);
3358 exec_time_end
= time
.time();
3359 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3360 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3363 if(not havemechanize
):
3364 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3365 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3369 def download_from_url_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3370 global geturls_download_sleep
, havezstd
, havebrotli
;
3372 sleep
= geturls_download_sleep
;
3375 urlparts
= urlparse
.urlparse(httpurl
);
3376 if(isinstance(httpheaders
, list)):
3377 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3378 httpheaders
= fix_header_names(httpheaders
);
3379 if(httpuseragent
is not None):
3380 if('User-Agent' in httpheaders
):
3381 httpheaders
['User-Agent'] = httpuseragent
;
3383 httpuseragent
.update({'User-Agent': httpuseragent
});
3384 if(httpreferer
is not None):
3385 if('Referer' in httpheaders
):
3386 httpheaders
['Referer'] = httpreferer
;
3388 httpuseragent
.update({'Referer': httpreferer
});
3389 if(urlparts
.username
is not None or urlparts
.password
is not None):
3390 if(sys
.version
[0]=="2"):
3391 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3392 if(sys
.version
[0]>="3"):
3393 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3394 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3395 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
3396 if(isinstance(httpheaders
, dict)):
3397 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
3398 geturls_opener
.addheaders
= httpheaders
;
3400 if(postdata
is not None and not isinstance(postdata
, dict)):
3401 postdata
= urlencode(postdata
);
3402 retrieved_body
= BytesIO();
3403 retrieved_headers
= BytesIO();
3405 if(httpmethod
=="GET"):
3406 geturls_text
= pycurl
.Curl();
3407 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3408 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3409 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3410 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3411 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3412 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3413 geturls_text
.perform();
3414 elif(httpmethod
=="POST"):
3415 geturls_text
= pycurl
.Curl();
3416 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3417 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3418 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3419 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3420 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3421 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3422 geturls_text
.setopt(geturls_text
.POST
, True);
3423 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
3424 geturls_text
.perform();
3426 geturls_text
= pycurl
.Curl();
3427 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3428 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3429 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3430 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3431 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3432 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3433 geturls_text
.perform();
3434 retrieved_headers
.seek(0);
3435 if(sys
.version
[0]=="2"):
3436 pycurlhead
= retrieved_headers
.read();
3437 if(sys
.version
[0]>="3"):
3438 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
3439 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
3440 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
3441 retrieved_body
.seek(0);
3442 except socket
.timeout
:
3443 log
.info("Error With URL "+httpurl
);
3445 except socket
.gaierror
:
3446 log
.info("Error With URL "+httpurl
);
3449 log
.info("Error With URL "+httpurl
);
3451 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
3452 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
3453 httpversionout
= pyhttpverinfo
[0];
3454 httpmethodout
= httpmethod
;
3455 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
3456 httpheaderout
= pycurlheadersout
;
3457 httpheadersentout
= httpheaders
;
3458 if(isinstance(httpheaderout
, list)):
3459 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
3460 if(sys
.version
[0]=="2"):
3462 prehttpheaderout
= httpheaderout
;
3463 httpheaderkeys
= httpheaderout
.keys();
3464 imax
= len(httpheaderkeys
);
3468 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3470 except AttributeError:
3472 httpheaderout
= fix_header_names(httpheaderout
);
3473 if(isinstance(httpheadersentout
, list)):
3474 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
3475 httpheadersentout
= fix_header_names(httpheadersentout
);
3476 log
.info("Downloading URL "+httpurl
);
3477 downloadsize
= httpheaderout
.get('Content-Length');
3478 if(downloadsize
is not None):
3479 downloadsize
= int(downloadsize
);
3480 if downloadsize
is None: downloadsize
= 0;
3483 log
.info("Downloading URL "+httpurl
);
3484 with
BytesIO() as strbuf
:
3486 databytes
= retrieved_body
.read(buffersize
);
3487 if not databytes
: break;
3488 datasize
= len(databytes
);
3489 fulldatasize
= datasize
+ fulldatasize
;
3492 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3493 downloaddiff
= fulldatasize
- prevdownsize
;
3494 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3495 prevdownsize
= fulldatasize
;
3496 strbuf
.write(databytes
);
3498 returnval_content
= strbuf
.read();
3499 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3501 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3504 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3506 returnval_content
= zlib
.decompress(returnval_content
);
3509 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3511 returnval_content
= brotli
.decompress(returnval_content
);
3512 except brotli
.error
:
3514 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3516 returnval_content
= zstandard
.decompress(returnval_content
);
3517 except zstandard
.error
:
3519 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
3520 geturls_text
.close();
3524 def download_from_url_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3525 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3529 def download_from_url_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3530 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3531 exec_time_start
= time
.time();
3532 myhash
= hashlib
.new("sha1");
3533 if(sys
.version
[0]=="2"):
3534 myhash
.update(httpurl
);
3535 myhash
.update(str(buffersize
));
3536 myhash
.update(str(exec_time_start
));
3537 if(sys
.version
[0]>="3"):
3538 myhash
.update(httpurl
.encode('utf-8'));
3539 myhash
.update(str(buffersize
).encode('utf-8'));
3540 myhash
.update(str(exec_time_start
).encode('utf-8'));
3541 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3543 sleep
= geturls_download_sleep
;
3546 pretmpfilename
= download_from_url_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3547 if(not pretmpfilename
):
3549 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3550 tmpfilename
= f
.name
;
3552 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3553 except AttributeError:
3555 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3560 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
3561 f
.write(pretmpfilename
['Content']);
3563 exec_time_end
= time
.time();
3564 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3565 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3569 def download_from_url_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3570 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3574 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3575 global geturls_download_sleep
, havezstd
, havebrotli
;
3577 sleep
= geturls_download_sleep
;
3580 if(not outfile
=="-"):
3581 outpath
= outpath
.rstrip(os
.path
.sep
);
3582 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3583 if(not os
.path
.exists(outpath
)):
3584 os
.makedirs(outpath
);
3585 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3587 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3589 pretmpfilename
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3590 if(not pretmpfilename
):
3592 tmpfilename
= pretmpfilename
['Filename'];
3593 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3595 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3596 exec_time_start
= time
.time();
3597 shutil
.move(tmpfilename
, filepath
);
3599 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3600 except AttributeError:
3602 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3607 exec_time_end
= time
.time();
3608 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3609 if(os
.path
.exists(tmpfilename
)):
3610 os
.remove(tmpfilename
);
3611 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3613 pretmpfilename
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3614 tmpfilename
= pretmpfilename
['Filename'];
3615 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3618 exec_time_start
= time
.time();
3619 with
open(tmpfilename
, 'rb') as ft
:
3622 databytes
= ft
.read(buffersize
[1]);
3623 if not databytes
: break;
3624 datasize
= len(databytes
);
3625 fulldatasize
= datasize
+ fulldatasize
;
3628 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3629 downloaddiff
= fulldatasize
- prevdownsize
;
3630 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3631 prevdownsize
= fulldatasize
;
3634 fdata
= f
.getvalue();
3637 os
.remove(tmpfilename
);
3638 exec_time_end
= time
.time();
3639 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3640 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3644 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3645 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3649 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3650 global geturls_download_sleep
, havezstd
, havebrotli
;
3652 sleep
= geturls_download_sleep
;
3655 urlparts
= urlparse
.urlparse(httpurl
);
3656 if(isinstance(httpheaders
, list)):
3657 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3658 httpheaders
= fix_header_names(httpheaders
);
3659 if(httpuseragent
is not None):
3660 if('User-Agent' in httpheaders
):
3661 httpheaders
['User-Agent'] = httpuseragent
;
3663 httpuseragent
.update({'User-Agent': httpuseragent
});
3664 if(httpreferer
is not None):
3665 if('Referer' in httpheaders
):
3666 httpheaders
['Referer'] = httpreferer
;
3668 httpuseragent
.update({'Referer': httpreferer
});
3669 if(urlparts
.username
is not None or urlparts
.password
is not None):
3670 if(sys
.version
[0]=="2"):
3671 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3672 if(sys
.version
[0]>="3"):
3673 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3674 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3675 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
3676 if(isinstance(httpheaders
, dict)):
3677 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
3678 geturls_opener
.addheaders
= httpheaders
;
3680 if(postdata
is not None and not isinstance(postdata
, dict)):
3681 postdata
= urlencode(postdata
);
3682 retrieved_body
= BytesIO();
3683 retrieved_headers
= BytesIO();
3685 if(httpmethod
=="GET"):
3686 geturls_text
= pycurl
.Curl();
3687 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3688 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
3689 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3690 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3691 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3692 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3693 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3694 geturls_text
.perform();
3695 elif(httpmethod
=="POST"):
3696 geturls_text
= pycurl
.Curl();
3697 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3698 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
3699 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3700 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3701 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3702 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3703 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3704 geturls_text
.setopt(geturls_text
.POST
, True);
3705 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
3706 geturls_text
.perform();
3708 geturls_text
= pycurl
.Curl();
3709 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3710 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
3711 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3712 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3713 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3714 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3715 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3716 geturls_text
.perform();
3717 retrieved_headers
.seek(0);
3718 if(sys
.version
[0]=="2"):
3719 pycurlhead
= retrieved_headers
.read();
3720 if(sys
.version
[0]>="3"):
3721 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
3722 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip())[0];
3723 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
3724 retrieved_body
.seek(0);
3725 except socket
.timeout
:
3726 log
.info("Error With URL "+httpurl
);
3728 except socket
.gaierror
:
3729 log
.info("Error With URL "+httpurl
);
3732 log
.info("Error With URL "+httpurl
);
3734 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
3735 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
3736 httpversionout
= pyhttpverinfo
[0];
3737 httpmethodout
= httpmethod
;
3738 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
3739 httpheaderout
= pycurlheadersout
;
3740 httpheadersentout
= httpheaders
;
3741 if(isinstance(httpheaderout
, list)):
3742 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
3743 if(sys
.version
[0]=="2"):
3745 prehttpheaderout
= httpheaderout
;
3746 httpheaderkeys
= httpheaderout
.keys();
3747 imax
= len(httpheaderkeys
);
3751 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3753 except AttributeError:
3755 httpheaderout
= fix_header_names(httpheaderout
);
3756 if(isinstance(httpheadersentout
, list)):
3757 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
3758 httpheadersentout
= fix_header_names(httpheadersentout
);
3759 log
.info("Downloading URL "+httpurl
);
3760 downloadsize
= httpheaderout
.get('Content-Length');
3761 if(downloadsize
is not None):
3762 downloadsize
= int(downloadsize
);
3763 if downloadsize
is None: downloadsize
= 0;
3766 log
.info("Downloading URL "+httpurl
);
3767 with
BytesIO() as strbuf
:
3769 databytes
= retrieved_body
.read(buffersize
);
3770 if not databytes
: break;
3771 datasize
= len(databytes
);
3772 fulldatasize
= datasize
+ fulldatasize
;
3775 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3776 downloaddiff
= fulldatasize
- prevdownsize
;
3777 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3778 prevdownsize
= fulldatasize
;
3779 strbuf
.write(databytes
);
3781 returnval_content
= strbuf
.read();
3782 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3784 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3787 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3789 returnval_content
= zlib
.decompress(returnval_content
);
3792 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3794 returnval_content
= brotli
.decompress(returnval_content
);
3795 except brotli
.error
:
3797 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3799 returnval_content
= zstandard
.decompress(returnval_content
);
3800 except zstandard
.error
:
3802 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
3803 geturls_text
.close();
3807 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3808 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3812 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3813 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3814 exec_time_start
= time
.time();
3815 myhash
= hashlib
.new("sha1");
3816 if(sys
.version
[0]=="2"):
3817 myhash
.update(httpurl
);
3818 myhash
.update(str(buffersize
));
3819 myhash
.update(str(exec_time_start
));
3820 if(sys
.version
[0]>="3"):
3821 myhash
.update(httpurl
.encode('utf-8'));
3822 myhash
.update(str(buffersize
).encode('utf-8'));
3823 myhash
.update(str(exec_time_start
).encode('utf-8'));
3824 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3826 sleep
= geturls_download_sleep
;
3829 pretmpfilename
= download_from_url_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3830 if(not pretmpfilename
):
3832 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3833 tmpfilename
= f
.name
;
3835 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3836 except AttributeError:
3838 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3843 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
3844 f
.write(pretmpfilename
['Content']);
3846 exec_time_end
= time
.time();
3847 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3848 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3852 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3853 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3857 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3858 global geturls_download_sleep
, havezstd
, havebrotli
;
3860 sleep
= geturls_download_sleep
;
3863 if(not outfile
=="-"):
3864 outpath
= outpath
.rstrip(os
.path
.sep
);
3865 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3866 if(not os
.path
.exists(outpath
)):
3867 os
.makedirs(outpath
);
3868 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3870 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3872 pretmpfilename
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3873 if(not pretmpfilename
):
3875 tmpfilename
= pretmpfilename
['Filename'];
3876 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3878 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3879 exec_time_start
= time
.time();
3880 shutil
.move(tmpfilename
, filepath
);
3882 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3883 except AttributeError:
3885 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3890 exec_time_end
= time
.time();
3891 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3892 if(os
.path
.exists(tmpfilename
)):
3893 os
.remove(tmpfilename
);
3894 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3896 pretmpfilename
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3897 tmpfilename
= pretmpfilename
['Filename'];
3898 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3901 exec_time_start
= time
.time();
3902 with
open(tmpfilename
, 'rb') as ft
:
3905 databytes
= ft
.read(buffersize
[1]);
3906 if not databytes
: break;
3907 datasize
= len(databytes
);
3908 fulldatasize
= datasize
+ fulldatasize
;
3911 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3912 downloaddiff
= fulldatasize
- prevdownsize
;
3913 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3914 prevdownsize
= fulldatasize
;
3917 fdata
= f
.getvalue();
3920 os
.remove(tmpfilename
);
3921 exec_time_end
= time
.time();
3922 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3923 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
3927 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3928 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3932 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3933 global geturls_download_sleep
, havezstd
, havebrotli
;
3935 sleep
= geturls_download_sleep
;
3938 urlparts
= urlparse
.urlparse(httpurl
);
3939 if(isinstance(httpheaders
, list)):
3940 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3941 httpheaders
= fix_header_names(httpheaders
);
3942 if(httpuseragent
is not None):
3943 if('User-Agent' in httpheaders
):
3944 httpheaders
['User-Agent'] = httpuseragent
;
3946 httpuseragent
.update({'User-Agent': httpuseragent
});
3947 if(httpreferer
is not None):
3948 if('Referer' in httpheaders
):
3949 httpheaders
['Referer'] = httpreferer
;
3951 httpuseragent
.update({'Referer': httpreferer
});
3952 if(urlparts
.username
is not None or urlparts
.password
is not None):
3953 if(sys
.version
[0]=="2"):
3954 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3955 if(sys
.version
[0]>="3"):
3956 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3957 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3958 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
3959 if(isinstance(httpheaders
, dict)):
3960 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
3961 geturls_opener
.addheaders
= httpheaders
;
3963 if(postdata
is not None and not isinstance(postdata
, dict)):
3964 postdata
= urlencode(postdata
);
3965 retrieved_body
= BytesIO();
3966 retrieved_headers
= BytesIO();
3968 if(httpmethod
=="GET"):
3969 geturls_text
= pycurl
.Curl();
3970 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3971 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
3972 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3973 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3974 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3975 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3976 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3977 geturls_text
.perform();
3978 elif(httpmethod
=="POST"):
3979 geturls_text
= pycurl
.Curl();
3980 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3981 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
3982 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3983 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3984 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3985 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3986 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3987 geturls_text
.setopt(geturls_text
.POST
, True);
3988 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
3989 geturls_text
.perform();
3991 geturls_text
= pycurl
.Curl();
3992 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3993 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
3994 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3995 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3996 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3997 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3998 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3999 geturls_text
.perform();
4000 retrieved_headers
.seek(0);
4001 if(sys
.version
[0]=="2"):
4002 pycurlhead
= retrieved_headers
.read();
4003 if(sys
.version
[0]>="3"):
4004 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
4005 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
4006 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
4007 retrieved_body
.seek(0);
4008 except socket
.timeout
:
4009 log
.info("Error With URL "+httpurl
);
4011 except socket
.gaierror
:
4012 log
.info("Error With URL "+httpurl
);
4015 log
.info("Error With URL "+httpurl
);
4017 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
4018 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
4019 httpversionout
= pyhttpverinfo
[0];
4020 httpmethodout
= httpmethod
;
4021 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
4022 httpheaderout
= pycurlheadersout
;
4023 httpheadersentout
= httpheaders
;
4024 if(isinstance(httpheaderout
, list)):
4025 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
4026 if(sys
.version
[0]=="2"):
4028 prehttpheaderout
= httpheaderout
;
4029 httpheaderkeys
= httpheaderout
.keys();
4030 imax
= len(httpheaderkeys
);
4034 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
4036 except AttributeError:
4038 httpheaderout
= fix_header_names(httpheaderout
);
4039 if(isinstance(httpheadersentout
, list)):
4040 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
4041 httpheadersentout
= fix_header_names(httpheadersentout
);
4042 log
.info("Downloading URL "+httpurl
);
4043 downloadsize
= httpheaderout
.get('Content-Length');
4044 if(downloadsize
is not None):
4045 downloadsize
= int(downloadsize
);
4046 if downloadsize
is None: downloadsize
= 0;
4049 log
.info("Downloading URL "+httpurl
);
4050 with
BytesIO() as strbuf
:
4052 databytes
= retrieved_body
.read(buffersize
);
4053 if not databytes
: break;
4054 datasize
= len(databytes
);
4055 fulldatasize
= datasize
+ fulldatasize
;
4058 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4059 downloaddiff
= fulldatasize
- prevdownsize
;
4060 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4061 prevdownsize
= fulldatasize
;
4062 strbuf
.write(databytes
);
4064 returnval_content
= strbuf
.read();
4065 if(httpheaderout
.get("Content-Encoding")=="gzip"):
4067 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
4070 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
4072 returnval_content
= zlib
.decompress(returnval_content
);
4075 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
4077 returnval_content
= brotli
.decompress(returnval_content
);
4078 except brotli
.error
:
4080 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
4082 returnval_content
= zstandard
.decompress(returnval_content
);
4083 except zstandard
.error
:
4085 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
};
4086 geturls_text
.close();
4090 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4091 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4095 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4096 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4097 exec_time_start
= time
.time();
4098 myhash
= hashlib
.new("sha1");
4099 if(sys
.version
[0]=="2"):
4100 myhash
.update(httpurl
);
4101 myhash
.update(str(buffersize
));
4102 myhash
.update(str(exec_time_start
));
4103 if(sys
.version
[0]>="3"):
4104 myhash
.update(httpurl
.encode('utf-8'));
4105 myhash
.update(str(buffersize
).encode('utf-8'));
4106 myhash
.update(str(exec_time_start
).encode('utf-8'));
4107 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4109 sleep
= geturls_download_sleep
;
4112 pretmpfilename
= download_from_url_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4113 if(not pretmpfilename
):
4115 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4116 tmpfilename
= f
.name
;
4118 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4119 except AttributeError:
4121 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4126 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
4127 f
.write(pretmpfilename
['Content']);
4129 exec_time_end
= time
.time();
4130 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4131 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4135 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4136 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4140 def download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4141 global geturls_download_sleep
, havezstd
, havebrotli
;
4143 sleep
= geturls_download_sleep
;
4146 if(not outfile
=="-"):
4147 outpath
= outpath
.rstrip(os
.path
.sep
);
4148 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4149 if(not os
.path
.exists(outpath
)):
4150 os
.makedirs(outpath
);
4151 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4153 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4155 pretmpfilename
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4156 if(not pretmpfilename
):
4158 tmpfilename
= pretmpfilename
['Filename'];
4159 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4161 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4162 exec_time_start
= time
.time();
4163 shutil
.move(tmpfilename
, filepath
);
4165 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4166 except AttributeError:
4168 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4173 exec_time_end
= time
.time();
4174 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4175 if(os
.path
.exists(tmpfilename
)):
4176 os
.remove(tmpfilename
);
4177 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4179 pretmpfilename
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4180 tmpfilename
= pretmpfilename
['Filename'];
4181 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4184 exec_time_start
= time
.time();
4185 with
open(tmpfilename
, 'rb') as ft
:
4188 databytes
= ft
.read(buffersize
[1]);
4189 if not databytes
: break;
4190 datasize
= len(databytes
);
4191 fulldatasize
= datasize
+ fulldatasize
;
4194 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4195 downloaddiff
= fulldatasize
- prevdownsize
;
4196 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4197 prevdownsize
= fulldatasize
;
4200 fdata
= f
.getvalue();
4203 os
.remove(tmpfilename
);
4204 exec_time_end
= time
.time();
4205 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4206 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4210 def download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4211 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4214 def download_file_from_ftp_file(url
):
4215 urlparts
= urlparse
.urlparse(url
);
4216 file_name
= os
.path
.basename(urlparts
.path
);
4217 file_dir
= os
.path
.dirname(urlparts
.path
);
4218 if(urlparts
.username
is not None):
4219 ftp_username
= urlparts
.username
;
4221 ftp_username
= "anonymous";
4222 if(urlparts
.password
is not None):
4223 ftp_password
= urlparts
.password
;
4224 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4225 ftp_password
= "anonymous";
4228 if(urlparts
.scheme
=="ftp"):
4230 elif(urlparts
.scheme
=="ftps"):
4234 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4236 ftp_port
= urlparts
.port
;
4237 if(urlparts
.port
is None):
4240 ftp
.connect(urlparts
.hostname
, ftp_port
);
4241 except socket
.gaierror
:
4242 log
.info("Error With URL "+httpurl
);
4244 except socket
.timeout
:
4245 log
.info("Error With URL "+httpurl
);
4247 ftp
.login(urlparts
.username
, urlparts
.password
);
4248 if(urlparts
.scheme
=="ftps"):
4250 ftpfile
= BytesIO();
4251 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
4252 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
4257 def download_file_from_ftp_string(url
):
4258 ftpfile
= download_file_from_ftp_file(url
);
4259 return ftpfile
.read();
4261 def download_from_url_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4262 global geturls_download_sleep
, havezstd
, havebrotli
;
4264 sleep
= geturls_download_sleep
;
4267 urlparts
= urlparse
.urlparse(httpurl
);
4268 if(isinstance(httpheaders
, list)):
4269 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4270 httpheaders
= fix_header_names(httpheaders
);
4271 if(httpuseragent
is not None):
4272 if('User-Agent' in httpheaders
):
4273 httpheaders
['User-Agent'] = httpuseragent
;
4275 httpuseragent
.update({'User-Agent': httpuseragent
});
4276 if(httpreferer
is not None):
4277 if('Referer' in httpheaders
):
4278 httpheaders
['Referer'] = httpreferer
;
4280 httpuseragent
.update({'Referer': httpreferer
});
4281 if(isinstance(httpheaders
, dict)):
4282 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
4284 geturls_text
= download_file_from_ftp_file(httpurl
);
4285 if(not geturls_text
):
4287 downloadsize
= None;
4288 if(downloadsize
is not None):
4289 downloadsize
= int(downloadsize
);
4290 if downloadsize
is None: downloadsize
= 0;
4293 log
.info("Downloading URL "+httpurl
);
4294 with
BytesIO() as strbuf
:
4296 databytes
= geturls_text
.read(buffersize
);
4297 if not databytes
: break;
4298 datasize
= len(databytes
);
4299 fulldatasize
= datasize
+ fulldatasize
;
4302 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4303 downloaddiff
= fulldatasize
- prevdownsize
;
4304 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4305 prevdownsize
= fulldatasize
;
4306 strbuf
.write(databytes
);
4308 returnval_content
= strbuf
.read();
4309 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
4310 geturls_text
.close();
4313 def download_from_url_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4314 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4315 exec_time_start
= time
.time();
4316 myhash
= hashlib
.new("sha1");
4317 if(sys
.version
[0]=="2"):
4318 myhash
.update(httpurl
);
4319 myhash
.update(str(buffersize
));
4320 myhash
.update(str(exec_time_start
));
4321 if(sys
.version
[0]>="3"):
4322 myhash
.update(httpurl
.encode('utf-8'));
4323 myhash
.update(str(buffersize
).encode('utf-8'));
4324 myhash
.update(str(exec_time_start
).encode('utf-8'));
4325 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4327 sleep
= geturls_download_sleep
;
4330 pretmpfilename
= download_from_url_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4331 if(not pretmpfilename
):
4333 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4334 tmpfilename
= f
.name
;
4336 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4337 except AttributeError:
4339 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4344 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
4345 f
.write(pretmpfilename
['Content']);
4347 exec_time_end
= time
.time();
4348 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4349 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4352 def download_from_url_to_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4353 global geturls_download_sleep
, havezstd
, havebrotli
;
4355 sleep
= geturls_download_sleep
;
4358 if(not outfile
=="-"):
4359 outpath
= outpath
.rstrip(os
.path
.sep
);
4360 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4361 if(not os
.path
.exists(outpath
)):
4362 os
.makedirs(outpath
);
4363 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4365 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4367 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4368 if(not pretmpfilename
):
4370 tmpfilename
= pretmpfilename
['Filename'];
4371 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4373 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4374 exec_time_start
= time
.time();
4375 shutil
.move(tmpfilename
, filepath
);
4376 exec_time_end
= time
.time();
4377 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4378 if(os
.path
.exists(tmpfilename
)):
4379 os
.remove(tmpfilename
);
4380 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4382 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4383 tmpfilename
= pretmpfilename
['Filename'];
4384 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4387 exec_time_start
= time
.time();
4388 with
open(tmpfilename
, 'rb') as ft
:
4391 databytes
= ft
.read(buffersize
[1]);
4392 if not databytes
: break;
4393 datasize
= len(databytes
);
4394 fulldatasize
= datasize
+ fulldatasize
;
4397 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4398 downloaddiff
= fulldatasize
- prevdownsize
;
4399 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4400 prevdownsize
= fulldatasize
;
4403 fdata
= f
.getvalue();
4406 os
.remove(tmpfilename
);
4407 exec_time_end
= time
.time();
4408 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4409 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4412 def upload_file_to_ftp_file(ftpfile
, url
):
4413 urlparts
= urlparse
.urlparse(url
);
4414 file_name
= os
.path
.basename(urlparts
.path
);
4415 file_dir
= os
.path
.dirname(urlparts
.path
);
4416 if(urlparts
.username
is not None):
4417 ftp_username
= urlparts
.username
;
4419 ftp_username
= "anonymous";
4420 if(urlparts
.password
is not None):
4421 ftp_password
= urlparts
.password
;
4422 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4423 ftp_password
= "anonymous";
4426 if(urlparts
.scheme
=="ftp"):
4428 elif(urlparts
.scheme
=="ftps"):
4432 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4434 ftp_port
= urlparts
.port
;
4435 if(urlparts
.port
is None):
4438 ftp
.connect(urlparts
.hostname
, ftp_port
);
4439 except socket
.gaierror
:
4440 log
.info("Error With URL "+httpurl
);
4442 except socket
.timeout
:
4443 log
.info("Error With URL "+httpurl
);
4445 ftp
.login(urlparts
.username
, urlparts
.password
);
4446 if(urlparts
.scheme
=="ftps"):
4448 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
4453 def upload_file_to_ftp_string(ftpstring
, url
):
4454 ftpfileo
= BytesIO(ftpstring
);
4455 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
4460 def download_file_from_sftp_file(url
):
4461 urlparts
= urlparse
.urlparse(url
);
4462 file_name
= os
.path
.basename(urlparts
.path
);
4463 file_dir
= os
.path
.dirname(urlparts
.path
);
4464 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4466 sftp_port
= urlparts
.port
;
4467 if(urlparts
.port
is None):
4470 sftp_port
= urlparts
.port
;
4471 if(urlparts
.username
is not None):
4472 sftp_username
= urlparts
.username
;
4474 sftp_username
= "anonymous";
4475 if(urlparts
.password
is not None):
4476 sftp_password
= urlparts
.password
;
4477 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4478 sftp_password
= "anonymous";
4481 if(urlparts
.scheme
!="sftp"):
4483 ssh
= paramiko
.SSHClient();
4484 ssh
.load_system_host_keys();
4485 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
4487 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
4488 except paramiko
.ssh_exception
.SSHException
:
4490 except socket
.gaierror
:
4491 log
.info("Error With URL "+httpurl
);
4493 except socket
.timeout
:
4494 log
.info("Error With URL "+httpurl
);
4496 sftp
= ssh
.open_sftp();
4497 sftpfile
= BytesIO();
4498 sftp
.getfo(urlparts
.path
, sftpfile
);
4501 sftpfile
.seek(0, 0);
4504 def download_file_from_sftp_file(url
):
4508 def download_file_from_sftp_string(url
):
4509 sftpfile
= download_file_from_sftp_file(url
);
4510 return sftpfile
.read();
4512 def download_file_from_ftp_string(url
):
4516 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4517 global geturls_download_sleep
, havezstd
, havebrotli
;
4519 sleep
= geturls_download_sleep
;
4522 urlparts
= urlparse
.urlparse(httpurl
);
4523 if(isinstance(httpheaders
, list)):
4524 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4525 httpheaders
= fix_header_names(httpheaders
);
4526 if(httpuseragent
is not None):
4527 if('User-Agent' in httpheaders
):
4528 httpheaders
['User-Agent'] = httpuseragent
;
4530 httpuseragent
.update({'User-Agent': httpuseragent
});
4531 if(httpreferer
is not None):
4532 if('Referer' in httpheaders
):
4533 httpheaders
['Referer'] = httpreferer
;
4535 httpuseragent
.update({'Referer': httpreferer
});
4536 if(isinstance(httpheaders
, dict)):
4537 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
4539 geturls_text
= download_file_from_sftp_file(httpurl
);
4540 if(not geturls_text
):
4542 downloadsize
= None;
4543 if(downloadsize
is not None):
4544 downloadsize
= int(downloadsize
);
4545 if downloadsize
is None: downloadsize
= 0;
4548 log
.info("Downloading URL "+httpurl
);
4549 with
BytesIO() as strbuf
:
4551 databytes
= geturls_text
.read(buffersize
);
4552 if not databytes
: break;
4553 datasize
= len(databytes
);
4554 fulldatasize
= datasize
+ fulldatasize
;
4557 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4558 downloaddiff
= fulldatasize
- prevdownsize
;
4559 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4560 prevdownsize
= fulldatasize
;
4561 strbuf
.write(databytes
);
4563 returnval_content
= strbuf
.read();
4564 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
4565 geturls_text
.close();
4568 if(not haveparamiko
):
4569 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4573 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4574 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4575 exec_time_start
= time
.time();
4576 myhash
= hashlib
.new("sha1");
4577 if(sys
.version
[0]=="2"):
4578 myhash
.update(httpurl
);
4579 myhash
.update(str(buffersize
));
4580 myhash
.update(str(exec_time_start
));
4581 if(sys
.version
[0]>="3"):
4582 myhash
.update(httpurl
.encode('utf-8'));
4583 myhash
.update(str(buffersize
).encode('utf-8'));
4584 myhash
.update(str(exec_time_start
).encode('utf-8'));
4585 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4587 sleep
= geturls_download_sleep
;
4590 pretmpfilename
= download_from_url_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4591 if(not pretmpfilename
):
4593 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4594 tmpfilename
= f
.name
;
4596 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4597 except AttributeError:
4599 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4604 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
4605 f
.write(pretmpfilename
['Content']);
4607 exec_time_end
= time
.time();
4608 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4609 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4612 if(not haveparamiko
):
4613 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4617 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4618 global geturls_download_sleep
, havezstd
, havebrotli
;
4620 sleep
= geturls_download_sleep
;
4623 if(not outfile
=="-"):
4624 outpath
= outpath
.rstrip(os
.path
.sep
);
4625 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4626 if(not os
.path
.exists(outpath
)):
4627 os
.makedirs(outpath
);
4628 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4630 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4632 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4633 if(not pretmpfilename
):
4635 tmpfilename
= pretmpfilename
['Filename'];
4636 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4638 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4639 exec_time_start
= time
.time();
4640 shutil
.move(tmpfilename
, filepath
);
4641 exec_time_end
= time
.time();
4642 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4643 if(os
.path
.exists(tmpfilename
)):
4644 os
.remove(tmpfilename
);
4645 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4647 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4648 tmpfilename
= pretmpfilename
['Filename'];
4649 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4652 exec_time_start
= time
.time();
4653 with
open(tmpfilename
, 'rb') as ft
:
4656 databytes
= ft
.read(buffersize
[1]);
4657 if not databytes
: break;
4658 datasize
= len(databytes
);
4659 fulldatasize
= datasize
+ fulldatasize
;
4662 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4663 downloaddiff
= fulldatasize
- prevdownsize
;
4664 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4665 prevdownsize
= fulldatasize
;
4668 fdata
= f
.getvalue();
4671 os
.remove(tmpfilename
);
4672 exec_time_end
= time
.time();
4673 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4674 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4677 if(not haveparamiko
):
4678 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4682 def upload_file_to_sftp_file(sftpfile
, url
):
4683 urlparts
= urlparse
.urlparse(url
);
4684 file_name
= os
.path
.basename(urlparts
.path
);
4685 file_dir
= os
.path
.dirname(urlparts
.path
);
4686 sftp_port
= urlparts
.port
;
4687 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4689 if(urlparts
.port
is None):
4692 sftp_port
= urlparts
.port
;
4693 if(urlparts
.username
is not None):
4694 sftp_username
= urlparts
.username
;
4696 sftp_username
= "anonymous";
4697 if(urlparts
.password
is not None):
4698 sftp_password
= urlparts
.password
;
4699 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4700 sftp_password
= "anonymous";
4703 if(urlparts
.scheme
!="sftp"):
4705 ssh
= paramiko
.SSHClient();
4706 ssh
.load_system_host_keys();
4707 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
4709 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
4710 except paramiko
.ssh_exception
.SSHException
:
4712 except socket
.gaierror
:
4713 log
.info("Error With URL "+httpurl
);
4715 except socket
.timeout
:
4716 log
.info("Error With URL "+httpurl
);
4718 sftp
= ssh
.open_sftp();
4719 sftp
.putfo(sftpfile
, urlparts
.path
);
4722 sftpfile
.seek(0, 0);
4725 def upload_file_to_sftp_file(sftpfile
, url
):
4729 def upload_file_to_sftp_string(sftpstring
, url
):
4730 sftpfileo
= BytesIO(sftpstring
);
4731 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
4735 def upload_file_to_sftp_string(url
):
4740 def download_file_from_pysftp_file(url
):
4741 urlparts
= urlparse
.urlparse(url
);
4742 file_name
= os
.path
.basename(urlparts
.path
);
4743 file_dir
= os
.path
.dirname(urlparts
.path
);
4744 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4746 sftp_port
= urlparts
.port
;
4747 if(urlparts
.port
is None):
4750 sftp_port
= urlparts
.port
;
4751 if(urlparts
.username
is not None):
4752 sftp_username
= urlparts
.username
;
4754 sftp_username
= "anonymous";
4755 if(urlparts
.password
is not None):
4756 sftp_password
= urlparts
.password
;
4757 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4758 sftp_password
= "anonymous";
4761 if(urlparts
.scheme
!="sftp"):
4764 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
4765 except paramiko
.ssh_exception
.SSHException
:
4767 except socket
.gaierror
:
4768 log
.info("Error With URL "+httpurl
);
4770 except socket
.timeout
:
4771 log
.info("Error With URL "+httpurl
);
4773 sftp
= ssh
.open_sftp();
4774 sftpfile
= BytesIO();
4775 sftp
.getfo(urlparts
.path
, sftpfile
);
4778 sftpfile
.seek(0, 0);
4781 def download_file_from_pysftp_file(url
):
4785 def download_file_from_pysftp_string(url
):
4786 sftpfile
= download_file_from_pysftp_file(url
);
4787 return sftpfile
.read();
4789 def download_file_from_ftp_string(url
):
4793 def download_from_url_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4794 global geturls_download_sleep
, havezstd
, havebrotli
;
4796 sleep
= geturls_download_sleep
;
4799 urlparts
= urlparse
.urlparse(httpurl
);
4800 if(isinstance(httpheaders
, list)):
4801 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4802 httpheaders
= fix_header_names(httpheaders
);
4803 if(isinstance(httpheaders
, dict)):
4804 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
4806 geturls_text
= download_file_from_pysftp_file(httpurl
);
4807 if(not geturls_text
):
4809 downloadsize
= None;
4810 if(downloadsize
is not None):
4811 downloadsize
= int(downloadsize
);
4812 if downloadsize
is None: downloadsize
= 0;
4815 log
.info("Downloading URL "+httpurl
);
4816 with
BytesIO() as strbuf
:
4818 databytes
= geturls_text
.read(buffersize
);
4819 if not databytes
: break;
4820 datasize
= len(databytes
);
4821 fulldatasize
= datasize
+ fulldatasize
;
4824 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4825 downloaddiff
= fulldatasize
- prevdownsize
;
4826 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4827 prevdownsize
= fulldatasize
;
4828 strbuf
.write(databytes
);
4830 returnval_content
= strbuf
.read();
4831 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
4832 geturls_text
.close();
4836 def download_from_url_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4840 def download_from_url_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4841 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4842 exec_time_start
= time
.time();
4843 myhash
= hashlib
.new("sha1");
4844 if(sys
.version
[0]=="2"):
4845 myhash
.update(httpurl
);
4846 myhash
.update(str(buffersize
));
4847 myhash
.update(str(exec_time_start
));
4848 if(sys
.version
[0]>="3"):
4849 myhash
.update(httpurl
.encode('utf-8'));
4850 myhash
.update(str(buffersize
).encode('utf-8'));
4851 myhash
.update(str(exec_time_start
).encode('utf-8'));
4852 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4854 sleep
= geturls_download_sleep
;
4857 pretmpfilename
= download_from_url_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4858 if(not pretmpfilename
):
4860 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4861 tmpfilename
= f
.name
;
4863 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4864 except AttributeError:
4866 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4871 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason')};
4872 f
.write(pretmpfilename
['Content']);
4874 exec_time_end
= time
.time();
4875 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4876 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4880 def download_from_url_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4884 def download_from_url_to_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4885 global geturls_download_sleep
, havezstd
, havebrotli
;
4887 sleep
= geturls_download_sleep
;
4890 if(not outfile
=="-"):
4891 outpath
= outpath
.rstrip(os
.path
.sep
);
4892 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4893 if(not os
.path
.exists(outpath
)):
4894 os
.makedirs(outpath
);
4895 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4897 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4899 pretmpfilename
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4900 if(not pretmpfilename
):
4902 tmpfilename
= pretmpfilename
['Filename'];
4903 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4905 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4906 exec_time_start
= time
.time();
4907 shutil
.move(tmpfilename
, filepath
);
4908 exec_time_end
= time
.time();
4909 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4910 if(os
.path
.exists(tmpfilename
)):
4911 os
.remove(tmpfilename
);
4912 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4914 pretmpfilename
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4915 tmpfilename
= pretmpfilename
['Filename'];
4916 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4919 exec_time_start
= time
.time();
4920 with
open(tmpfilename
, 'rb') as ft
:
4923 databytes
= ft
.read(buffersize
[1]);
4924 if not databytes
: break;
4925 datasize
= len(databytes
);
4926 fulldatasize
= datasize
+ fulldatasize
;
4929 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4930 downloaddiff
= fulldatasize
- prevdownsize
;
4931 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4932 prevdownsize
= fulldatasize
;
4935 fdata
= f
.getvalue();
4938 os
.remove(tmpfilename
);
4939 exec_time_end
= time
.time();
4940 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4941 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': None, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code'], 'Reason': pretmpfilename
['Reason']};
4945 def download_from_url_to_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4949 def upload_file_to_pysftp_file(sftpfile
, url
):
4950 urlparts
= urlparse
.urlparse(url
);
4951 file_name
= os
.path
.basename(urlparts
.path
);
4952 file_dir
= os
.path
.dirname(urlparts
.path
);
4953 sftp_port
= urlparts
.port
;
4954 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4956 if(urlparts
.port
is None):
4959 sftp_port
= urlparts
.port
;
4960 if(urlparts
.username
is not None):
4961 sftp_username
= urlparts
.username
;
4963 sftp_username
= "anonymous";
4964 if(urlparts
.password
is not None):
4965 sftp_password
= urlparts
.password
;
4966 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4967 sftp_password
= "anonymous";
4970 if(urlparts
.scheme
!="sftp"):
4973 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
4974 except paramiko
.ssh_exception
.SSHException
:
4976 except socket
.gaierror
:
4977 log
.info("Error With URL "+httpurl
);
4979 except socket
.timeout
:
4980 log
.info("Error With URL "+httpurl
);
4982 sftp
= ssh
.open_sftp();
4983 sftp
.putfo(sftpfile
, urlparts
.path
);
4986 sftpfile
.seek(0, 0);
4989 def upload_file_to_pysftp_file(sftpfile
, url
):
4993 def upload_file_to_pysftp_string(sftpstring
, url
):
4994 sftpfileo
= BytesIO(sftpstring
);
4995 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
4999 def upload_file_to_pysftp_string(url
):