4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
15 $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
18 from __future__
import division
, absolute_import
, print_function
;
19 import re
, os
, sys
, hashlib
, shutil
, platform
, tempfile
, urllib
, zlib
, bz2
, time
, argparse
, cgi
, subprocess
, socket
, email
.utils
, datetime
, time
;
20 import logging
as log
;
21 from ftplib
import FTP
, FTP_TLS
;
22 from base64
import b64encode
;
29 havemechanize
= False;
34 havemechanize
= False;
61 from httplib2
import HTTPConnectionWithTimeout
, HTTPSConnectionWithTimeout
;
101 if(sys
.version
[0]=="2"):
103 from io
import StringIO
, BytesIO
;
106 from cStringIO
import StringIO
;
107 from cStringIO
import StringIO
as BytesIO
;
109 from StringIO
import StringIO
;
110 from StringIO
import StringIO
as BytesIO
;
111 # From http://python-future.org/compatible_idioms.html
112 from urlparse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
;
113 from urllib
import urlencode
;
114 from urllib
import urlopen
as urlopenalt
;
115 from urllib2
import urlopen
, Request
, install_opener
, HTTPError
, URLError
, build_opener
, HTTPCookieProcessor
;
116 import urlparse
, cookielib
;
117 from httplib
import HTTPConnection
, HTTPSConnection
;
118 if(sys
.version
[0]>="3"):
119 from io
import StringIO
, BytesIO
;
120 # From http://python-future.org/compatible_idioms.html
121 from urllib
.parse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
, urlencode
;
122 from urllib
.request
import urlopen
, Request
, install_opener
, build_opener
, HTTPCookieProcessor
;
123 from urllib
.error
import HTTPError
, URLError
;
124 import urllib
.parse
as urlparse
;
125 import http
.cookiejar
as cookielib
;
126 from http
.client
import HTTPConnection
, HTTPSConnection
;
128 __program_name__
= "PyWWW-Get";
129 __program_alt_name__
= "PyWWWGet";
130 __program_small_name__
= "wwwget";
131 __project__
= __program_name__
;
132 __project_url__
= "https://github.com/GameMaker2k/PyWWW-Get";
133 __version_info__
= (2, 0, 2, "RC 1", 1);
134 __version_date_info__
= (2023, 10, 5, "RC 1", 1);
135 __version_date__
= str(__version_date_info__
[0])+"."+str(__version_date_info__
[1]).zfill(2)+"."+str(__version_date_info__
[2]).zfill(2);
136 __revision__
= __version_info__
[3];
137 __revision_id__
= "$Id$";
138 if(__version_info__
[4] is not None):
139 __version_date_plusrc__
= __version_date__
+"-"+str(__version_date_info__
[4]);
140 if(__version_info__
[4] is None):
141 __version_date_plusrc__
= __version_date__
;
142 if(__version_info__
[3] is not None):
143 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2])+" "+str(__version_info__
[3]);
144 if(__version_info__
[3] is None):
145 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2]);
147 tmpfileprefix
= "py"+str(sys
.version_info
[0])+__program_small_name__
+str(__version_info__
[0])+"-";
149 pytempdir
= tempfile
.gettempdir();
151 PyBitness
= platform
.architecture();
152 if(PyBitness
=="32bit" or PyBitness
=="32"):
154 elif(PyBitness
=="64bit" or PyBitness
=="64"):
159 compression_supported_list
= ['identity', 'gzip', 'deflate', 'bzip2'];
161 compression_supported_list
.append('br');
163 compression_supported_list
.append('zstd');
165 compression_supported_list
.append('lzma');
166 compression_supported_list
.append('xz');
167 compression_supported
= ', '.join(compression_supported_list
);
169 geturls_cj
= cookielib
.CookieJar();
170 windowsNT4_ua_string
= "Windows NT 4.0";
171 windowsNT4_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
172 windows2k_ua_string
= "Windows NT 5.0";
173 windows2k_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
174 windowsXP_ua_string
= "Windows NT 5.1";
175 windowsXP_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
176 windowsXP64_ua_string
= "Windows NT 5.2; Win64; x64";
177 windowsXP64_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
178 windows7_ua_string
= "Windows NT 6.1; Win64; x64";
179 windows7_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
180 windows8_ua_string
= "Windows NT 6.2; Win64; x64";
181 windows8_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
182 windows81_ua_string
= "Windows NT 6.3; Win64; x64";
183 windows81_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
184 windows10_ua_string
= "Windows NT 10.0; Win64; x64";
185 windows10_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
186 windows11_ua_string
= "Windows NT 11.0; Win64; x64";
187 windows11_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
188 geturls_ua_firefox_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:109.0) Gecko/20100101 Firefox/117.0";
189 geturls_ua_seamonkey_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
190 geturls_ua_chrome_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
191 geturls_ua_chromium_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
192 geturls_ua_palemoon_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
193 geturls_ua_opera_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
194 geturls_ua_vivaldi_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
195 geturls_ua_internet_explorer_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; Trident/7.0; rv:11.0) like Gecko";
196 geturls_ua_microsoft_edge_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
197 geturls_ua_pywwwget_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
198 if(platform
.python_implementation()!=""):
199 py_implementation
= platform
.python_implementation();
200 if(platform
.python_implementation()==""):
201 py_implementation
= "Python";
202 geturls_ua_pywwwget_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
203 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
204 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
205 geturls_ua
= geturls_ua_firefox_windows7
;
206 geturls_headers_firefox_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
207 geturls_headers_seamonkey_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
208 geturls_headers_chrome_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
209 geturls_headers_chrome_windows7
.update(windows7_ua_addon
);
210 geturls_headers_chromium_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
211 geturls_headers_chromium_windows7
.update(windows7_ua_addon
);
212 geturls_headers_palemoon_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers_opera_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
214 geturls_headers_opera_windows7
.update(windows7_ua_addon
);
215 geturls_headers_vivaldi_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
216 geturls_headers_vivaldi_windows7
.update(windows7_ua_addon
);
217 geturls_headers_internet_explorer_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218 geturls_headers_microsoft_edge_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
219 geturls_headers_microsoft_edge_windows7
.update(windows7_ua_addon
);
220 geturls_headers_pywwwget_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
221 geturls_headers_pywwwget_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
222 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
223 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
224 geturls_headers
= geturls_headers_firefox_windows7
;
225 geturls_download_sleep
= 0;
227 def verbose_printout(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
228 if(outtype
=="print" and dbgenable
):
231 elif(outtype
=="log" and dbgenable
):
232 logging
.info(dbgtxt
);
234 elif(outtype
=="warning" and dbgenable
):
235 logging
.warning(dbgtxt
);
237 elif(outtype
=="error" and dbgenable
):
238 logging
.error(dbgtxt
);
240 elif(outtype
=="critical" and dbgenable
):
241 logging
.critical(dbgtxt
);
243 elif(outtype
=="exception" and dbgenable
):
244 logging
.exception(dbgtxt
);
246 elif(outtype
=="logalt" and dbgenable
):
247 logging
.log(dgblevel
, dbgtxt
);
249 elif(outtype
=="debug" and dbgenable
):
250 logging
.debug(dbgtxt
);
258 def verbose_printout_return(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
259 dbgout
= verbose_printout(dbgtxt
, outtype
, dbgenable
, dgblevel
);
264 def add_url_param(url
, **params
):
266 parts
= list(urlparse
.urlsplit(url
));
267 d
= dict(cgi
.parse_qsl(parts
[n
])); # use cgi.parse_qs for list values
269 parts
[n
]=urlencode(d
);
270 return urlparse
.urlunsplit(parts
);
272 os
.environ
["PATH"] = os
.environ
["PATH"] + os
.pathsep
+ os
.path
.dirname(os
.path
.realpath(__file__
)) + os
.pathsep
+ os
.getcwd();
273 def which_exec(execfile):
274 for path
in os
.environ
["PATH"].split(":"):
275 if os
.path
.exists(path
+ "/" + execfile):
276 return path
+ "/" + execfile;
278 def listize(varlist
):
286 newlistreg
.update({ilx
: varlist
[il
]});
287 newlistrev
.update({varlist
[il
]: ilx
});
290 newlistfull
= {1: newlistreg
, 2: newlistrev
, 'reg': newlistreg
, 'rev': newlistrev
};
293 def twolistize(varlist
):
303 newlistnamereg
.update({ilx
: varlist
[il
][0].strip()});
304 newlistnamerev
.update({varlist
[il
][0].strip(): ilx
});
305 newlistdescreg
.update({ilx
: varlist
[il
][1].strip()});
306 newlistdescrev
.update({varlist
[il
][1].strip(): ilx
});
309 newlistnametmp
= {1: newlistnamereg
, 2: newlistnamerev
, 'reg': newlistnamereg
, 'rev': newlistnamerev
};
310 newlistdesctmp
= {1: newlistdescreg
, 2: newlistdescrev
, 'reg': newlistdescreg
, 'rev': newlistdescrev
};
311 newlistfull
= {1: newlistnametmp
, 2: newlistdesctmp
, 'name': newlistnametmp
, 'desc': newlistdesctmp
}
314 def arglistize(proexec
, *varlist
):
318 newarglist
= [proexec
];
320 if varlist
[il
][0] is not None:
321 newarglist
.append(varlist
[il
][0]);
322 if varlist
[il
][1] is not None:
323 newarglist
.append(varlist
[il
][1]);
327 def fix_header_names(header_dict
):
328 if(sys
.version
[0]=="2"):
329 header_dict
= {k
.title(): v
for k
, v
in header_dict
.iteritems()};
330 if(sys
.version
[0]>="3"):
331 header_dict
= {k
.title(): v
for k
, v
in header_dict
.items()};
334 # hms_string by ArcGIS Python Recipes
335 # https://arcpy.wordpress.com/2012/04/20/146/
336 def hms_string(sec_elapsed
):
337 h
= int(sec_elapsed
/ (60 * 60));
338 m
= int((sec_elapsed
% (60 * 60)) / 60);
339 s
= sec_elapsed
% 60.0;
340 return "{}:{:>02}:{:>05.2f}".format(h
, m
, s
);
342 # get_readable_size by Lipis
343 # http://stackoverflow.com/posts/14998888/revisions
344 def get_readable_size(bytes
, precision
=1, unit
="IEC"):
346 if(unit
!="IEC" and unit
!="SI"):
349 units
= [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
350 unitswos
= ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
353 units
= [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
354 unitswos
= ["B","kB","MB","GB","TB","PB","EB","ZB"];
359 if abs(bytes
) < unitsize
:
360 strformat
= "%3."+str(precision
)+"f%s";
361 pre_return_val
= (strformat
% (bytes
, unit
));
362 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
363 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
364 alt_return_val
= pre_return_val
.split();
365 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
368 strformat
= "%."+str(precision
)+"f%s";
369 pre_return_val
= (strformat
% (bytes
, "YiB"));
370 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
371 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
372 alt_return_val
= pre_return_val
.split();
373 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
376 def get_readable_size_from_file(infile
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
378 usehashtypes
= usehashtypes
.lower();
379 getfilesize
= os
.path
.getsize(infile
);
380 return_val
= get_readable_size(getfilesize
, precision
, unit
);
382 hashtypelist
= usehashtypes
.split(",");
383 openfile
= open(infile
, "rb");
384 filecontents
= openfile
.read();
387 listnumend
= len(hashtypelist
);
388 while(listnumcount
< listnumend
):
389 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
390 hashtypelistup
= hashtypelistlow
.upper();
391 filehash
= hashlib
.new(hashtypelistup
);
392 filehash
.update(filecontents
);
393 filegethash
= filehash
.hexdigest();
394 return_val
.update({hashtypelistup
: filegethash
});
398 def get_readable_size_from_string(instring
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
400 usehashtypes
= usehashtypes
.lower();
401 getfilesize
= len(instring
);
402 return_val
= get_readable_size(getfilesize
, precision
, unit
);
404 hashtypelist
= usehashtypes
.split(",");
406 listnumend
= len(hashtypelist
);
407 while(listnumcount
< listnumend
):
408 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
409 hashtypelistup
= hashtypelistlow
.upper();
410 filehash
= hashlib
.new(hashtypelistup
);
411 if(sys
.version
[0]=="2"):
412 filehash
.update(instring
);
413 if(sys
.version
[0]>="3"):
414 filehash
.update(instring
.encode('utf-8'));
415 filegethash
= filehash
.hexdigest();
416 return_val
.update({hashtypelistup
: filegethash
});
420 def http_status_to_reason(code
):
423 101: 'Switching Protocols',
428 203: 'Non-Authoritative Information',
430 205: 'Reset Content',
431 206: 'Partial Content',
433 208: 'Already Reported',
435 300: 'Multiple Choices',
436 301: 'Moved Permanently',
441 307: 'Temporary Redirect',
442 308: 'Permanent Redirect',
445 402: 'Payment Required',
448 405: 'Method Not Allowed',
449 406: 'Not Acceptable',
450 407: 'Proxy Authentication Required',
451 408: 'Request Timeout',
454 411: 'Length Required',
455 412: 'Precondition Failed',
456 413: 'Payload Too Large',
458 415: 'Unsupported Media Type',
459 416: 'Range Not Satisfiable',
460 417: 'Expectation Failed',
461 421: 'Misdirected Request',
462 422: 'Unprocessable Entity',
464 424: 'Failed Dependency',
465 426: 'Upgrade Required',
466 428: 'Precondition Required',
467 429: 'Too Many Requests',
468 431: 'Request Header Fields Too Large',
469 451: 'Unavailable For Legal Reasons',
470 500: 'Internal Server Error',
471 501: 'Not Implemented',
473 503: 'Service Unavailable',
474 504: 'Gateway Timeout',
475 505: 'HTTP Version Not Supported',
476 506: 'Variant Also Negotiates',
477 507: 'Insufficient Storage',
478 508: 'Loop Detected',
480 511: 'Network Authentication Required'
482 return reasons
.get(code
, 'Unknown Status Code');
484 def ftp_status_to_reason(code
):
486 110: 'Restart marker reply',
487 120: 'Service ready in nnn minutes',
488 125: 'Data connection already open; transfer starting',
489 150: 'File status okay; about to open data connection',
491 202: 'Command not implemented, superfluous at this site',
492 211: 'System status, or system help reply',
493 212: 'Directory status',
496 215: 'NAME system type',
497 220: 'Service ready for new user',
498 221: 'Service closing control connection',
499 225: 'Data connection open; no transfer in progress',
500 226: 'Closing data connection',
501 227: 'Entering Passive Mode',
502 230: 'User logged in, proceed',
503 250: 'Requested file action okay, completed',
504 257: '"PATHNAME" created',
505 331: 'User name okay, need password',
506 332: 'Need account for login',
507 350: 'Requested file action pending further information',
508 421: 'Service not available, closing control connection',
509 425: 'Can\'t open data connection',
510 426: 'Connection closed; transfer aborted',
511 450: 'Requested file action not taken',
512 451: 'Requested action aborted. Local error in processing',
513 452: 'Requested action not taken. Insufficient storage space in system',
514 500: 'Syntax error, command unrecognized',
515 501: 'Syntax error in parameters or arguments',
516 502: 'Command not implemented',
517 503: 'Bad sequence of commands',
518 504: 'Command not implemented for that parameter',
519 530: 'Not logged in',
520 532: 'Need account for storing files',
521 550: 'Requested action not taken. File unavailable',
522 551: 'Requested action aborted. Page type unknown',
523 552: 'Requested file action aborted. Exceeded storage allocation',
524 553: 'Requested action not taken. File name not allowed'
526 return reasons
.get(code
, 'Unknown Status Code');
528 def sftp_status_to_reason(code
):
532 2: 'SSH_FX_NO_SUCH_FILE',
533 3: 'SSH_FX_PERMISSION_DENIED',
535 5: 'SSH_FX_BAD_MESSAGE',
536 6: 'SSH_FX_NO_CONNECTION',
537 7: 'SSH_FX_CONNECTION_LOST',
538 8: 'SSH_FX_OP_UNSUPPORTED'
540 return reasons
.get(code
, 'Unknown Status Code');
542 def make_http_headers_from_dict_to_list(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
543 if isinstance(headers
, dict):
545 if(sys
.version
[0]=="2"):
546 for headkey
, headvalue
in headers
.iteritems():
547 returnval
.append((headkey
, headvalue
));
548 if(sys
.version
[0]>="3"):
549 for headkey
, headvalue
in headers
.items():
550 returnval
.append((headkey
, headvalue
));
551 elif isinstance(headers
, list):
557 def make_http_headers_from_dict_to_pycurl(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
558 if isinstance(headers
, dict):
560 if(sys
.version
[0]=="2"):
561 for headkey
, headvalue
in headers
.iteritems():
562 returnval
.append(headkey
+": "+headvalue
);
563 if(sys
.version
[0]>="3"):
564 for headkey
, headvalue
in headers
.items():
565 returnval
.append(headkey
+": "+headvalue
);
566 elif isinstance(headers
, list):
572 def make_http_headers_from_pycurl_to_dict(headers
):
574 headers
= headers
.strip().split('\r\n');
575 for header
in headers
:
576 parts
= header
.split(': ', 1)
579 header_dict
[key
.title()] = value
;
582 def make_http_headers_from_list_to_dict(headers
=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua
), ("Accept-Encoding", compression_supported
), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
583 if isinstance(headers
, list):
588 returnval
.update({headers
[mli
][0]: headers
[mli
][1]});
590 elif isinstance(headers
, dict):
596 def get_httplib_support(checkvalue
=None):
597 global haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
599 returnval
.append("ftp");
600 returnval
.append("httplib");
602 returnval
.append("httplib2");
603 returnval
.append("urllib");
605 returnval
.append("urllib3");
606 returnval
.append("request3");
607 returnval
.append("request");
609 returnval
.append("requests");
611 returnval
.append("aiohttp");
613 returnval
.append("httpx");
614 returnval
.append("httpx2");
616 returnval
.append("mechanize");
618 returnval
.append("pycurl");
619 if(hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
620 returnval
.append("pycurl2");
621 if(hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
622 returnval
.append("pycurl3");
624 returnval
.append("sftp");
626 returnval
.append("pysftp");
627 if(not checkvalue
is None):
628 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
629 checkvalue
= "urllib";
630 if(checkvalue
=="httplib1"):
631 checkvalue
= "httplib";
632 if(checkvalue
in returnval
):
638 def check_httplib_support(checkvalue
="urllib"):
639 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
640 checkvalue
= "urllib";
641 if(checkvalue
=="httplib1"):
642 checkvalue
= "httplib";
643 returnval
= get_httplib_support(checkvalue
);
646 def get_httplib_support_list():
647 returnval
= get_httplib_support(None);
650 def download_from_url(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", buffersize
=524288, sleep
=-1, timeout
=10):
651 global geturls_download_sleep
, havezstd
, havebrotli
, haveaiohttp
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
653 sleep
= geturls_download_sleep
;
656 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
657 httplibuse
= "urllib";
658 if(httplibuse
=="httplib1"):
659 httplibuse
= "httplib";
660 if(not haverequests
and httplibuse
=="requests"):
661 httplibuse
= "urllib";
662 if(not haveaiohttp
and httplibuse
=="aiohttp"):
663 httplibuse
= "urllib";
664 if(not havehttpx
and httplibuse
=="httpx"):
665 httplibuse
= "urllib";
666 if(not havehttpx
and httplibuse
=="httpx2"):
667 httplibuse
= "urllib";
668 if(not havehttpcore
and httplibuse
=="httpcore"):
669 httplibuse
= "urllib";
670 if(not havehttpcore
and httplibuse
=="httpcore2"):
671 httplibuse
= "urllib";
672 if(not havemechanize
and httplibuse
=="mechanize"):
673 httplibuse
= "urllib";
674 if(not havepycurl
and httplibuse
=="pycurl"):
675 httplibuse
= "urllib";
676 if(not havepycurl
and httplibuse
=="pycurl2"):
677 httplibuse
= "urllib";
678 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
679 httplibuse
= "pycurl";
680 if(not havepycurl
and httplibuse
=="pycurl3"):
681 httplibuse
= "urllib";
682 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
683 httplibuse
= "pycurl2";
684 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
685 httplibuse
= "pycurl";
686 if(not havehttplib2
and httplibuse
=="httplib2"):
687 httplibuse
= "httplib";
688 if(not haveparamiko
and httplibuse
=="sftp"):
690 if(not havepysftp
and httplibuse
=="pysftp"):
692 urlparts
= urlparse
.urlparse(httpurl
);
693 if(isinstance(httpheaders
, list)):
694 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
695 httpheaders
= fix_header_names(httpheaders
);
696 if(httpuseragent
is not None):
697 if('User-Agent' in httpheaders
):
698 httpheaders
['User-Agent'] = httpuseragent
;
700 httpuseragent
.update({'User-Agent': httpuseragent
});
701 if(httpreferer
is not None):
702 if('Referer' in httpheaders
):
703 httpheaders
['Referer'] = httpreferer
;
705 httpuseragent
.update({'Referer': httpreferer
});
706 if(urlparts
.username
is not None or urlparts
.password
is not None):
707 if(sys
.version
[0]=="2"):
708 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
709 if(sys
.version
[0]>="3"):
710 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
711 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
712 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
713 if(httplibuse
=="urllib" or httplibuse
=="mechanize"):
714 if(isinstance(httpheaders
, dict)):
715 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
716 if(httplibuse
=="pycurl" or httplibuse
=="pycurl2" or httplibuse
=="pycurl3"):
717 if(isinstance(httpheaders
, dict)):
718 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
719 geturls_opener
.addheaders
= httpheaders
;
721 if(postdata
is not None and not isinstance(postdata
, dict)):
722 postdata
= urlencode(postdata
);
723 if(httplibuse
=="urllib" or httplibuse
=="request"):
724 geturls_request
= Request(httpurl
);
726 if(httpmethod
=="GET"):
727 geturls_text
= geturls_opener
.open(geturls_request
);
728 elif(httpmethod
=="POST"):
729 geturls_text
= geturls_opener
.open(geturls_request
, data
=postdata
);
731 geturls_text
= geturls_opener
.open(geturls_request
);
732 except HTTPError
as geturls_text_error
:
733 geturls_text
= geturls_text_error
;
734 log
.info("Error With URL "+httpurl
);
736 log
.info("Error With URL "+httpurl
);
738 except socket
.timeout
:
739 log
.info("Error With URL "+httpurl
);
741 httpcodeout
= geturls_text
.getcode();
743 httpcodereason
= geturls_text
.reason
;
744 except AttributeError:
745 httpcodereason
= http_status_to_reason(geturls_text
.getcode());
747 httpversionout
= geturls_text
.version
;
748 except AttributeError:
749 httpversionout
= "1.1";
750 httpmethodout
= geturls_request
.get_method();
751 httpurlout
= geturls_text
.geturl();
752 httpheaderout
= geturls_text
.info();
753 httpheadersentout
= httpheaders
;
754 elif(httplibuse
=="httplib"):
755 if(urlparts
[0]=="http"):
756 httpconn
= HTTPConnection(urlparts
[1], timeout
=timeout
);
757 elif(urlparts
[0]=="https"):
758 httpconn
= HTTPSConnection(urlparts
[1], timeout
=timeout
);
761 if(postdata
is not None and not isinstance(postdata
, dict)):
762 postdata
= urlencode(postdata
);
764 if(httpmethod
=="GET"):
765 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
766 elif(httpmethod
=="POST"):
767 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
769 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
770 except socket
.timeout
:
771 log
.info("Error With URL "+httpurl
);
773 except socket
.gaierror
:
774 log
.info("Error With URL "+httpurl
);
776 except BlockingIOError
:
777 log
.info("Error With URL "+httpurl
);
779 geturls_text
= httpconn
.getresponse();
780 httpcodeout
= geturls_text
.status
;
781 httpcodereason
= geturls_text
.reason
;
782 if(geturls_text
.version
=="10"):
783 httpversionout
= "1.0";
785 httpversionout
= "1.1";
786 httpmethodout
= geturls_text
._method
;
787 httpurlout
= httpurl
;
788 httpheaderout
= geturls_text
.getheaders();
789 httpheadersentout
= httpheaders
;
790 elif(httplibuse
=="httplib2"):
791 if(urlparts
[0]=="http"):
792 httpconn
= HTTPConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
793 elif(urlparts
[0]=="https"):
794 httpconn
= HTTPSConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
797 if(postdata
is not None and not isinstance(postdata
, dict)):
798 postdata
= urlencode(postdata
);
800 if(httpmethod
=="GET"):
801 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
802 elif(httpmethod
=="POST"):
803 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
805 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
806 except socket
.timeout
:
807 log
.info("Error With URL "+httpurl
);
809 except socket
.gaierror
:
810 log
.info("Error With URL "+httpurl
);
812 except BlockingIOError
:
813 log
.info("Error With URL "+httpurl
);
815 geturls_text
= httpconn
.getresponse();
816 httpcodeout
= geturls_text
.status
;
817 httpcodereason
= geturls_text
.reason
;
818 if(geturls_text
.version
=="10"):
819 httpversionout
= "1.0";
821 httpversionout
= "1.1";
822 httpmethodout
= httpmethod
;
823 httpurlout
= httpurl
;
824 httpheaderout
= geturls_text
.getheaders();
825 httpheadersentout
= httpheaders
;
826 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
827 timeout
= urllib3
.util
.Timeout(connect
=timeout
, read
=timeout
);
828 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
, timeout
=timeout
);
830 if(httpmethod
=="GET"):
831 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
832 elif(httpmethod
=="POST"):
833 geturls_text
= urllib_pool
.request("POST", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
835 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
836 except urllib3
.exceptions
.ConnectTimeoutError
:
837 log
.info("Error With URL "+httpurl
);
839 except urllib3
.exceptions
.ConnectError
:
840 log
.info("Error With URL "+httpurl
);
842 except urllib3
.exceptions
.MaxRetryError
:
843 log
.info("Error With URL "+httpurl
);
845 except socket
.timeout
:
846 log
.info("Error With URL "+httpurl
);
849 log
.info("Error With URL "+httpurl
);
851 httpcodeout
= geturls_text
.status
;
852 httpcodereason
= geturls_text
.reason
;
853 if(geturls_text
.version
=="10"):
854 httpversionout
= "1.0";
856 httpversionout
= "1.1";
857 httpmethodout
= httpmethod
;
858 httpurlout
= geturls_text
.geturl();
859 httpheaderout
= geturls_text
.info();
860 httpheadersentout
= httpheaders
;
861 elif(httplibuse
=="requests"):
863 reqsession
= requests
.Session();
864 if(httpmethod
=="GET"):
865 geturls_text
= reqsession
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
866 elif(httpmethod
=="POST"):
867 geturls_text
= reqsession
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
869 geturls_text
= reqsession
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
870 except requests
.exceptions
.ConnectTimeout
:
871 log
.info("Error With URL "+httpurl
);
873 except requests
.exceptions
.ConnectError
:
874 log
.info("Error With URL "+httpurl
);
876 except socket
.timeout
:
877 log
.info("Error With URL "+httpurl
);
879 httpcodeout
= geturls_text
.status_code
;
880 httpcodereason
= geturls_text
.reason
;
881 if(geturls_text
.raw
.version
=="10"):
882 httpversionout
= "1.0";
884 httpversionout
= "1.1";
885 httpmethodout
= httpmethod
;
886 httpurlout
= geturls_text
.url
;
887 httpheaderout
= geturls_text
.headers
;
888 httpheadersentout
= geturls_text
.request
.headers
;
889 elif(httplibuse
=="aiohttp"):
891 reqsession
= aiohttp
.ClientSession(cookie_jar
=httpcookie
, headers
=httpheaders
, timeout
=timeout
, read_timeout
=timeout
, conn_timeout
=timeout
, read_bufsize
=buffersize
);
892 if(httpmethod
=="GET"):
893 geturls_text
= reqsession
.get(httpurl
);
894 elif(httpmethod
=="POST"):
895 geturls_text
= reqsession
.post(httpurl
, data
=postdata
);
897 geturls_text
= reqsession
.get(httpurl
);
898 except aiohttp
.exceptions
.ConnectTimeout
:
899 log
.info("Error With URL "+httpurl
);
901 except aiohttp
.exceptions
.ConnectError
:
902 log
.info("Error With URL "+httpurl
);
904 except socket
.timeout
:
905 log
.info("Error With URL "+httpurl
);
907 httpcodeout
= geturls_text
.status
;
908 httpcodereason
= geturls_text
.reason
;
909 httpversionout
= geturls_text
.version
;
910 httpmethodout
= geturls_text
.method
;
911 httpurlout
= geturls_text
.url
;
912 httpheaderout
= geturls_text
.headers
;
913 httpheadersentout
= geturls_text
.request_info
.headers
;
914 elif(httplibuse
=="httpx"):
916 if(httpmethod
=="GET"):
917 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
918 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
919 elif(httpmethod
=="POST"):
920 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
921 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
923 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
924 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
925 except httpx
.ConnectTimeout
:
926 log
.info("Error With URL "+httpurl
);
928 except httpx
.ConnectError
:
929 log
.info("Error With URL "+httpurl
);
931 except socket
.timeout
:
932 log
.info("Error With URL "+httpurl
);
934 httpcodeout
= geturls_text
.status_code
;
936 httpcodereason
= geturls_text
.reason_phrase
;
938 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
939 httpversionout
= geturls_text
.http_version
;
940 httpmethodout
= httpmethod
;
941 httpurlout
= str(geturls_text
.url
);
942 httpheaderout
= geturls_text
.headers
;
943 httpheadersentout
= geturls_text
.request
.headers
;
944 elif(httplibuse
=="httpx2"):
946 if(httpmethod
=="GET"):
947 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
948 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
949 elif(httpmethod
=="POST"):
950 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
951 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
953 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
954 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
955 except httpx
.ConnectTimeout
:
956 log
.info("Error With URL "+httpurl
);
958 except httpx
.ConnectError
:
959 log
.info("Error With URL "+httpurl
);
961 except socket
.timeout
:
962 log
.info("Error With URL "+httpurl
);
964 httpcodeout
= geturls_text
.status_code
;
966 httpcodereason
= geturls_text
.reason_phrase
;
968 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
969 httpversionout
= geturls_text
.http_version
;
970 httpmethodout
= httpmethod
;
971 httpurlout
= str(geturls_text
.url
);
972 httpheaderout
= geturls_text
.headers
;
973 httpheadersentout
= geturls_text
.request
.headers
;
974 elif(httplibuse
=="httpcore"):
976 if(httpmethod
=="GET"):
977 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
978 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
979 elif(httpmethod
=="POST"):
980 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
981 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
983 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
984 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
985 except httpcore
.ConnectTimeout
:
986 log
.info("Error With URL "+httpurl
);
988 except httpcore
.ConnectError
:
989 log
.info("Error With URL "+httpurl
);
991 except socket
.timeout
:
992 log
.info("Error With URL "+httpurl
);
994 httpcodeout
= geturls_text
.status
;
995 httpcodereason
= http_status_to_reason(geturls_text
.status
);
996 httpversionout
= "1.1";
997 httpmethodout
= httpmethod
;
998 httpurlout
= str(httpurl
);
999 httpheaderout
= geturls_text
.headers
;
1000 httpheadersentout
= httpheaders
;
1001 elif(httplibuse
=="httpcore2"):
1003 if(httpmethod
=="GET"):
1004 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
1005 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
1006 elif(httpmethod
=="POST"):
1007 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
1008 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
1010 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
1011 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
1012 except httpcore
.ConnectTimeout
:
1013 log
.info("Error With URL "+httpurl
);
1015 except httpcore
.ConnectError
:
1016 log
.info("Error With URL "+httpurl
);
1018 except socket
.timeout
:
1019 log
.info("Error With URL "+httpurl
);
1021 httpcodeout
= geturls_text
.status
;
1022 httpcodereason
= http_status_to_reason(geturls_text
.status
);
1023 httpversionout
= "1.1";
1024 httpmethodout
= httpmethod
;
1025 httpurlout
= str(httpurl
);
1026 httpheaderout
= geturls_text
.headers
;
1027 httpheadersentout
= httpheaders
;
1028 elif(httplibuse
=="mechanize"):
1029 geturls_opener
= mechanize
.Browser();
1030 if(isinstance(httpheaders
, dict)):
1031 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
1033 geturls_opener
.addheaders
= httpheaders
;
1034 geturls_opener
.set_cookiejar(httpcookie
);
1035 geturls_opener
.set_handle_robots(False);
1036 if(postdata
is not None and not isinstance(postdata
, dict)):
1037 postdata
= urlencode(postdata
);
1039 if(httpmethod
=="GET"):
1040 geturls_text
= geturls_opener
.open(httpurl
);
1041 elif(httpmethod
=="POST"):
1042 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
1044 geturls_text
= geturls_opener
.open(httpurl
);
1045 except mechanize
.HTTPError
as geturls_text_error
:
1046 geturls_text
= geturls_text_error
;
1047 log
.info("Error With URL "+httpurl
);
1049 log
.info("Error With URL "+httpurl
);
1051 except socket
.timeout
:
1052 log
.info("Error With URL "+httpurl
);
1054 httpcodeout
= geturls_text
.code
;
1055 httpcodereason
= geturls_text
.msg
;
1056 httpversionout
= "1.1";
1057 httpmethodout
= httpmethod
;
1058 httpurlout
= geturls_text
.geturl();
1059 httpheaderout
= geturls_text
.info();
1060 reqhead
= geturls_opener
.request
;
1061 httpheadersentout
= reqhead
.header_items();
1062 elif(httplibuse
=="pycurl"):
1063 retrieved_body
= BytesIO();
1064 retrieved_headers
= BytesIO();
1066 if(httpmethod
=="GET"):
1067 geturls_text
= pycurl
.Curl();
1068 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1069 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_1_1
);
1070 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1071 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1072 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1073 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1074 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1075 geturls_text
.perform();
1076 elif(httpmethod
=="POST"):
1077 geturls_text
= pycurl
.Curl();
1078 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1079 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_1_1
);
1080 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1081 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1082 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1083 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1084 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1085 geturls_text
.setopt(geturls_text
.POST
, True);
1086 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
1087 geturls_text
.perform();
1089 geturls_text
= pycurl
.Curl();
1090 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1091 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_1_1
);
1092 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1093 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1094 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1095 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1096 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1097 geturls_text
.perform();
1098 retrieved_headers
.seek(0);
1099 if(sys
.version
[0]=="2"):
1100 pycurlhead
= retrieved_headers
.read();
1101 if(sys
.version
[0]>="3"):
1102 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
1103 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
1104 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
1105 retrieved_body
.seek(0);
1106 except socket
.timeout
:
1107 log
.info("Error With URL "+httpurl
);
1109 except socket
.gaierror
:
1110 log
.info("Error With URL "+httpurl
);
1113 log
.info("Error With URL "+httpurl
);
1115 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
1116 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
1117 httpversionout
= pyhttpverinfo
[0];
1118 httpmethodout
= httpmethod
;
1119 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
1120 httpheaderout
= pycurlheadersout
;
1121 httpheadersentout
= httpheaders
;
1122 elif(httplibuse
=="pycurl2"):
1123 retrieved_body
= BytesIO();
1124 retrieved_headers
= BytesIO();
1126 if(httpmethod
=="GET"):
1127 geturls_text
= pycurl
.Curl();
1128 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
1129 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1130 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1131 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1132 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1133 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1134 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1135 geturls_text
.perform();
1136 elif(httpmethod
=="POST"):
1137 geturls_text
= pycurl
.Curl();
1138 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
1139 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1140 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1141 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1142 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1143 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1144 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1145 geturls_text
.setopt(geturls_text
.POST
, True);
1146 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
1147 geturls_text
.perform();
1149 geturls_text
= pycurl
.Curl();
1150 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
1151 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1152 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1153 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1154 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1155 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1156 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1157 geturls_text
.perform();
1158 retrieved_headers
.seek(0);
1159 if(sys
.version
[0]=="2"):
1160 pycurlhead
= retrieved_headers
.read();
1161 if(sys
.version
[0]>="3"):
1162 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
1163 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip())[0];
1164 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
1165 retrieved_body
.seek(0);
1166 except socket
.timeout
:
1167 log
.info("Error With URL "+httpurl
);
1169 except socket
.gaierror
:
1170 log
.info("Error With URL "+httpurl
);
1173 log
.info("Error With URL "+httpurl
);
1175 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
1176 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
1177 httpversionout
= pyhttpverinfo
[0];
1178 httpmethodout
= httpmethod
;
1179 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
1180 httpheaderout
= pycurlheadersout
;
1181 httpheadersentout
= httpheaders
;
1182 elif(httplibuse
=="pycurl3"):
1183 retrieved_body
= BytesIO();
1184 retrieved_headers
= BytesIO();
1186 if(httpmethod
=="GET"):
1187 geturls_text
= pycurl
.Curl();
1188 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
1189 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1190 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1191 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1192 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1193 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1194 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1195 geturls_text
.perform();
1196 elif(httpmethod
=="POST"):
1197 geturls_text
= pycurl
.Curl();
1198 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
1199 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1200 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1201 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1202 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1203 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1204 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1205 geturls_text
.setopt(geturls_text
.POST
, True);
1206 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
1207 geturls_text
.perform();
1209 geturls_text
= pycurl
.Curl();
1210 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
1211 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
1212 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
1213 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
1214 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
1215 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
1216 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
1217 geturls_text
.perform();
1218 retrieved_headers
.seek(0);
1219 if(sys
.version
[0]=="2"):
1220 pycurlhead
= retrieved_headers
.read();
1221 if(sys
.version
[0]>="3"):
1222 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
1223 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
1224 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
1225 retrieved_body
.seek(0);
1226 except socket
.timeout
:
1227 log
.info("Error With URL "+httpurl
);
1229 except socket
.gaierror
:
1230 log
.info("Error With URL "+httpurl
);
1233 log
.info("Error With URL "+httpurl
);
1235 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
1236 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
1237 httpversionout
= pyhttpverinfo
[0];
1238 httpmethodout
= httpmethod
;
1239 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
1240 httpheaderout
= pycurlheadersout
;
1241 httpheadersentout
= httpheaders
;
1242 elif(httplibuse
=="ftp"):
1243 geturls_text
= download_file_from_ftp_file(httpurl
);
1244 if(not geturls_text
):
1246 downloadsize
= None;
1247 if(downloadsize
is not None):
1248 downloadsize
= int(downloadsize
);
1249 if downloadsize
is None: downloadsize
= 0;
1252 log
.info("Downloading URL "+httpurl
);
1253 with
BytesIO() as strbuf
:
1255 databytes
= geturls_text
.read(buffersize
);
1256 if not databytes
: break;
1257 datasize
= len(databytes
);
1258 fulldatasize
= datasize
+ fulldatasize
;
1261 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1262 downloaddiff
= fulldatasize
- prevdownsize
;
1263 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1264 prevdownsize
= fulldatasize
;
1265 strbuf
.write(databytes
);
1267 returnval_content
= strbuf
.read();
1268 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
1269 geturls_text
.close();
1270 elif(httplibuse
=="sftp"):
1271 geturls_text
= download_file_from_sftp_file(httpurl
);
1272 if(not geturls_text
):
1274 downloadsize
= None;
1275 if(downloadsize
is not None):
1276 downloadsize
= int(downloadsize
);
1277 if downloadsize
is None: downloadsize
= 0;
1280 log
.info("Downloading URL "+httpurl
);
1281 with
BytesIO() as strbuf
:
1283 databytes
= geturls_text
.read(buffersize
);
1284 if not databytes
: break;
1285 datasize
= len(databytes
);
1286 fulldatasize
= datasize
+ fulldatasize
;
1289 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1290 downloaddiff
= fulldatasize
- prevdownsize
;
1291 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1292 prevdownsize
= fulldatasize
;
1293 strbuf
.write(databytes
);
1295 returnval_content
= strbuf
.read();
1296 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
1297 geturls_text
.close();
1299 elif(httplibuse
=="pysftp"):
1300 geturls_text
= download_file_from_pysftp_file(httpurl
);
1301 if(not geturls_text
):
1303 downloadsize
= None;
1304 if(downloadsize
is not None):
1305 downloadsize
= int(downloadsize
);
1306 if downloadsize
is None: downloadsize
= 0;
1309 log
.info("Downloading URL "+httpurl
);
1310 with
BytesIO() as strbuf
:
1312 databytes
= geturls_text
.read(buffersize
);
1313 if not databytes
: break;
1314 datasize
= len(databytes
);
1315 fulldatasize
= datasize
+ fulldatasize
;
1318 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1319 downloaddiff
= fulldatasize
- prevdownsize
;
1320 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1321 prevdownsize
= fulldatasize
;
1322 strbuf
.write(databytes
);
1324 returnval_content
= strbuf
.read();
1325 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
1326 geturls_text
.close();
1330 if(isinstance(httpheaderout
, list) and (httplibuse
!="pycurl" and httplibuse
!="pycurl2" and httplibuse
!="pycurl3")):
1331 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1332 if(isinstance(httpheaderout
, list) and (httplibuse
=="pycurl" or httplibuse
=="pycurl2" or httplibuse
=="pycurl3")):
1333 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
1334 if(sys
.version
[0]=="2"):
1336 prehttpheaderout
= httpheaderout
;
1337 httpheaderkeys
= httpheaderout
.keys();
1338 imax
= len(httpheaderkeys
);
1342 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1344 except AttributeError:
1346 httpheaderout
= fix_header_names(httpheaderout
);
1347 if(isinstance(httpheadersentout
, list) and (httplibuse
!="pycurl" and httplibuse
!="pycurl2" and httplibuse
!="pycurl3")):
1348 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1349 if(isinstance(httpheadersentout
, list) and (httplibuse
=="pycurl" or httplibuse
=="pycurl2" or httplibuse
=="pycurl3")):
1350 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
1351 httpheadersentout
= fix_header_names(httpheadersentout
);
1352 log
.info("Downloading URL "+httpurl
);
1353 if(httplibuse
=="urllib" or httplibuse
=="request" or httplibuse
=="request3" or httplibuse
=="aiohttp" or httplibuse
=="httplib" or httplibuse
=="httplib2" or httplibuse
=="urllib3" or httplibuse
=="mechanize"):
1354 downloadsize
= httpheaderout
.get('Content-Length');
1355 if(downloadsize
is not None):
1356 downloadsize
= int(downloadsize
);
1357 if downloadsize
is None: downloadsize
= 0;
1360 log
.info("Downloading URL "+httpurl
);
1361 with
BytesIO() as strbuf
:
1363 databytes
= geturls_text
.read(buffersize
);
1364 if not databytes
: break;
1365 datasize
= len(databytes
);
1366 fulldatasize
= datasize
+ fulldatasize
;
1369 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1370 downloaddiff
= fulldatasize
- prevdownsize
;
1371 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1372 prevdownsize
= fulldatasize
;
1373 strbuf
.write(databytes
);
1375 returnval_content
= strbuf
.read();
1376 geturls_text
.close();
1377 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1379 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1382 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1384 returnval_content
= zlib
.decompress(returnval_content
);
1387 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1389 returnval_content
= brotli
.decompress(returnval_content
);
1390 except brotli
.error
:
1392 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1394 returnval_content
= zstandard
.decompress(returnval_content
);
1395 except zstandard
.error
:
1397 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1399 returnval_content
= lzma
.decompress(returnval_content
);
1400 except zstandard
.error
:
1402 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1404 returnval_content
= bz2
.decompress(returnval_content
);
1405 except zstandard
.error
:
1407 elif(httplibuse
=="httpx" or httplibuse
=="httpx2" or httplibuse
=="httpcore" or httplibuse
=="httpcore2"):
1408 downloadsize
= httpheaderout
.get('Content-Length');
1409 if(downloadsize
is not None):
1410 downloadsize
= int(downloadsize
);
1411 if downloadsize
is None: downloadsize
= 0;
1414 log
.info("Downloading URL "+httpurl
);
1415 with
BytesIO() as strbuf
:
1417 databytes
= geturls_text
.read();
1418 if not databytes
: break;
1419 datasize
= len(databytes
);
1420 fulldatasize
= datasize
+ fulldatasize
;
1423 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1424 downloaddiff
= fulldatasize
- prevdownsize
;
1425 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1426 prevdownsize
= fulldatasize
;
1427 strbuf
.write(databytes
);
1430 returnval_content
= strbuf
.read();
1431 geturls_text
.close();
1432 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1434 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1437 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1439 returnval_content
= zlib
.decompress(returnval_content
);
1442 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1444 returnval_content
= brotli
.decompress(returnval_content
);
1445 except brotli
.error
:
1447 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1449 returnval_content
= zstandard
.decompress(returnval_content
);
1450 except zstandard
.error
:
1452 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1454 returnval_content
= lzma
.decompress(returnval_content
);
1455 except zstandard
.error
:
1457 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1459 returnval_content
= bz2
.decompress(returnval_content
);
1460 except zstandard
.error
:
1462 elif(httplibuse
=="requests"):
1463 log
.info("Downloading URL "+httpurl
);
1464 downloadsize
= httpheaderout
.get('Content-Length');
1465 if(downloadsize
is not None):
1466 downloadsize
= int(downloadsize
);
1467 if downloadsize
is None: downloadsize
= 0;
1470 log
.info("Downloading URL "+httpurl
);
1471 with
BytesIO() as strbuf
:
1473 databytes
= geturls_text
.raw
.read(buffersize
);
1474 if not databytes
: break;
1475 datasize
= len(databytes
);
1476 fulldatasize
= datasize
+ fulldatasize
;
1479 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1480 downloaddiff
= fulldatasize
- prevdownsize
;
1481 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1482 prevdownsize
= fulldatasize
;
1483 strbuf
.write(databytes
);
1485 returnval_content
= strbuf
.read();
1486 geturls_text
.close();
1487 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1489 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1492 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1494 returnval_content
= zlib
.decompress(returnval_content
);
1497 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1499 returnval_content
= brotli
.decompress(returnval_content
);
1500 except brotli
.error
:
1502 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1504 returnval_content
= zstandard
.decompress(returnval_content
);
1505 except zstandard
.error
:
1507 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1509 returnval_content
= lzma
.decompress(returnval_content
);
1510 except zstandard
.error
:
1512 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1514 returnval_content
= bz2
.decompress(returnval_content
);
1515 except zstandard
.error
:
1517 elif(httplibuse
=="pycurl" or httplibuse
=="pycurl2" or httplibuse
=="pycurl3"):
1518 log
.info("Downloading URL "+httpurl
);
1519 downloadsize
= httpheaderout
.get('Content-Length');
1520 if(downloadsize
is not None):
1521 downloadsize
= int(downloadsize
);
1522 if downloadsize
is None: downloadsize
= 0;
1525 log
.info("Downloading URL "+httpurl
);
1526 with
BytesIO() as strbuf
:
1528 databytes
= retrieved_body
.read(buffersize
);
1529 if not databytes
: break;
1530 datasize
= len(databytes
);
1531 fulldatasize
= datasize
+ fulldatasize
;
1534 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1535 downloaddiff
= fulldatasize
- prevdownsize
;
1536 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1537 prevdownsize
= fulldatasize
;
1538 strbuf
.write(databytes
);
1540 returnval_content
= strbuf
.read();
1541 geturls_text
.close();
1542 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1544 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1547 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1549 returnval_content
= zlib
.decompress(returnval_content
);
1552 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1554 returnval_content
= brotli
.decompress(returnval_content
);
1555 except brotli
.error
:
1557 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1559 returnval_content
= zstandard
.decompress(returnval_content
);
1560 except zstandard
.error
:
1562 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1564 returnval_content
= lzma
.decompress(returnval_content
);
1565 except zstandard
.error
:
1567 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1569 returnval_content
= bz2
.decompress(returnval_content
);
1570 except zstandard
.error
:
1572 elif(httplibuse
=="ftp" or httplibuse
=="sftp" or httplibuse
=="pysftp"):
1576 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': httplibuse
};
1579 def download_from_url_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1580 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
, haveaiohttp
, haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
1581 exec_time_start
= time
.time();
1582 myhash
= hashlib
.new("sha1");
1583 if(sys
.version
[0]=="2"):
1584 myhash
.update(httpurl
);
1585 myhash
.update(str(buffersize
));
1586 myhash
.update(str(exec_time_start
));
1587 if(sys
.version
[0]>="3"):
1588 myhash
.update(httpurl
.encode('utf-8'));
1589 myhash
.update(str(buffersize
).encode('utf-8'));
1590 myhash
.update(str(exec_time_start
).encode('utf-8'));
1591 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1593 sleep
= geturls_download_sleep
;
1596 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
1597 httplibuse
= "urllib";
1598 if(httplibuse
=="httplib1"):
1599 httplibuse
= "httplib";
1600 if(not haverequests
and httplibuse
=="requests"):
1601 httplibuse
= "urllib";
1602 if(not haveaiohttp
and httplibuse
=="aiohttp"):
1603 httplibuse
= "urllib";
1604 if(not havehttpx
and httplibuse
=="httpx"):
1605 httplibuse
= "urllib";
1606 if(not havehttpx
and httplibuse
=="httpx2"):
1607 httplibuse
= "urllib";
1608 if(not havehttpcore
and httplibuse
=="httpcore"):
1609 httplibuse
= "urllib";
1610 if(not havehttpcore
and httplibuse
=="httpcore2"):
1611 httplibuse
= "urllib";
1612 if(not havemechanize
and httplibuse
=="mechanize"):
1613 httplibuse
= "urllib";
1614 if(not havepycurl
and httplibuse
=="pycurl"):
1615 httplibuse
= "urllib";
1616 if(not havepycurl
and httplibuse
=="pycurl2"):
1617 httplibuse
= "urllib";
1618 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1619 httplibuse
= "pycurl";
1620 if(not havepycurl
and httplibuse
=="pycurl3"):
1621 httplibuse
= "urllib";
1622 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1623 httplibuse
= "pycurl2";
1624 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1625 httplibuse
= "pycurl";
1626 if(not havehttplib2
and httplibuse
=="httplib2"):
1627 httplibuse
= "httplib";
1628 if(not haveparamiko
and httplibuse
=="sftp"):
1630 if(not haveparamiko
and httplibuse
=="pysftp"):
1632 pretmpfilename
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, buffersize
, sleep
, timeout
);
1633 if(not pretmpfilename
):
1635 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1636 tmpfilename
= f
.name
;
1638 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1639 except AttributeError:
1641 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1646 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1647 f
.write(pretmpfilename
.get('Content'));
1649 exec_time_end
= time
.time();
1650 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1651 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1654 def download_from_url_to_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1655 global geturls_download_sleep
, havezstd
, havebrotli
, haveaiohttp
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
1657 sleep
= geturls_download_sleep
;
1660 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
1661 httplibuse
= "urllib";
1662 if(httplibuse
=="httplib1"):
1663 httplibuse
= "httplib";
1664 if(not haverequests
and httplibuse
=="requests"):
1665 httplibuse
= "urllib";
1666 if(not haveaiohttp
and httplibuse
=="aiohttp"):
1667 httplibuse
= "urllib";
1668 if(not havehttpx
and httplibuse
=="httpx"):
1669 httplibuse
= "urllib";
1670 if(not havehttpx
and httplibuse
=="httpx2"):
1671 httplibuse
= "urllib";
1672 if(not havehttpcore
and httplibuse
=="httpcore"):
1673 httplibuse
= "urllib";
1674 if(not havehttpcore
and httplibuse
=="httpcore2"):
1675 httplibuse
= "urllib";
1676 if(not havemechanize
and httplibuse
=="mechanize"):
1677 httplibuse
= "urllib";
1678 if(not havepycurl
and httplibuse
=="pycurl"):
1679 httplibuse
= "urllib";
1680 if(not havepycurl
and httplibuse
=="pycurl2"):
1681 httplibuse
= "urllib";
1682 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1683 httplibuse
= "pycurl";
1684 if(not havepycurl
and httplibuse
=="pycurl3"):
1685 httplibuse
= "urllib";
1686 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1687 httplibuse
= "pycurl2";
1688 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
1689 httplibuse
= "pycurl";
1690 if(not havehttplib2
and httplibuse
=="httplib2"):
1691 httplibuse
= "httplib";
1692 if(not haveparamiko
and httplibuse
=="sftp"):
1694 if(not havepysftp
and httplibuse
=="pysftp"):
1696 if(not outfile
=="-"):
1697 outpath
= outpath
.rstrip(os
.path
.sep
);
1698 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1699 if(not os
.path
.exists(outpath
)):
1700 os
.makedirs(outpath
);
1701 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1703 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1705 pretmpfilename
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, ranges
, buffersize
[0], sleep
, timeout
);
1706 if(not pretmpfilename
):
1708 tmpfilename
= pretmpfilename
.get('Filename');
1709 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1711 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1712 exec_time_start
= time
.time();
1713 shutil
.move(tmpfilename
, filepath
);
1715 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1716 except AttributeError:
1718 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1723 exec_time_end
= time
.time();
1724 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1725 if(os
.path
.exists(tmpfilename
)):
1726 os
.remove(tmpfilename
);
1727 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1729 pretmpfilename
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, ranges
, buffersize
[0], sleep
, timeout
);
1730 tmpfilename
= pretmpfilename
.get('Filename');
1731 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1734 exec_time_start
= time
.time();
1735 with
open(tmpfilename
, 'rb') as ft
:
1738 databytes
= ft
.read(buffersize
[1]);
1739 if not databytes
: break;
1740 datasize
= len(databytes
);
1741 fulldatasize
= datasize
+ fulldatasize
;
1744 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1745 downloaddiff
= fulldatasize
- prevdownsize
;
1746 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1747 prevdownsize
= fulldatasize
;
1750 fdata
= f
.getvalue();
1753 os
.remove(tmpfilename
);
1754 exec_time_end
= time
.time();
1755 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1756 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': httplibuse
};
1759 def download_from_url_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1760 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", buffersize
, sleep
, timeout
);
1763 def download_from_url_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1764 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", buffersize
, sleep
, timeout
);
1767 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1768 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "request3", buffersize
, sleep
, timeout
);
1771 def download_from_url_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1772 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib", buffersize
, sleep
, timeout
);
1775 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1776 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib2", buffersize
, sleep
, timeout
);
1779 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1780 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib3", buffersize
, sleep
, timeout
);
1783 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1784 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "requests", buffersize
, sleep
, timeout
);
1787 def download_from_url_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1788 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "aiohttp", buffersize
, sleep
, timeout
);
1791 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1792 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx", buffersize
, sleep
, timeout
);
1795 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1796 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx2", buffersize
, sleep
, timeout
);
1799 def download_from_url_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1800 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore", buffersize
, sleep
, timeout
);
1803 def download_from_url_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1804 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore2", buffersize
, sleep
, timeout
);
1807 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1808 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "mechanize", buffersize
, sleep
, timeout
);
1811 def download_from_url_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1812 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl", buffersize
, sleep
, timeout
);
1815 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1816 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl2", buffersize
, sleep
, timeout
);
1819 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1820 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl3", buffersize
, sleep
, timeout
);
1823 def download_from_url_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1824 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "ftp", buffersize
, sleep
, timeout
);
1827 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1828 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "sftp", buffersize
, sleep
, timeout
);
1831 def download_from_url_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1832 returnval
= download_from_url(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pysftp", buffersize
, sleep
, timeout
);
1835 def download_from_url_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1836 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", ranges
, buffersize
, sleep
, timeout
);
1839 def download_from_url_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1840 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", ranges
, buffersize
, sleep
, timeout
);
1843 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1844 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "request3", ranges
, buffersize
, sleep
, timeout
);
1847 def download_from_url_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1848 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib", ranges
, buffersize
, sleep
, timeout
);
1851 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1852 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib2", ranges
, buffersize
, sleep
, timeout
);
1855 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1856 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib3", ranges
, buffersize
, sleep
, timeout
);
1859 def download_from_url_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1860 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "requests", ranges
, buffersize
, sleep
, timeout
);
1863 def download_from_url_file_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1864 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "aiohttp", ranges
, buffersize
, sleep
, timeout
);
1867 def download_from_url_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1868 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx", ranges
, buffersize
, sleep
, timeout
);
1871 def download_from_url_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1872 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx2", ranges
, buffersize
, sleep
, timeout
);
1875 def download_from_url_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1876 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore", ranges
, buffersize
, sleep
, timeout
);
1879 def download_from_url_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1880 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore2", ranges
, buffersize
, sleep
, timeout
);
1883 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1884 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "mechanize", ranges
, buffersize
, sleep
, timeout
);
1887 def download_from_url_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1888 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl", ranges
, buffersize
, sleep
, timeout
);
1891 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1892 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl2", ranges
, buffersize
, sleep
, timeout
);
1895 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1896 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl3", ranges
, buffersize
, sleep
, timeout
);
1899 def download_from_url_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1900 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "ftp", ranges
, buffersize
, sleep
, timeout
);
1903 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1904 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "sftp", ranges
, buffersize
, sleep
, timeout
);
1907 def download_from_url_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1908 returnval
= download_from_url_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pysftp", ranges
, buffersize
, sleep
, timeout
);
1911 def download_from_url_to_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1912 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1915 def download_from_url_to_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1916 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "request", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1919 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1920 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1923 def download_from_url_to_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1924 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1927 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1928 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httplib2", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1931 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1932 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "urllib3", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1935 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1936 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "requests", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1939 def download_from_url_to_file_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1940 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "aiohttp", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1943 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1944 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1947 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1948 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpx2", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1951 def download_from_url_to_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1952 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1955 def download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1956 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "httpcore2", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1959 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1960 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "mechanize", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1963 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1964 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1967 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1968 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl2", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1971 def download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1972 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pycurl3", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1975 def download_from_url_to_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1976 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "ftp", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1979 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1980 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "sftp", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1983 def download_from_url_to_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1984 returnval
= download_from_url_to_file(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, "pysftp", outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
1987 def download_file_from_ftp_file(url
):
1988 urlparts
= urlparse
.urlparse(url
);
1989 file_name
= os
.path
.basename(urlparts
.path
);
1990 file_dir
= os
.path
.dirname(urlparts
.path
);
1991 if(urlparts
.username
is not None):
1992 ftp_username
= urlparts
.username
;
1994 ftp_username
= "anonymous";
1995 if(urlparts
.password
is not None):
1996 ftp_password
= urlparts
.password
;
1997 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
1998 ftp_password
= "anonymous";
2001 if(urlparts
.scheme
=="ftp"):
2003 elif(urlparts
.scheme
=="ftps"):
2007 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2009 ftp_port
= urlparts
.port
;
2010 if(urlparts
.port
is None):
2013 ftp
.connect(urlparts
.hostname
, ftp_port
);
2014 except socket
.gaierror
:
2015 log
.info("Error With URL "+httpurl
);
2017 except socket
.timeout
:
2018 log
.info("Error With URL "+httpurl
);
2020 ftp
.login(urlparts
.username
, urlparts
.password
);
2021 if(urlparts
.scheme
=="ftps"):
2023 ftpfile
= BytesIO();
2024 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
2025 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2030 def download_file_from_ftp_string(url
):
2031 ftpfile
= download_file_from_ftp_file(url
);
2032 return ftpfile
.read();
2034 def upload_file_to_ftp_file(ftpfile
, url
):
2035 urlparts
= urlparse
.urlparse(url
);
2036 file_name
= os
.path
.basename(urlparts
.path
);
2037 file_dir
= os
.path
.dirname(urlparts
.path
);
2038 if(urlparts
.username
is not None):
2039 ftp_username
= urlparts
.username
;
2041 ftp_username
= "anonymous";
2042 if(urlparts
.password
is not None):
2043 ftp_password
= urlparts
.password
;
2044 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2045 ftp_password
= "anonymous";
2048 if(urlparts
.scheme
=="ftp"):
2050 elif(urlparts
.scheme
=="ftps"):
2054 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2056 ftp_port
= urlparts
.port
;
2057 if(urlparts
.port
is None):
2060 ftp
.connect(urlparts
.hostname
, ftp_port
);
2061 except socket
.gaierror
:
2062 log
.info("Error With URL "+httpurl
);
2064 except socket
.timeout
:
2065 log
.info("Error With URL "+httpurl
);
2067 ftp
.login(urlparts
.username
, urlparts
.password
);
2068 if(urlparts
.scheme
=="ftps"):
2070 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
2075 def upload_file_to_ftp_string(ftpstring
, url
):
2076 ftpfileo
= BytesIO(ftpstring
);
2077 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
2082 def download_file_from_sftp_file(url
):
2083 urlparts
= urlparse
.urlparse(url
);
2084 file_name
= os
.path
.basename(urlparts
.path
);
2085 file_dir
= os
.path
.dirname(urlparts
.path
);
2086 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2088 sftp_port
= urlparts
.port
;
2089 if(urlparts
.port
is None):
2092 sftp_port
= urlparts
.port
;
2093 if(urlparts
.username
is not None):
2094 sftp_username
= urlparts
.username
;
2096 sftp_username
= "anonymous";
2097 if(urlparts
.password
is not None):
2098 sftp_password
= urlparts
.password
;
2099 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2100 sftp_password
= "anonymous";
2103 if(urlparts
.scheme
!="sftp"):
2105 ssh
= paramiko
.SSHClient();
2106 ssh
.load_system_host_keys();
2107 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
2109 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
2110 except paramiko
.ssh_exception
.SSHException
:
2112 except socket
.gaierror
:
2113 log
.info("Error With URL "+httpurl
);
2115 except socket
.timeout
:
2116 log
.info("Error With URL "+httpurl
);
2118 sftp
= ssh
.open_sftp();
2119 sftpfile
= BytesIO();
2120 sftp
.getfo(urlparts
.path
, sftpfile
);
2123 sftpfile
.seek(0, 0);
2126 def download_file_from_sftp_file(url
):
2130 def download_file_from_sftp_string(url
):
2131 sftpfile
= download_file_from_sftp_file(url
);
2132 return sftpfile
.read();
2134 def download_file_from_ftp_string(url
):
2138 def upload_file_to_sftp_file(sftpfile
, url
):
2139 urlparts
= urlparse
.urlparse(url
);
2140 file_name
= os
.path
.basename(urlparts
.path
);
2141 file_dir
= os
.path
.dirname(urlparts
.path
);
2142 sftp_port
= urlparts
.port
;
2143 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2145 if(urlparts
.port
is None):
2148 sftp_port
= urlparts
.port
;
2149 if(urlparts
.username
is not None):
2150 sftp_username
= urlparts
.username
;
2152 sftp_username
= "anonymous";
2153 if(urlparts
.password
is not None):
2154 sftp_password
= urlparts
.password
;
2155 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2156 sftp_password
= "anonymous";
2159 if(urlparts
.scheme
!="sftp"):
2161 ssh
= paramiko
.SSHClient();
2162 ssh
.load_system_host_keys();
2163 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
2165 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
2166 except paramiko
.ssh_exception
.SSHException
:
2168 except socket
.gaierror
:
2169 log
.info("Error With URL "+httpurl
);
2171 except socket
.timeout
:
2172 log
.info("Error With URL "+httpurl
);
2174 sftp
= ssh
.open_sftp();
2175 sftp
.putfo(sftpfile
, urlparts
.path
);
2178 sftpfile
.seek(0, 0);
2181 def upload_file_to_sftp_file(sftpfile
, url
):
2185 def upload_file_to_sftp_string(sftpstring
, url
):
2186 sftpfileo
= BytesIO(sftpstring
);
2187 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
2191 def upload_file_to_sftp_string(url
):
2196 def download_file_from_pysftp_file(url
):
2197 urlparts
= urlparse
.urlparse(url
);
2198 file_name
= os
.path
.basename(urlparts
.path
);
2199 file_dir
= os
.path
.dirname(urlparts
.path
);
2200 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2202 sftp_port
= urlparts
.port
;
2203 if(urlparts
.port
is None):
2206 sftp_port
= urlparts
.port
;
2207 if(urlparts
.username
is not None):
2208 sftp_username
= urlparts
.username
;
2210 sftp_username
= "anonymous";
2211 if(urlparts
.password
is not None):
2212 sftp_password
= urlparts
.password
;
2213 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2214 sftp_password
= "anonymous";
2217 if(urlparts
.scheme
!="sftp"):
2220 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
2221 except paramiko
.ssh_exception
.SSHException
:
2223 except socket
.gaierror
:
2224 log
.info("Error With URL "+httpurl
);
2226 except socket
.timeout
:
2227 log
.info("Error With URL "+httpurl
);
2229 sftp
= ssh
.open_sftp();
2230 sftpfile
= BytesIO();
2231 sftp
.getfo(urlparts
.path
, sftpfile
);
2234 sftpfile
.seek(0, 0);
2237 def download_file_from_pysftp_file(url
):
2241 def download_file_from_pysftp_string(url
):
2242 sftpfile
= download_file_from_pysftp_file(url
);
2243 return sftpfile
.read();
2245 def download_file_from_ftp_string(url
):
2249 def upload_file_to_pysftp_file(sftpfile
, url
):
2250 urlparts
= urlparse
.urlparse(url
);
2251 file_name
= os
.path
.basename(urlparts
.path
);
2252 file_dir
= os
.path
.dirname(urlparts
.path
);
2253 sftp_port
= urlparts
.port
;
2254 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2256 if(urlparts
.port
is None):
2259 sftp_port
= urlparts
.port
;
2260 if(urlparts
.username
is not None):
2261 sftp_username
= urlparts
.username
;
2263 sftp_username
= "anonymous";
2264 if(urlparts
.password
is not None):
2265 sftp_password
= urlparts
.password
;
2266 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2267 sftp_password
= "anonymous";
2270 if(urlparts
.scheme
!="sftp"):
2273 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
2274 except paramiko
.ssh_exception
.SSHException
:
2276 except socket
.gaierror
:
2277 log
.info("Error With URL "+httpurl
);
2279 except socket
.timeout
:
2280 log
.info("Error With URL "+httpurl
);
2282 sftp
= ssh
.open_sftp();
2283 sftp
.putfo(sftpfile
, urlparts
.path
);
2286 sftpfile
.seek(0, 0);
2289 def upload_file_to_pysftp_file(sftpfile
, url
):
2293 def upload_file_to_pysftp_string(sftpstring
, url
):
2294 sftpfileo
= BytesIO(sftpstring
);
2295 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
2299 def upload_file_to_pysftp_string(url
):