?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Update pyhttpserv.py
[PyWWW-Get.git] / pywwwget.py
blobd3691cdbfcf340b904d47ce9164f0f05fc2c1465
1 #!/usr/bin/env python
2
3 '''
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
11
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
14
15 $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
16 '''
17
18 from __future__ import division, absolute_import, print_function;
19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, bz2, time, argparse, cgi, subprocess, socket, email.utils, datetime, time;
20 import logging as log;
21 from ftplib import FTP, FTP_TLS;
22 from base64 import b64encode;
23 haverequests = False;
24 try:
25 import requests;
26 haverequests = True;
27 except ImportError:
28 haverequests = False;
29 havemechanize = False;
30 try:
31 import mechanize;
32 havemechanize = True;
33 except ImportError:
34 havemechanize = False;
35 havepycurl = False;
36 try:
37 import pycurl;
38 havepycurl = True;
39 except ImportError:
40 havepycurl = False;
41 haveparamiko = False;
42 try:
43 import paramiko;
44 haveparamiko = True;
45 except ImportError:
46 haveparamiko = False;
47 havepysftp = False;
48 try:
49 import pysftp;
50 havepysftp = True;
51 except ImportError:
52 havepysftp = False;
53 haveurllib3 = False;
54 try:
55 import urllib3;
56 haveurllib3 = True;
57 except ImportError:
58 haveurllib3 = False;
59 havehttplib2 = False;
60 try:
61 from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
62 havehttplib2 = True;
63 except ImportError:
64 havehttplib2 = False;
65 havehttpx = False;
66 try:
67 import httpx;
68 havehttpx = True;
69 except ImportError:
70 havehttpx = False;
71 havehttpcore = False;
72 try:
73 import httpcore;
74 havehttpcore = True;
75 except ImportError:
76 havehttpcore = False;
77 haveaiohttp = False;
78 try:
79 import aiohttp;
80 haveaiohttp = True;
81 except ImportError:
82 haveaiohttp = False;
83 havebrotli = False;
84 try:
85 import brotli;
86 havebrotli = True;
87 except ImportError:
88 havebrotli = False;
89 havezstd = False;
90 try:
91 import zstandard;
92 havezstd = True;
93 except ImportError:
94 havezstd = False;
95 havelzma = False;
96 try:
97 import lzma;
98 havelzma = True;
99 except ImportError:
100 havelzma = False;
101 if(sys.version[0]=="2"):
102 try:
103 from io import StringIO, BytesIO;
104 except ImportError:
105 try:
106 from cStringIO import StringIO;
107 from cStringIO import StringIO as BytesIO;
108 except ImportError:
109 from StringIO import StringIO;
110 from StringIO import StringIO as BytesIO;
111 # From http://python-future.org/compatible_idioms.html
112 from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
113 from urllib import urlencode;
114 from urllib import urlopen as urlopenalt;
115 from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
116 import urlparse, cookielib;
117 from httplib import HTTPConnection, HTTPSConnection;
118 if(sys.version[0]>="3"):
119 from io import StringIO, BytesIO;
120 # From http://python-future.org/compatible_idioms.html
121 from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
122 from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
123 from urllib.error import HTTPError, URLError;
124 import urllib.parse as urlparse;
125 import http.cookiejar as cookielib;
126 from http.client import HTTPConnection, HTTPSConnection;
127
128 __program_name__ = "PyWWW-Get";
129 __program_alt_name__ = "PyWWWGet";
130 __program_small_name__ = "wwwget";
131 __project__ = __program_name__;
132 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
133 __version_info__ = (2, 0, 2, "RC 1", 1);
134 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
135 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
136 __revision__ = __version_info__[3];
137 __revision_id__ = "$Id$";
138 if(__version_info__[4] is not None):
139 __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
140 if(__version_info__[4] is None):
141 __version_date_plusrc__ = __version_date__;
142 if(__version_info__[3] is not None):
143 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
144 if(__version_info__[3] is None):
145 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
146
147 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
148 tmpfilesuffix = "-";
149 pytempdir = tempfile.gettempdir();
150
151 PyBitness = platform.architecture();
152 if(PyBitness=="32bit" or PyBitness=="32"):
153 PyBitness = "32";
154 elif(PyBitness=="64bit" or PyBitness=="64"):
155 PyBitness = "64";
156 else:
157 PyBitness = "32";
158
159 compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2'];
160 if(havebrotli):
161 compression_supported_list.append('br');
162 if(havezstd):
163 compression_supported_list.append('zstd');
164 if(havelzma):
165 compression_supported_list.append('lzma');
166 compression_supported_list.append('xz');
167 compression_supported = ', '.join(compression_supported_list);
168
169 geturls_cj = cookielib.CookieJar();
170 windowsNT4_ua_string = "Windows NT 4.0";
171 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
172 windows2k_ua_string = "Windows NT 5.0";
173 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
174 windowsXP_ua_string = "Windows NT 5.1";
175 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
176 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
177 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
178 windows7_ua_string = "Windows NT 6.1; Win64; x64";
179 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
180 windows8_ua_string = "Windows NT 6.2; Win64; x64";
181 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
182 windows81_ua_string = "Windows NT 6.3; Win64; x64";
183 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
184 windows10_ua_string = "Windows NT 10.0; Win64; x64";
185 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
186 windows11_ua_string = "Windows NT 11.0; Win64; x64";
187 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
188 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
189 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
190 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
191 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
192 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
193 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
194 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
195 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
196 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
197 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
198 if(platform.python_implementation()!=""):
199 py_implementation = platform.python_implementation();
200 if(platform.python_implementation()==""):
201 py_implementation = "Python";
202 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
203 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
204 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
205 geturls_ua = geturls_ua_firefox_windows7;
206 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
207 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
208 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
209 geturls_headers_chrome_windows7.update(windows7_ua_addon);
210 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
211 geturls_headers_chromium_windows7.update(windows7_ua_addon);
212 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
214 geturls_headers_opera_windows7.update(windows7_ua_addon);
215 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
216 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
217 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
219 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
220 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
221 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
222 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
223 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
224 geturls_headers = geturls_headers_firefox_windows7;
225 geturls_download_sleep = 0;
226
227 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
228 if(outtype=="print" and dbgenable):
229 print(dbgtxt);
230 return True;
231 elif(outtype=="log" and dbgenable):
232 logging.info(dbgtxt);
233 return True;
234 elif(outtype=="warning" and dbgenable):
235 logging.warning(dbgtxt);
236 return True;
237 elif(outtype=="error" and dbgenable):
238 logging.error(dbgtxt);
239 return True;
240 elif(outtype=="critical" and dbgenable):
241 logging.critical(dbgtxt);
242 return True;
243 elif(outtype=="exception" and dbgenable):
244 logging.exception(dbgtxt);
245 return True;
246 elif(outtype=="logalt" and dbgenable):
247 logging.log(dgblevel, dbgtxt);
248 return True;
249 elif(outtype=="debug" and dbgenable):
250 logging.debug(dbgtxt);
251 return True;
252 elif(not dbgenable):
253 return True;
254 else:
255 return False;
256 return False;
257
258 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
259 dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
260 if(not dbgout):
261 return False;
262 return dbgtxt;
263
264 def add_url_param(url, **params):
265 n=3;
266 parts = list(urlparse.urlsplit(url));
267 d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
268 d.update(params);
269 parts[n]=urlencode(d);
270 return urlparse.urlunsplit(parts);
271
272 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
273 def which_exec(execfile):
274 for path in os.environ["PATH"].split(":"):
275 if os.path.exists(path + "/" + execfile):
276 return path + "/" + execfile;
277
278 def listize(varlist):
279 il = 0;
280 ix = len(varlist);
281 ilx = 1;
282 newlistreg = {};
283 newlistrev = {};
284 newlistfull = {};
285 while(il < ix):
286 newlistreg.update({ilx: varlist[il]});
287 newlistrev.update({varlist[il]: ilx});
288 ilx = ilx + 1;
289 il = il + 1;
290 newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
291 return newlistfull;
292
293 def twolistize(varlist):
294 il = 0;
295 ix = len(varlist);
296 ilx = 1;
297 newlistnamereg = {};
298 newlistnamerev = {};
299 newlistdescreg = {};
300 newlistdescrev = {};
301 newlistfull = {};
302 while(il < ix):
303 newlistnamereg.update({ilx: varlist[il][0].strip()});
304 newlistnamerev.update({varlist[il][0].strip(): ilx});
305 newlistdescreg.update({ilx: varlist[il][1].strip()});
306 newlistdescrev.update({varlist[il][1].strip(): ilx});
307 ilx = ilx + 1;
308 il = il + 1;
309 newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
310 newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
311 newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
312 return newlistfull;
313
314 def arglistize(proexec, *varlist):
315 il = 0;
316 ix = len(varlist);
317 ilx = 1;
318 newarglist = [proexec];
319 while(il < ix):
320 if varlist[il][0] is not None:
321 newarglist.append(varlist[il][0]);
322 if varlist[il][1] is not None:
323 newarglist.append(varlist[il][1]);
324 il = il + 1;
325 return newarglist;
326
327 def fix_header_names(header_dict):
328 if(sys.version[0]=="2"):
329 header_dict = {k.title(): v for k, v in header_dict.iteritems()};
330 if(sys.version[0]>="3"):
331 header_dict = {k.title(): v for k, v in header_dict.items()};
332 return header_dict;
333
334 # hms_string by ArcGIS Python Recipes
335 # https://arcpy.wordpress.com/2012/04/20/146/
336 def hms_string(sec_elapsed):
337 h = int(sec_elapsed / (60 * 60));
338 m = int((sec_elapsed % (60 * 60)) / 60);
339 s = sec_elapsed % 60.0;
340 return "{}:{:>02}:{:>05.2f}".format(h, m, s);
341
342 # get_readable_size by Lipis
343 # http://stackoverflow.com/posts/14998888/revisions
344 def get_readable_size(bytes, precision=1, unit="IEC"):
345 unit = unit.upper();
346 if(unit!="IEC" and unit!="SI"):
347 unit = "IEC";
348 if(unit=="IEC"):
349 units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
350 unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
351 unitsize = 1024.0;
352 if(unit=="SI"):
353 units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
354 unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
355 unitsize = 1000.0;
356 return_val = {};
357 orgbytes = bytes;
358 for unit in units:
359 if abs(bytes) < unitsize:
360 strformat = "%3."+str(precision)+"f%s";
361 pre_return_val = (strformat % (bytes, unit));
362 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
363 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
364 alt_return_val = pre_return_val.split();
365 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
366 return return_val;
367 bytes /= unitsize;
368 strformat = "%."+str(precision)+"f%s";
369 pre_return_val = (strformat % (bytes, "YiB"));
370 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
371 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
372 alt_return_val = pre_return_val.split();
373 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
374 return return_val;
375
376 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
377 unit = unit.upper();
378 usehashtypes = usehashtypes.lower();
379 getfilesize = os.path.getsize(infile);
380 return_val = get_readable_size(getfilesize, precision, unit);
381 if(usehashes):
382 hashtypelist = usehashtypes.split(",");
383 openfile = open(infile, "rb");
384 filecontents = openfile.read();
385 openfile.close();
386 listnumcount = 0;
387 listnumend = len(hashtypelist);
388 while(listnumcount < listnumend):
389 hashtypelistlow = hashtypelist[listnumcount].strip();
390 hashtypelistup = hashtypelistlow.upper();
391 filehash = hashlib.new(hashtypelistup);
392 filehash.update(filecontents);
393 filegethash = filehash.hexdigest();
394 return_val.update({hashtypelistup: filegethash});
395 listnumcount += 1;
396 return return_val;
397
398 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
399 unit = unit.upper();
400 usehashtypes = usehashtypes.lower();
401 getfilesize = len(instring);
402 return_val = get_readable_size(getfilesize, precision, unit);
403 if(usehashes):
404 hashtypelist = usehashtypes.split(",");
405 listnumcount = 0;
406 listnumend = len(hashtypelist);
407 while(listnumcount < listnumend):
408 hashtypelistlow = hashtypelist[listnumcount].strip();
409 hashtypelistup = hashtypelistlow.upper();
410 filehash = hashlib.new(hashtypelistup);
411 if(sys.version[0]=="2"):
412 filehash.update(instring);
413 if(sys.version[0]>="3"):
414 filehash.update(instring.encode('utf-8'));
415 filegethash = filehash.hexdigest();
416 return_val.update({hashtypelistup: filegethash});
417 listnumcount += 1;
418 return return_val;
419
420 def http_status_to_reason(code):
421 reasons = {
422 100: 'Continue',
423 101: 'Switching Protocols',
424 102: 'Processing',
425 200: 'OK',
426 201: 'Created',
427 202: 'Accepted',
428 203: 'Non-Authoritative Information',
429 204: 'No Content',
430 205: 'Reset Content',
431 206: 'Partial Content',
432 207: 'Multi-Status',
433 208: 'Already Reported',
434 226: 'IM Used',
435 300: 'Multiple Choices',
436 301: 'Moved Permanently',
437 302: 'Found',
438 303: 'See Other',
439 304: 'Not Modified',
440 305: 'Use Proxy',
441 307: 'Temporary Redirect',
442 308: 'Permanent Redirect',
443 400: 'Bad Request',
444 401: 'Unauthorized',
445 402: 'Payment Required',
446 403: 'Forbidden',
447 404: 'Not Found',
448 405: 'Method Not Allowed',
449 406: 'Not Acceptable',
450 407: 'Proxy Authentication Required',
451 408: 'Request Timeout',
452 409: 'Conflict',
453 410: 'Gone',
454 411: 'Length Required',
455 412: 'Precondition Failed',
456 413: 'Payload Too Large',
457 414: 'URI Too Long',
458 415: 'Unsupported Media Type',
459 416: 'Range Not Satisfiable',
460 417: 'Expectation Failed',
461 421: 'Misdirected Request',
462 422: 'Unprocessable Entity',
463 423: 'Locked',
464 424: 'Failed Dependency',
465 426: 'Upgrade Required',
466 428: 'Precondition Required',
467 429: 'Too Many Requests',
468 431: 'Request Header Fields Too Large',
469 451: 'Unavailable For Legal Reasons',
470 500: 'Internal Server Error',
471 501: 'Not Implemented',
472 502: 'Bad Gateway',
473 503: 'Service Unavailable',
474 504: 'Gateway Timeout',
475 505: 'HTTP Version Not Supported',
476 506: 'Variant Also Negotiates',
477 507: 'Insufficient Storage',
478 508: 'Loop Detected',
479 510: 'Not Extended',
480 511: 'Network Authentication Required'
481 }
482 return reasons.get(code, 'Unknown Status Code');
483
484 def ftp_status_to_reason(code):
485 reasons = {
486 110: 'Restart marker reply',
487 120: 'Service ready in nnn minutes',
488 125: 'Data connection already open; transfer starting',
489 150: 'File status okay; about to open data connection',
490 200: 'Command okay',
491 202: 'Command not implemented, superfluous at this site',
492 211: 'System status, or system help reply',
493 212: 'Directory status',
494 213: 'File status',
495 214: 'Help message',
496 215: 'NAME system type',
497 220: 'Service ready for new user',
498 221: 'Service closing control connection',
499 225: 'Data connection open; no transfer in progress',
500 226: 'Closing data connection',
501 227: 'Entering Passive Mode',
502 230: 'User logged in, proceed',
503 250: 'Requested file action okay, completed',
504 257: '"PATHNAME" created',
505 331: 'User name okay, need password',
506 332: 'Need account for login',
507 350: 'Requested file action pending further information',
508 421: 'Service not available, closing control connection',
509 425: 'Can\'t open data connection',
510 426: 'Connection closed; transfer aborted',
511 450: 'Requested file action not taken',
512 451: 'Requested action aborted. Local error in processing',
513 452: 'Requested action not taken. Insufficient storage space in system',
514 500: 'Syntax error, command unrecognized',
515 501: 'Syntax error in parameters or arguments',
516 502: 'Command not implemented',
517 503: 'Bad sequence of commands',
518 504: 'Command not implemented for that parameter',
519 530: 'Not logged in',
520 532: 'Need account for storing files',
521 550: 'Requested action not taken. File unavailable',
522 551: 'Requested action aborted. Page type unknown',
523 552: 'Requested file action aborted. Exceeded storage allocation',
524 553: 'Requested action not taken. File name not allowed'
525 };
526 return reasons.get(code, 'Unknown Status Code');
527
528 def sftp_status_to_reason(code):
529 reasons = {
530 0: 'SSH_FX_OK',
531 1: 'SSH_FX_EOF',
532 2: 'SSH_FX_NO_SUCH_FILE',
533 3: 'SSH_FX_PERMISSION_DENIED',
534 4: 'SSH_FX_FAILURE',
535 5: 'SSH_FX_BAD_MESSAGE',
536 6: 'SSH_FX_NO_CONNECTION',
537 7: 'SSH_FX_CONNECTION_LOST',
538 8: 'SSH_FX_OP_UNSUPPORTED'
539 };
540 return reasons.get(code, 'Unknown Status Code');
541
542 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
543 if isinstance(headers, dict):
544 returnval = [];
545 if(sys.version[0]=="2"):
546 for headkey, headvalue in headers.iteritems():
547 returnval.append((headkey, headvalue));
548 if(sys.version[0]>="3"):
549 for headkey, headvalue in headers.items():
550 returnval.append((headkey, headvalue));
551 elif isinstance(headers, list):
552 returnval = headers;
553 else:
554 returnval = False;
555 return returnval;
556
557 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
558 if isinstance(headers, dict):
559 returnval = [];
560 if(sys.version[0]=="2"):
561 for headkey, headvalue in headers.iteritems():
562 returnval.append(headkey+": "+headvalue);
563 if(sys.version[0]>="3"):
564 for headkey, headvalue in headers.items():
565 returnval.append(headkey+": "+headvalue);
566 elif isinstance(headers, list):
567 returnval = headers;
568 else:
569 returnval = False;
570 return returnval;
571
572 def make_http_headers_from_pycurl_to_dict(headers):
573 header_dict = {};
574 headers = headers.strip().split('\r\n');
575 for header in headers:
576 parts = header.split(': ', 1)
577 if(len(parts) == 2):
578 key, value = parts;
579 header_dict[key.title()] = value;
580 return header_dict;
581
582 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
583 if isinstance(headers, list):
584 returnval = {};
585 mli = 0;
586 mlil = len(headers);
587 while(mli<mlil):
588 returnval.update({headers[mli][0]: headers[mli][1]});
589 mli = mli + 1;
590 elif isinstance(headers, dict):
591 returnval = headers;
592 else:
593 returnval = False;
594 return returnval;
595
596 def get_httplib_support(checkvalue=None):
597 global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
598 returnval = [];
599 returnval.append("ftp");
600 returnval.append("httplib");
601 if(havehttplib2):
602 returnval.append("httplib2");
603 returnval.append("urllib");
604 if(haveurllib3):
605 returnval.append("urllib3");
606 returnval.append("request3");
607 returnval.append("request");
608 if(haverequests):
609 returnval.append("requests");
610 if(haveaiohttp):
611 returnval.append("aiohttp");
612 if(havehttpx):
613 returnval.append("httpx");
614 returnval.append("httpx2");
615 if(havemechanize):
616 returnval.append("mechanize");
617 if(havepycurl):
618 returnval.append("pycurl");
619 if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
620 returnval.append("pycurl2");
621 if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
622 returnval.append("pycurl3");
623 if(haveparamiko):
624 returnval.append("sftp");
625 if(havepysftp):
626 returnval.append("pysftp");
627 if(not checkvalue is None):
628 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
629 checkvalue = "urllib";
630 if(checkvalue=="httplib1"):
631 checkvalue = "httplib";
632 if(checkvalue in returnval):
633 returnval = True;
634 else:
635 returnval = False;
636 return returnval;
637
638 def check_httplib_support(checkvalue="urllib"):
639 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
640 checkvalue = "urllib";
641 if(checkvalue=="httplib1"):
642 checkvalue = "httplib";
643 returnval = get_httplib_support(checkvalue);
644 return returnval;
645
646 def get_httplib_support_list():
647 returnval = get_httplib_support(None);
648 return returnval;
649
650 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
651 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
652 if(sleep<0):
653 sleep = geturls_download_sleep;
654 if(timeout<=0):
655 timeout = 10;
656 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
657 httplibuse = "urllib";
658 if(httplibuse=="httplib1"):
659 httplibuse = "httplib";
660 if(not haverequests and httplibuse=="requests"):
661 httplibuse = "urllib";
662 if(not haveaiohttp and httplibuse=="aiohttp"):
663 httplibuse = "urllib";
664 if(not havehttpx and httplibuse=="httpx"):
665 httplibuse = "urllib";
666 if(not havehttpx and httplibuse=="httpx2"):
667 httplibuse = "urllib";
668 if(not havehttpcore and httplibuse=="httpcore"):
669 httplibuse = "urllib";
670 if(not havehttpcore and httplibuse=="httpcore2"):
671 httplibuse = "urllib";
672 if(not havemechanize and httplibuse=="mechanize"):
673 httplibuse = "urllib";
674 if(not havepycurl and httplibuse=="pycurl"):
675 httplibuse = "urllib";
676 if(not havepycurl and httplibuse=="pycurl2"):
677 httplibuse = "urllib";
678 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
679 httplibuse = "pycurl";
680 if(not havepycurl and httplibuse=="pycurl3"):
681 httplibuse = "urllib";
682 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
683 httplibuse = "pycurl2";
684 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
685 httplibuse = "pycurl";
686 if(not havehttplib2 and httplibuse=="httplib2"):
687 httplibuse = "httplib";
688 if(not haveparamiko and httplibuse=="sftp"):
689 httplibuse = "ftp";
690 if(not havepysftp and httplibuse=="pysftp"):
691 httplibuse = "ftp";
692 urlparts = urlparse.urlparse(httpurl);
693 if(isinstance(httpheaders, list)):
694 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
695 httpheaders = fix_header_names(httpheaders);
696 if(httpuseragent is not None):
697 if('User-Agent' in httpheaders):
698 httpheaders['User-Agent'] = httpuseragent;
699 else:
700 httpuseragent.update({'User-Agent': httpuseragent});
701 if(httpreferer is not None):
702 if('Referer' in httpheaders):
703 httpheaders['Referer'] = httpreferer;
704 else:
705 httpuseragent.update({'Referer': httpreferer});
706 if(urlparts.username is not None or urlparts.password is not None):
707 if(sys.version[0]=="2"):
708 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
709 if(sys.version[0]>="3"):
710 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
711 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
712 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
713 if(httplibuse=="urllib" or httplibuse=="mechanize"):
714 if(isinstance(httpheaders, dict)):
715 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
716 if(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
717 if(isinstance(httpheaders, dict)):
718 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
719 geturls_opener.addheaders = httpheaders;
720 time.sleep(sleep);
721 if(postdata is not None and not isinstance(postdata, dict)):
722 postdata = urlencode(postdata);
723 if(httplibuse=="urllib" or httplibuse=="request"):
724 geturls_request = Request(httpurl);
725 try:
726 if(httpmethod=="GET"):
727 geturls_text = geturls_opener.open(geturls_request);
728 elif(httpmethod=="POST"):
729 geturls_text = geturls_opener.open(geturls_request, data=postdata);
730 else:
731 geturls_text = geturls_opener.open(geturls_request);
732 except HTTPError as geturls_text_error:
733 geturls_text = geturls_text_error;
734 log.info("Error With URL "+httpurl);
735 except URLError:
736 log.info("Error With URL "+httpurl);
737 return False;
738 except socket.timeout:
739 log.info("Error With URL "+httpurl);
740 return False;
741 httpcodeout = geturls_text.getcode();
742 try:
743 httpcodereason = geturls_text.reason;
744 except AttributeError:
745 httpcodereason = http_status_to_reason(geturls_text.getcode());
746 try:
747 httpversionout = geturls_text.version;
748 except AttributeError:
749 httpversionout = "1.1";
750 httpmethodout = geturls_request.get_method();
751 httpurlout = geturls_text.geturl();
752 httpheaderout = geturls_text.info();
753 httpheadersentout = httpheaders;
754 elif(httplibuse=="httplib"):
755 if(urlparts[0]=="http"):
756 httpconn = HTTPConnection(urlparts[1], timeout=timeout);
757 elif(urlparts[0]=="https"):
758 httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
759 else:
760 return False;
761 if(postdata is not None and not isinstance(postdata, dict)):
762 postdata = urlencode(postdata);
763 try:
764 if(httpmethod=="GET"):
765 httpconn.request("GET", urlparts[2], headers=httpheaders);
766 elif(httpmethod=="POST"):
767 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
768 else:
769 httpconn.request("GET", urlparts[2], headers=httpheaders);
770 except socket.timeout:
771 log.info("Error With URL "+httpurl);
772 return False;
773 except socket.gaierror:
774 log.info("Error With URL "+httpurl);
775 return False;
776 except BlockingIOError:
777 log.info("Error With URL "+httpurl);
778 return False;
779 geturls_text = httpconn.getresponse();
780 httpcodeout = geturls_text.status;
781 httpcodereason = geturls_text.reason;
782 if(geturls_text.version=="10"):
783 httpversionout = "1.0";
784 else:
785 httpversionout = "1.1";
786 httpmethodout = geturls_text._method;
787 httpurlout = httpurl;
788 httpheaderout = geturls_text.getheaders();
789 httpheadersentout = httpheaders;
790 elif(httplibuse=="httplib2"):
791 if(urlparts[0]=="http"):
792 httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
793 elif(urlparts[0]=="https"):
794 httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
795 else:
796 return False;
797 if(postdata is not None and not isinstance(postdata, dict)):
798 postdata = urlencode(postdata);
799 try:
800 if(httpmethod=="GET"):
801 httpconn.request("GET", urlparts[2], headers=httpheaders);
802 elif(httpmethod=="POST"):
803 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
804 else:
805 httpconn.request("GET", urlparts[2], headers=httpheaders);
806 except socket.timeout:
807 log.info("Error With URL "+httpurl);
808 return False;
809 except socket.gaierror:
810 log.info("Error With URL "+httpurl);
811 return False;
812 except BlockingIOError:
813 log.info("Error With URL "+httpurl);
814 return False;
815 geturls_text = httpconn.getresponse();
816 httpcodeout = geturls_text.status;
817 httpcodereason = geturls_text.reason;
818 if(geturls_text.version=="10"):
819 httpversionout = "1.0";
820 else:
821 httpversionout = "1.1";
822 httpmethodout = httpmethod;
823 httpurlout = httpurl;
824 httpheaderout = geturls_text.getheaders();
825 httpheadersentout = httpheaders;
826 elif(httplibuse=="urllib3" or httplibuse=="request3"):
827 timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
828 urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
829 try:
830 if(httpmethod=="GET"):
831 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
832 elif(httpmethod=="POST"):
833 geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
834 else:
835 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
836 except urllib3.exceptions.ConnectTimeoutError:
837 log.info("Error With URL "+httpurl);
838 return False;
839 except urllib3.exceptions.ConnectError:
840 log.info("Error With URL "+httpurl);
841 return False;
842 except urllib3.exceptions.MaxRetryError:
843 log.info("Error With URL "+httpurl);
844 return False;
845 except socket.timeout:
846 log.info("Error With URL "+httpurl);
847 return False;
848 except ValueError:
849 log.info("Error With URL "+httpurl);
850 return False;
851 httpcodeout = geturls_text.status;
852 httpcodereason = geturls_text.reason;
853 if(geturls_text.version=="10"):
854 httpversionout = "1.0";
855 else:
856 httpversionout = "1.1";
857 httpmethodout = httpmethod;
858 httpurlout = geturls_text.geturl();
859 httpheaderout = geturls_text.info();
860 httpheadersentout = httpheaders;
861 elif(httplibuse=="requests"):
862 try:
863 reqsession = requests.Session();
864 if(httpmethod=="GET"):
865 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
866 elif(httpmethod=="POST"):
867 geturls_text = reqsession.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
868 else:
869 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
870 except requests.exceptions.ConnectTimeout:
871 log.info("Error With URL "+httpurl);
872 return False;
873 except requests.exceptions.ConnectError:
874 log.info("Error With URL "+httpurl);
875 return False;
876 except socket.timeout:
877 log.info("Error With URL "+httpurl);
878 return False;
879 httpcodeout = geturls_text.status_code;
880 httpcodereason = geturls_text.reason;
881 if(geturls_text.raw.version=="10"):
882 httpversionout = "1.0";
883 else:
884 httpversionout = "1.1";
885 httpmethodout = httpmethod;
886 httpurlout = geturls_text.url;
887 httpheaderout = geturls_text.headers;
888 httpheadersentout = geturls_text.request.headers;
889 elif(httplibuse=="aiohttp"):
890 try:
891 reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize);
892 if(httpmethod=="GET"):
893 geturls_text = reqsession.get(httpurl);
894 elif(httpmethod=="POST"):
895 geturls_text = reqsession.post(httpurl, data=postdata);
896 else:
897 geturls_text = reqsession.get(httpurl);
898 except aiohttp.exceptions.ConnectTimeout:
899 log.info("Error With URL "+httpurl);
900 return False;
901 except aiohttp.exceptions.ConnectError:
902 log.info("Error With URL "+httpurl);
903 return False;
904 except socket.timeout:
905 log.info("Error With URL "+httpurl);
906 return False;
907 httpcodeout = geturls_text.status;
908 httpcodereason = geturls_text.reason;
909 httpversionout = geturls_text.version;
910 httpmethodout = geturls_text.method;
911 httpurlout = geturls_text.url;
912 httpheaderout = geturls_text.headers;
913 httpheadersentout = geturls_text.request_info.headers;
914 elif(httplibuse=="httpx"):
915 try:
916 if(httpmethod=="GET"):
917 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
918 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
919 elif(httpmethod=="POST"):
920 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
921 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
922 else:
923 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
924 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
925 except httpx.ConnectTimeout:
926 log.info("Error With URL "+httpurl);
927 return False;
928 except httpx.ConnectError:
929 log.info("Error With URL "+httpurl);
930 return False;
931 except socket.timeout:
932 log.info("Error With URL "+httpurl);
933 return False;
934 httpcodeout = geturls_text.status_code;
935 try:
936 httpcodereason = geturls_text.reason_phrase;
937 except:
938 httpcodereason = http_status_to_reason(geturls_text.status_code);
939 httpversionout = geturls_text.http_version;
940 httpmethodout = httpmethod;
941 httpurlout = str(geturls_text.url);
942 httpheaderout = geturls_text.headers;
943 httpheadersentout = geturls_text.request.headers;
944 elif(httplibuse=="httpx2"):
945 try:
946 if(httpmethod=="GET"):
947 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
948 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
949 elif(httpmethod=="POST"):
950 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
951 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
952 else:
953 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
954 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
955 except httpx.ConnectTimeout:
956 log.info("Error With URL "+httpurl);
957 return False;
958 except httpx.ConnectError:
959 log.info("Error With URL "+httpurl);
960 return False;
961 except socket.timeout:
962 log.info("Error With URL "+httpurl);
963 return False;
964 httpcodeout = geturls_text.status_code;
965 try:
966 httpcodereason = geturls_text.reason_phrase;
967 except:
968 httpcodereason = http_status_to_reason(geturls_text.status_code);
969 httpversionout = geturls_text.http_version;
970 httpmethodout = httpmethod;
971 httpurlout = str(geturls_text.url);
972 httpheaderout = geturls_text.headers;
973 httpheadersentout = geturls_text.request.headers;
974 elif(httplibuse=="httpcore"):
975 try:
976 if(httpmethod=="GET"):
977 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
978 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
979 elif(httpmethod=="POST"):
980 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
981 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
982 else:
983 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
984 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
985 except httpcore.ConnectTimeout:
986 log.info("Error With URL "+httpurl);
987 return False;
988 except httpcore.ConnectError:
989 log.info("Error With URL "+httpurl);
990 return False;
991 except socket.timeout:
992 log.info("Error With URL "+httpurl);
993 return False;
994 httpcodeout = geturls_text.status;
995 httpcodereason = http_status_to_reason(geturls_text.status);
996 httpversionout = "1.1";
997 httpmethodout = httpmethod;
998 httpurlout = str(httpurl);
999 httpheaderout = geturls_text.headers;
1000 httpheadersentout = httpheaders;
1001 elif(httplibuse=="httpcore2"):
1002 try:
1003 if(httpmethod=="GET"):
1004 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1005 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1006 elif(httpmethod=="POST"):
1007 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1008 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1009 else:
1010 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1011 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1012 except httpcore.ConnectTimeout:
1013 log.info("Error With URL "+httpurl);
1014 return False;
1015 except httpcore.ConnectError:
1016 log.info("Error With URL "+httpurl);
1017 return False;
1018 except socket.timeout:
1019 log.info("Error With URL "+httpurl);
1020 return False;
1021 httpcodeout = geturls_text.status;
1022 httpcodereason = http_status_to_reason(geturls_text.status);
1023 httpversionout = "1.1";
1024 httpmethodout = httpmethod;
1025 httpurlout = str(httpurl);
1026 httpheaderout = geturls_text.headers;
1027 httpheadersentout = httpheaders;
1028 elif(httplibuse=="mechanize"):
1029 geturls_opener = mechanize.Browser();
1030 if(isinstance(httpheaders, dict)):
1031 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1032 time.sleep(sleep);
1033 geturls_opener.addheaders = httpheaders;
1034 geturls_opener.set_cookiejar(httpcookie);
1035 geturls_opener.set_handle_robots(False);
1036 if(postdata is not None and not isinstance(postdata, dict)):
1037 postdata = urlencode(postdata);
1038 try:
1039 if(httpmethod=="GET"):
1040 geturls_text = geturls_opener.open(httpurl);
1041 elif(httpmethod=="POST"):
1042 geturls_text = geturls_opener.open(httpurl, data=postdata);
1043 else:
1044 geturls_text = geturls_opener.open(httpurl);
1045 except mechanize.HTTPError as geturls_text_error:
1046 geturls_text = geturls_text_error;
1047 log.info("Error With URL "+httpurl);
1048 except URLError:
1049 log.info("Error With URL "+httpurl);
1050 return False;
1051 except socket.timeout:
1052 log.info("Error With URL "+httpurl);
1053 return False;
1054 httpcodeout = geturls_text.code;
1055 httpcodereason = geturls_text.msg;
1056 httpversionout = "1.1";
1057 httpmethodout = httpmethod;
1058 httpurlout = geturls_text.geturl();
1059 httpheaderout = geturls_text.info();
1060 reqhead = geturls_opener.request;
1061 httpheadersentout = reqhead.header_items();
1062 elif(httplibuse=="pycurl"):
1063 retrieved_body = BytesIO();
1064 retrieved_headers = BytesIO();
1065 try:
1066 if(httpmethod=="GET"):
1067 geturls_text = pycurl.Curl();
1068 geturls_text.setopt(geturls_text.URL, httpurl);
1069 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1070 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1071 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1072 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1073 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1074 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1075 geturls_text.perform();
1076 elif(httpmethod=="POST"):
1077 geturls_text = pycurl.Curl();
1078 geturls_text.setopt(geturls_text.URL, httpurl);
1079 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1080 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1081 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1082 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1083 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1084 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1085 geturls_text.setopt(geturls_text.POST, True);
1086 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1087 geturls_text.perform();
1088 else:
1089 geturls_text = pycurl.Curl();
1090 geturls_text.setopt(geturls_text.URL, httpurl);
1091 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1092 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1093 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1094 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1095 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1096 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1097 geturls_text.perform();
1098 retrieved_headers.seek(0);
1099 if(sys.version[0]=="2"):
1100 pycurlhead = retrieved_headers.read();
1101 if(sys.version[0]>="3"):
1102 pycurlhead = retrieved_headers.read().decode('UTF-8');
1103 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1104 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1105 retrieved_body.seek(0);
1106 except socket.timeout:
1107 log.info("Error With URL "+httpurl);
1108 return False;
1109 except socket.gaierror:
1110 log.info("Error With URL "+httpurl);
1111 return False;
1112 except ValueError:
1113 log.info("Error With URL "+httpurl);
1114 return False;
1115 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1116 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1117 httpversionout = pyhttpverinfo[0];
1118 httpmethodout = httpmethod;
1119 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1120 httpheaderout = pycurlheadersout;
1121 httpheadersentout = httpheaders;
1122 elif(httplibuse=="pycurl2"):
1123 retrieved_body = BytesIO();
1124 retrieved_headers = BytesIO();
1125 try:
1126 if(httpmethod=="GET"):
1127 geturls_text = pycurl.Curl();
1128 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1129 geturls_text.setopt(geturls_text.URL, httpurl);
1130 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1131 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1132 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1133 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1134 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1135 geturls_text.perform();
1136 elif(httpmethod=="POST"):
1137 geturls_text = pycurl.Curl();
1138 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1139 geturls_text.setopt(geturls_text.URL, httpurl);
1140 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1141 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1142 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1143 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1144 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1145 geturls_text.setopt(geturls_text.POST, True);
1146 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1147 geturls_text.perform();
1148 else:
1149 geturls_text = pycurl.Curl();
1150 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1151 geturls_text.setopt(geturls_text.URL, httpurl);
1152 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1153 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1154 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1155 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1156 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1157 geturls_text.perform();
1158 retrieved_headers.seek(0);
1159 if(sys.version[0]=="2"):
1160 pycurlhead = retrieved_headers.read();
1161 if(sys.version[0]>="3"):
1162 pycurlhead = retrieved_headers.read().decode('UTF-8');
1163 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
1164 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1165 retrieved_body.seek(0);
1166 except socket.timeout:
1167 log.info("Error With URL "+httpurl);
1168 return False;
1169 except socket.gaierror:
1170 log.info("Error With URL "+httpurl);
1171 return False;
1172 except ValueError:
1173 log.info("Error With URL "+httpurl);
1174 return False;
1175 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1176 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1177 httpversionout = pyhttpverinfo[0];
1178 httpmethodout = httpmethod;
1179 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1180 httpheaderout = pycurlheadersout;
1181 httpheadersentout = httpheaders;
1182 elif(httplibuse=="pycurl3"):
1183 retrieved_body = BytesIO();
1184 retrieved_headers = BytesIO();
1185 try:
1186 if(httpmethod=="GET"):
1187 geturls_text = pycurl.Curl();
1188 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1189 geturls_text.setopt(geturls_text.URL, httpurl);
1190 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1191 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1192 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1193 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1194 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1195 geturls_text.perform();
1196 elif(httpmethod=="POST"):
1197 geturls_text = pycurl.Curl();
1198 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1199 geturls_text.setopt(geturls_text.URL, httpurl);
1200 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1201 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1202 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1203 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1204 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1205 geturls_text.setopt(geturls_text.POST, True);
1206 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1207 geturls_text.perform();
1208 else:
1209 geturls_text = pycurl.Curl();
1210 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1211 geturls_text.setopt(geturls_text.URL, httpurl);
1212 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1213 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1214 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1215 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1216 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1217 geturls_text.perform();
1218 retrieved_headers.seek(0);
1219 if(sys.version[0]=="2"):
1220 pycurlhead = retrieved_headers.read();
1221 if(sys.version[0]>="3"):
1222 pycurlhead = retrieved_headers.read().decode('UTF-8');
1223 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1224 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1225 retrieved_body.seek(0);
1226 except socket.timeout:
1227 log.info("Error With URL "+httpurl);
1228 return False;
1229 except socket.gaierror:
1230 log.info("Error With URL "+httpurl);
1231 return False;
1232 except ValueError:
1233 log.info("Error With URL "+httpurl);
1234 return False;
1235 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1236 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1237 httpversionout = pyhttpverinfo[0];
1238 httpmethodout = httpmethod;
1239 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1240 httpheaderout = pycurlheadersout;
1241 httpheadersentout = httpheaders;
1242 elif(httplibuse=="ftp"):
1243 geturls_text = download_file_from_ftp_file(httpurl);
1244 if(not geturls_text):
1245 return False;
1246 downloadsize = None;
1247 if(downloadsize is not None):
1248 downloadsize = int(downloadsize);
1249 if downloadsize is None: downloadsize = 0;
1250 fulldatasize = 0;
1251 prevdownsize = 0;
1252 log.info("Downloading URL "+httpurl);
1253 with BytesIO() as strbuf:
1254 while True:
1255 databytes = geturls_text.read(buffersize);
1256 if not databytes: break;
1257 datasize = len(databytes);
1258 fulldatasize = datasize + fulldatasize;
1259 percentage = "";
1260 if(downloadsize>0):
1261 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1262 downloaddiff = fulldatasize - prevdownsize;
1263 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1264 prevdownsize = fulldatasize;
1265 strbuf.write(databytes);
1266 strbuf.seek(0);
1267 returnval_content = strbuf.read();
1268 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1269 geturls_text.close();
1270 elif(httplibuse=="sftp"):
1271 geturls_text = download_file_from_sftp_file(httpurl);
1272 if(not geturls_text):
1273 return False;
1274 downloadsize = None;
1275 if(downloadsize is not None):
1276 downloadsize = int(downloadsize);
1277 if downloadsize is None: downloadsize = 0;
1278 fulldatasize = 0;
1279 prevdownsize = 0;
1280 log.info("Downloading URL "+httpurl);
1281 with BytesIO() as strbuf:
1282 while True:
1283 databytes = geturls_text.read(buffersize);
1284 if not databytes: break;
1285 datasize = len(databytes);
1286 fulldatasize = datasize + fulldatasize;
1287 percentage = "";
1288 if(downloadsize>0):
1289 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1290 downloaddiff = fulldatasize - prevdownsize;
1291 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1292 prevdownsize = fulldatasize;
1293 strbuf.write(databytes);
1294 strbuf.seek(0);
1295 returnval_content = strbuf.read();
1296 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1297 geturls_text.close();
1298 return returnval;
1299 elif(httplibuse=="pysftp"):
1300 geturls_text = download_file_from_pysftp_file(httpurl);
1301 if(not geturls_text):
1302 return False;
1303 downloadsize = None;
1304 if(downloadsize is not None):
1305 downloadsize = int(downloadsize);
1306 if downloadsize is None: downloadsize = 0;
1307 fulldatasize = 0;
1308 prevdownsize = 0;
1309 log.info("Downloading URL "+httpurl);
1310 with BytesIO() as strbuf:
1311 while True:
1312 databytes = geturls_text.read(buffersize);
1313 if not databytes: break;
1314 datasize = len(databytes);
1315 fulldatasize = datasize + fulldatasize;
1316 percentage = "";
1317 if(downloadsize>0):
1318 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1319 downloaddiff = fulldatasize - prevdownsize;
1320 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1321 prevdownsize = fulldatasize;
1322 strbuf.write(databytes);
1323 strbuf.seek(0);
1324 returnval_content = strbuf.read();
1325 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1326 geturls_text.close();
1327 return returnval;
1328 else:
1329 returnval = False;
1330 if(isinstance(httpheaderout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1331 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1332 if(isinstance(httpheaderout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1333 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
1334 if(sys.version[0]=="2"):
1335 try:
1336 prehttpheaderout = httpheaderout;
1337 httpheaderkeys = httpheaderout.keys();
1338 imax = len(httpheaderkeys);
1339 ic = 0;
1340 httpheaderout = {};
1341 while(ic < imax):
1342 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1343 ic += 1;
1344 except AttributeError:
1345 pass;
1346 httpheaderout = fix_header_names(httpheaderout);
1347 if(isinstance(httpheadersentout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1348 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1349 if(isinstance(httpheadersentout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1350 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
1351 httpheadersentout = fix_header_names(httpheadersentout);
1352 log.info("Downloading URL "+httpurl);
1353 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"):
1354 downloadsize = httpheaderout.get('Content-Length');
1355 if(downloadsize is not None):
1356 downloadsize = int(downloadsize);
1357 if downloadsize is None: downloadsize = 0;
1358 fulldatasize = 0;
1359 prevdownsize = 0;
1360 log.info("Downloading URL "+httpurl);
1361 with BytesIO() as strbuf:
1362 while True:
1363 databytes = geturls_text.read(buffersize);
1364 if not databytes: break;
1365 datasize = len(databytes);
1366 fulldatasize = datasize + fulldatasize;
1367 percentage = "";
1368 if(downloadsize>0):
1369 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1370 downloaddiff = fulldatasize - prevdownsize;
1371 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1372 prevdownsize = fulldatasize;
1373 strbuf.write(databytes);
1374 strbuf.seek(0);
1375 returnval_content = strbuf.read();
1376 geturls_text.close();
1377 if(httpheaderout.get("Content-Encoding")=="gzip"):
1378 try:
1379 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1380 except zlib.error:
1381 pass;
1382 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1383 try:
1384 returnval_content = zlib.decompress(returnval_content);
1385 except zlib.error:
1386 pass;
1387 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1388 try:
1389 returnval_content = brotli.decompress(returnval_content);
1390 except brotli.error:
1391 pass;
1392 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1393 try:
1394 returnval_content = zstandard.decompress(returnval_content);
1395 except zstandard.error:
1396 pass;
1397 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1398 try:
1399 returnval_content = lzma.decompress(returnval_content);
1400 except zstandard.error:
1401 pass;
1402 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1403 try:
1404 returnval_content = bz2.decompress(returnval_content);
1405 except zstandard.error:
1406 pass;
1407 elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1408 downloadsize = httpheaderout.get('Content-Length');
1409 if(downloadsize is not None):
1410 downloadsize = int(downloadsize);
1411 if downloadsize is None: downloadsize = 0;
1412 fulldatasize = 0;
1413 prevdownsize = 0;
1414 log.info("Downloading URL "+httpurl);
1415 with BytesIO() as strbuf:
1416 while True:
1417 databytes = geturls_text.read();
1418 if not databytes: break;
1419 datasize = len(databytes);
1420 fulldatasize = datasize + fulldatasize;
1421 percentage = "";
1422 if(downloadsize>0):
1423 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1424 downloaddiff = fulldatasize - prevdownsize;
1425 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1426 prevdownsize = fulldatasize;
1427 strbuf.write(databytes);
1428 break;
1429 strbuf.seek(0);
1430 returnval_content = strbuf.read();
1431 geturls_text.close();
1432 if(httpheaderout.get("Content-Encoding")=="gzip"):
1433 try:
1434 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1435 except zlib.error:
1436 pass;
1437 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1438 try:
1439 returnval_content = zlib.decompress(returnval_content);
1440 except zlib.error:
1441 pass;
1442 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1443 try:
1444 returnval_content = brotli.decompress(returnval_content);
1445 except brotli.error:
1446 pass;
1447 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1448 try:
1449 returnval_content = zstandard.decompress(returnval_content);
1450 except zstandard.error:
1451 pass;
1452 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1453 try:
1454 returnval_content = lzma.decompress(returnval_content);
1455 except zstandard.error:
1456 pass;
1457 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1458 try:
1459 returnval_content = bz2.decompress(returnval_content);
1460 except zstandard.error:
1461 pass;
1462 elif(httplibuse=="requests"):
1463 log.info("Downloading URL "+httpurl);
1464 downloadsize = httpheaderout.get('Content-Length');
1465 if(downloadsize is not None):
1466 downloadsize = int(downloadsize);
1467 if downloadsize is None: downloadsize = 0;
1468 fulldatasize = 0;
1469 prevdownsize = 0;
1470 log.info("Downloading URL "+httpurl);
1471 with BytesIO() as strbuf:
1472 while True:
1473 databytes = geturls_text.raw.read(buffersize);
1474 if not databytes: break;
1475 datasize = len(databytes);
1476 fulldatasize = datasize + fulldatasize;
1477 percentage = "";
1478 if(downloadsize>0):
1479 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1480 downloaddiff = fulldatasize - prevdownsize;
1481 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1482 prevdownsize = fulldatasize;
1483 strbuf.write(databytes);
1484 strbuf.seek(0);
1485 returnval_content = strbuf.read();
1486 geturls_text.close();
1487 if(httpheaderout.get("Content-Encoding")=="gzip"):
1488 try:
1489 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1490 except zlib.error:
1491 pass;
1492 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1493 try:
1494 returnval_content = zlib.decompress(returnval_content);
1495 except zlib.error:
1496 pass;
1497 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1498 try:
1499 returnval_content = brotli.decompress(returnval_content);
1500 except brotli.error:
1501 pass;
1502 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1503 try:
1504 returnval_content = zstandard.decompress(returnval_content);
1505 except zstandard.error:
1506 pass;
1507 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1508 try:
1509 returnval_content = lzma.decompress(returnval_content);
1510 except zstandard.error:
1511 pass;
1512 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1513 try:
1514 returnval_content = bz2.decompress(returnval_content);
1515 except zstandard.error:
1516 pass;
1517 elif(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
1518 log.info("Downloading URL "+httpurl);
1519 downloadsize = httpheaderout.get('Content-Length');
1520 if(downloadsize is not None):
1521 downloadsize = int(downloadsize);
1522 if downloadsize is None: downloadsize = 0;
1523 fulldatasize = 0;
1524 prevdownsize = 0;
1525 log.info("Downloading URL "+httpurl);
1526 with BytesIO() as strbuf:
1527 while True:
1528 databytes = retrieved_body.read(buffersize);
1529 if not databytes: break;
1530 datasize = len(databytes);
1531 fulldatasize = datasize + fulldatasize;
1532 percentage = "";
1533 if(downloadsize>0):
1534 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1535 downloaddiff = fulldatasize - prevdownsize;
1536 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1537 prevdownsize = fulldatasize;
1538 strbuf.write(databytes);
1539 strbuf.seek(0);
1540 returnval_content = strbuf.read();
1541 geturls_text.close();
1542 if(httpheaderout.get("Content-Encoding")=="gzip"):
1543 try:
1544 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1545 except zlib.error:
1546 pass;
1547 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1548 try:
1549 returnval_content = zlib.decompress(returnval_content);
1550 except zlib.error:
1551 pass;
1552 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1553 try:
1554 returnval_content = brotli.decompress(returnval_content);
1555 except brotli.error:
1556 pass;
1557 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1558 try:
1559 returnval_content = zstandard.decompress(returnval_content);
1560 except zstandard.error:
1561 pass;
1562 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1563 try:
1564 returnval_content = lzma.decompress(returnval_content);
1565 except zstandard.error:
1566 pass;
1567 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1568 try:
1569 returnval_content = bz2.decompress(returnval_content);
1570 except zstandard.error:
1571 pass;
1572 elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1573 pass;
1574 else:
1575 returnval = False;
1576 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse};
1577 return returnval;
1578
1579 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1580 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1581 exec_time_start = time.time();
1582 myhash = hashlib.new("sha1");
1583 if(sys.version[0]=="2"):
1584 myhash.update(httpurl);
1585 myhash.update(str(buffersize));
1586 myhash.update(str(exec_time_start));
1587 if(sys.version[0]>="3"):
1588 myhash.update(httpurl.encode('utf-8'));
1589 myhash.update(str(buffersize).encode('utf-8'));
1590 myhash.update(str(exec_time_start).encode('utf-8'));
1591 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1592 if(sleep<0):
1593 sleep = geturls_download_sleep;
1594 if(timeout<=0):
1595 timeout = 10;
1596 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1597 httplibuse = "urllib";
1598 if(httplibuse=="httplib1"):
1599 httplibuse = "httplib";
1600 if(not haverequests and httplibuse=="requests"):
1601 httplibuse = "urllib";
1602 if(not haveaiohttp and httplibuse=="aiohttp"):
1603 httplibuse = "urllib";
1604 if(not havehttpx and httplibuse=="httpx"):
1605 httplibuse = "urllib";
1606 if(not havehttpx and httplibuse=="httpx2"):
1607 httplibuse = "urllib";
1608 if(not havehttpcore and httplibuse=="httpcore"):
1609 httplibuse = "urllib";
1610 if(not havehttpcore and httplibuse=="httpcore2"):
1611 httplibuse = "urllib";
1612 if(not havemechanize and httplibuse=="mechanize"):
1613 httplibuse = "urllib";
1614 if(not havepycurl and httplibuse=="pycurl"):
1615 httplibuse = "urllib";
1616 if(not havepycurl and httplibuse=="pycurl2"):
1617 httplibuse = "urllib";
1618 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1619 httplibuse = "pycurl";
1620 if(not havepycurl and httplibuse=="pycurl3"):
1621 httplibuse = "urllib";
1622 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1623 httplibuse = "pycurl2";
1624 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1625 httplibuse = "pycurl";
1626 if(not havehttplib2 and httplibuse=="httplib2"):
1627 httplibuse = "httplib";
1628 if(not haveparamiko and httplibuse=="sftp"):
1629 httplibuse = "ftp";
1630 if(not haveparamiko and httplibuse=="pysftp"):
1631 httplibuse = "ftp";
1632 pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1633 if(not pretmpfilename):
1634 return False;
1635 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1636 tmpfilename = f.name;
1637 try:
1638 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1639 except AttributeError:
1640 try:
1641 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1642 except ValueError:
1643 pass;
1644 except ValueError:
1645 pass;
1646 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1647 f.write(pretmpfilename.get('Content'));
1648 f.close();
1649 exec_time_end = time.time();
1650 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1651 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1652 return returnval;
1653
1654 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1655 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1656 if(sleep<0):
1657 sleep = geturls_download_sleep;
1658 if(timeout<=0):
1659 timeout = 10;
1660 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1661 httplibuse = "urllib";
1662 if(httplibuse=="httplib1"):
1663 httplibuse = "httplib";
1664 if(not haverequests and httplibuse=="requests"):
1665 httplibuse = "urllib";
1666 if(not haveaiohttp and httplibuse=="aiohttp"):
1667 httplibuse = "urllib";
1668 if(not havehttpx and httplibuse=="httpx"):
1669 httplibuse = "urllib";
1670 if(not havehttpx and httplibuse=="httpx2"):
1671 httplibuse = "urllib";
1672 if(not havehttpcore and httplibuse=="httpcore"):
1673 httplibuse = "urllib";
1674 if(not havehttpcore and httplibuse=="httpcore2"):
1675 httplibuse = "urllib";
1676 if(not havemechanize and httplibuse=="mechanize"):
1677 httplibuse = "urllib";
1678 if(not havepycurl and httplibuse=="pycurl"):
1679 httplibuse = "urllib";
1680 if(not havepycurl and httplibuse=="pycurl2"):
1681 httplibuse = "urllib";
1682 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1683 httplibuse = "pycurl";
1684 if(not havepycurl and httplibuse=="pycurl3"):
1685 httplibuse = "urllib";
1686 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1687 httplibuse = "pycurl2";
1688 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1689 httplibuse = "pycurl";
1690 if(not havehttplib2 and httplibuse=="httplib2"):
1691 httplibuse = "httplib";
1692 if(not haveparamiko and httplibuse=="sftp"):
1693 httplibuse = "ftp";
1694 if(not havepysftp and httplibuse=="pysftp"):
1695 httplibuse = "ftp";
1696 if(not outfile=="-"):
1697 outpath = outpath.rstrip(os.path.sep);
1698 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1699 if(not os.path.exists(outpath)):
1700 os.makedirs(outpath);
1701 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1702 return False;
1703 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1704 return False;
1705 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1706 if(not pretmpfilename):
1707 return False;
1708 tmpfilename = pretmpfilename.get('Filename');
1709 downloadsize = int(os.path.getsize(tmpfilename));
1710 fulldatasize = 0;
1711 log.info("Moving file "+tmpfilename+" to "+filepath);
1712 exec_time_start = time.time();
1713 shutil.move(tmpfilename, filepath);
1714 try:
1715 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1716 except AttributeError:
1717 try:
1718 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1719 except ValueError:
1720 pass;
1721 except ValueError:
1722 pass;
1723 exec_time_end = time.time();
1724 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1725 if(os.path.exists(tmpfilename)):
1726 os.remove(tmpfilename);
1727 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1728 if(outfile=="-"):
1729 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1730 tmpfilename = pretmpfilename.get('Filename');
1731 downloadsize = int(os.path.getsize(tmpfilename));
1732 fulldatasize = 0;
1733 prevdownsize = 0;
1734 exec_time_start = time.time();
1735 with open(tmpfilename, 'rb') as ft:
1736 f = BytesIO();
1737 while True:
1738 databytes = ft.read(buffersize[1]);
1739 if not databytes: break;
1740 datasize = len(databytes);
1741 fulldatasize = datasize + fulldatasize;
1742 percentage = "";
1743 if(downloadsize>0):
1744 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1745 downloaddiff = fulldatasize - prevdownsize;
1746 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1747 prevdownsize = fulldatasize;
1748 f.write(databytes);
1749 f.seek(0);
1750 fdata = f.getvalue();
1751 f.close();
1752 ft.close();
1753 os.remove(tmpfilename);
1754 exec_time_end = time.time();
1755 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1756 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse};
1757 return returnval;
1758
1759 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1760 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1761 return returnval;
1762
1763 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1764 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1765 return returnval;
1766
1767 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1768 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout);
1769 return returnval;
1770
1771 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1772 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout);
1773 return returnval;
1774
1775 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1776 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout);
1777 return returnval;
1778
1779 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1780 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout);
1781 return returnval;
1782
1783 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1784 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout);
1785 return returnval;
1786
1787 def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1788 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout);
1789 return returnval;
1790
1791 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1792 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout);
1793 return returnval;
1794
1795 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1796 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout);
1797 return returnval;
1798
1799 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1800 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout);
1801 return returnval;
1802
1803 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1804 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout);
1805 return returnval;
1806
1807 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1808 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout);
1809 return returnval;
1810
1811 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1812 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout);
1813 return returnval;
1814
1815 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1816 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout);
1817 return returnval;
1818
1819 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1820 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout);
1821 return returnval;
1822
1823 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1824 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout);
1825 return returnval;
1826
1827 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1828 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout);
1829 return returnval;
1830
1831 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1832 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout);
1833 return returnval;
1834
1835 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1836 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1837 return returnval;
1838
1839 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1840 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1841 return returnval;
1842
1843 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1844 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout);
1845 return returnval;
1846
1847 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1848 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout);
1849 return returnval;
1850
1851 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1852 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout);
1853 return returnval;
1854
1855 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1856 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout);
1857 return returnval;
1858
1859 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1860 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout);
1861 return returnval;
1862
1863 def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1864 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout);
1865 return returnval;
1866
1867 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1868 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout);
1869 return returnval;
1870
1871 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1872 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout);
1873 return returnval;
1874
1875 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1876 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout);
1877 return returnval;
1878
1879 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1880 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout);
1881 return returnval;
1882
1883 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1884 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout);
1885 return returnval;
1886
1887 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1888 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout);
1889 return returnval;
1890
1891 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1892 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout);
1893 return returnval;
1894
1895 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1896 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout);
1897 return returnval;
1898
1899 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1900 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout);
1901 return returnval;
1902
1903 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1904 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout);
1905 return returnval;
1906
1907 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1908 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout);
1909 return returnval;
1910
1911 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1912 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1913 return returnval;
1914
1915 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1916 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout);
1917 return returnval;
1918
1919 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1920 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1921 return returnval;
1922
1923 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1924 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout);
1925 return returnval;
1926
1927 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1928 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout);
1929 return returnval;
1930
1931 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1932 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout);
1933 return returnval;
1934
1935 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1936 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout);
1937 return returnval;
1938
1939 def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1940 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout);
1941 return returnval;
1942
1943 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1944 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout);
1945 return returnval;
1946
1947 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1948 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout);
1949 return returnval;
1950
1951 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1952 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout);
1953 return returnval;
1954
1955 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1956 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout);
1957 return returnval;
1958
1959 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1960 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout);
1961 return returnval;
1962
1963 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1964 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout);
1965 return returnval;
1966
1967 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1968 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout);
1969 return returnval;
1970
1971 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1972 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout);
1973 return returnval;
1974
1975 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1976 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout);
1977 return returnval;
1978
1979 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1980 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout);
1981 return returnval;
1982
1983 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1984 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout);
1985 return returnval;
1986
1987 def download_file_from_ftp_file(url):
1988 urlparts = urlparse.urlparse(url);
1989 file_name = os.path.basename(urlparts.path);
1990 file_dir = os.path.dirname(urlparts.path);
1991 if(urlparts.username is not None):
1992 ftp_username = urlparts.username;
1993 else:
1994 ftp_username = "anonymous";
1995 if(urlparts.password is not None):
1996 ftp_password = urlparts.password;
1997 elif(urlparts.password is None and urlparts.username=="anonymous"):
1998 ftp_password = "anonymous";
1999 else:
2000 ftp_password = "";
2001 if(urlparts.scheme=="ftp"):
2002 ftp = FTP();
2003 elif(urlparts.scheme=="ftps"):
2004 ftp = FTP_TLS();
2005 else:
2006 return False;
2007 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2008 return False;
2009 ftp_port = urlparts.port;
2010 if(urlparts.port is None):
2011 ftp_port = 21;
2012 try:
2013 ftp.connect(urlparts.hostname, ftp_port);
2014 except socket.gaierror:
2015 log.info("Error With URL "+httpurl);
2016 return False;
2017 except socket.timeout:
2018 log.info("Error With URL "+httpurl);
2019 return False;
2020 ftp.login(urlparts.username, urlparts.password);
2021 if(urlparts.scheme=="ftps"):
2022 ftp.prot_p();
2023 ftpfile = BytesIO();
2024 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2025 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2026 ftp.close();
2027 ftpfile.seek(0, 0);
2028 return ftpfile;
2029
2030 def download_file_from_ftp_string(url):
2031 ftpfile = download_file_from_ftp_file(url);
2032 return ftpfile.read();
2033
2034 def upload_file_to_ftp_file(ftpfile, url):
2035 urlparts = urlparse.urlparse(url);
2036 file_name = os.path.basename(urlparts.path);
2037 file_dir = os.path.dirname(urlparts.path);
2038 if(urlparts.username is not None):
2039 ftp_username = urlparts.username;
2040 else:
2041 ftp_username = "anonymous";
2042 if(urlparts.password is not None):
2043 ftp_password = urlparts.password;
2044 elif(urlparts.password is None and urlparts.username=="anonymous"):
2045 ftp_password = "anonymous";
2046 else:
2047 ftp_password = "";
2048 if(urlparts.scheme=="ftp"):
2049 ftp = FTP();
2050 elif(urlparts.scheme=="ftps"):
2051 ftp = FTP_TLS();
2052 else:
2053 return False;
2054 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2055 return False;
2056 ftp_port = urlparts.port;
2057 if(urlparts.port is None):
2058 ftp_port = 21;
2059 try:
2060 ftp.connect(urlparts.hostname, ftp_port);
2061 except socket.gaierror:
2062 log.info("Error With URL "+httpurl);
2063 return False;
2064 except socket.timeout:
2065 log.info("Error With URL "+httpurl);
2066 return False;
2067 ftp.login(urlparts.username, urlparts.password);
2068 if(urlparts.scheme=="ftps"):
2069 ftp.prot_p();
2070 ftp.storbinary("STOR "+urlparts.path, ftpfile);
2071 ftp.close();
2072 ftpfile.seek(0, 0);
2073 return ftpfile;
2074
2075 def upload_file_to_ftp_string(ftpstring, url):
2076 ftpfileo = BytesIO(ftpstring);
2077 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2078 ftpfileo.close();
2079 return ftpfile;
2080
2081 if(haveparamiko):
2082 def download_file_from_sftp_file(url):
2083 urlparts = urlparse.urlparse(url);
2084 file_name = os.path.basename(urlparts.path);
2085 file_dir = os.path.dirname(urlparts.path);
2086 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2087 return False;
2088 sftp_port = urlparts.port;
2089 if(urlparts.port is None):
2090 sftp_port = 22;
2091 else:
2092 sftp_port = urlparts.port;
2093 if(urlparts.username is not None):
2094 sftp_username = urlparts.username;
2095 else:
2096 sftp_username = "anonymous";
2097 if(urlparts.password is not None):
2098 sftp_password = urlparts.password;
2099 elif(urlparts.password is None and urlparts.username=="anonymous"):
2100 sftp_password = "anonymous";
2101 else:
2102 sftp_password = "";
2103 if(urlparts.scheme!="sftp"):
2104 return False;
2105 ssh = paramiko.SSHClient();
2106 ssh.load_system_host_keys();
2107 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2108 try:
2109 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2110 except paramiko.ssh_exception.SSHException:
2111 return False;
2112 except socket.gaierror:
2113 log.info("Error With URL "+httpurl);
2114 return False;
2115 except socket.timeout:
2116 log.info("Error With URL "+httpurl);
2117 return False;
2118 sftp = ssh.open_sftp();
2119 sftpfile = BytesIO();
2120 sftp.getfo(urlparts.path, sftpfile);
2121 sftp.close();
2122 ssh.close();
2123 sftpfile.seek(0, 0);
2124 return sftpfile;
2125 else:
2126 def download_file_from_sftp_file(url):
2127 return False;
2128
2129 if(haveparamiko):
2130 def download_file_from_sftp_string(url):
2131 sftpfile = download_file_from_sftp_file(url);
2132 return sftpfile.read();
2133 else:
2134 def download_file_from_ftp_string(url):
2135 return False;
2136
2137 if(haveparamiko):
2138 def upload_file_to_sftp_file(sftpfile, url):
2139 urlparts = urlparse.urlparse(url);
2140 file_name = os.path.basename(urlparts.path);
2141 file_dir = os.path.dirname(urlparts.path);
2142 sftp_port = urlparts.port;
2143 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2144 return False;
2145 if(urlparts.port is None):
2146 sftp_port = 22;
2147 else:
2148 sftp_port = urlparts.port;
2149 if(urlparts.username is not None):
2150 sftp_username = urlparts.username;
2151 else:
2152 sftp_username = "anonymous";
2153 if(urlparts.password is not None):
2154 sftp_password = urlparts.password;
2155 elif(urlparts.password is None and urlparts.username=="anonymous"):
2156 sftp_password = "anonymous";
2157 else:
2158 sftp_password = "";
2159 if(urlparts.scheme!="sftp"):
2160 return False;
2161 ssh = paramiko.SSHClient();
2162 ssh.load_system_host_keys();
2163 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2164 try:
2165 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2166 except paramiko.ssh_exception.SSHException:
2167 return False;
2168 except socket.gaierror:
2169 log.info("Error With URL "+httpurl);
2170 return False;
2171 except socket.timeout:
2172 log.info("Error With URL "+httpurl);
2173 return False;
2174 sftp = ssh.open_sftp();
2175 sftp.putfo(sftpfile, urlparts.path);
2176 sftp.close();
2177 ssh.close();
2178 sftpfile.seek(0, 0);
2179 return sftpfile;
2180 else:
2181 def upload_file_to_sftp_file(sftpfile, url):
2182 return False;
2183
2184 if(haveparamiko):
2185 def upload_file_to_sftp_string(sftpstring, url):
2186 sftpfileo = BytesIO(sftpstring);
2187 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2188 sftpfileo.close();
2189 return sftpfile;
2190 else:
2191 def upload_file_to_sftp_string(url):
2192 return False;
2193
2194
2195 if(havepysftp):
2196 def download_file_from_pysftp_file(url):
2197 urlparts = urlparse.urlparse(url);
2198 file_name = os.path.basename(urlparts.path);
2199 file_dir = os.path.dirname(urlparts.path);
2200 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2201 return False;
2202 sftp_port = urlparts.port;
2203 if(urlparts.port is None):
2204 sftp_port = 22;
2205 else:
2206 sftp_port = urlparts.port;
2207 if(urlparts.username is not None):
2208 sftp_username = urlparts.username;
2209 else:
2210 sftp_username = "anonymous";
2211 if(urlparts.password is not None):
2212 sftp_password = urlparts.password;
2213 elif(urlparts.password is None and urlparts.username=="anonymous"):
2214 sftp_password = "anonymous";
2215 else:
2216 sftp_password = "";
2217 if(urlparts.scheme!="sftp"):
2218 return False;
2219 try:
2220 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2221 except paramiko.ssh_exception.SSHException:
2222 return False;
2223 except socket.gaierror:
2224 log.info("Error With URL "+httpurl);
2225 return False;
2226 except socket.timeout:
2227 log.info("Error With URL "+httpurl);
2228 return False;
2229 sftp = ssh.open_sftp();
2230 sftpfile = BytesIO();
2231 sftp.getfo(urlparts.path, sftpfile);
2232 sftp.close();
2233 ssh.close();
2234 sftpfile.seek(0, 0);
2235 return sftpfile;
2236 else:
2237 def download_file_from_pysftp_file(url):
2238 return False;
2239
2240 if(havepysftp):
2241 def download_file_from_pysftp_string(url):
2242 sftpfile = download_file_from_pysftp_file(url);
2243 return sftpfile.read();
2244 else:
2245 def download_file_from_ftp_string(url):
2246 return False;
2247
2248 if(havepysftp):
2249 def upload_file_to_pysftp_file(sftpfile, url):
2250 urlparts = urlparse.urlparse(url);
2251 file_name = os.path.basename(urlparts.path);
2252 file_dir = os.path.dirname(urlparts.path);
2253 sftp_port = urlparts.port;
2254 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2255 return False;
2256 if(urlparts.port is None):
2257 sftp_port = 22;
2258 else:
2259 sftp_port = urlparts.port;
2260 if(urlparts.username is not None):
2261 sftp_username = urlparts.username;
2262 else:
2263 sftp_username = "anonymous";
2264 if(urlparts.password is not None):
2265 sftp_password = urlparts.password;
2266 elif(urlparts.password is None and urlparts.username=="anonymous"):
2267 sftp_password = "anonymous";
2268 else:
2269 sftp_password = "";
2270 if(urlparts.scheme!="sftp"):
2271 return False;
2272 try:
2273 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2274 except paramiko.ssh_exception.SSHException:
2275 return False;
2276 except socket.gaierror:
2277 log.info("Error With URL "+httpurl);
2278 return False;
2279 except socket.timeout:
2280 log.info("Error With URL "+httpurl);
2281 return False;
2282 sftp = ssh.open_sftp();
2283 sftp.putfo(sftpfile, urlparts.path);
2284 sftp.close();
2285 ssh.close();
2286 sftpfile.seek(0, 0);
2287 return sftpfile;
2288 else:
2289 def upload_file_to_pysftp_file(sftpfile, url):
2290 return False;
2291
2292 if(havepysftp):
2293 def upload_file_to_pysftp_string(sftpstring, url):
2294 sftpfileo = BytesIO(sftpstring);
2295 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2296 sftpfileo.close();
2297 return sftpfile;
2298 else:
2299 def upload_file_to_pysftp_string(url):
2300 return False;
python libary/module to download files.