?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Delete pycatfile.py
[PyWWW-Get.git] / pywwwget.py
blobf9c8fcfc1f5a79c308af4fba34cc68ae8efc595a
1 #!/usr/bin/env python
2
3 '''
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
11
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
14
15 $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
16 '''
17
18 from __future__ import division, absolute_import, print_function;
19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, bz2, time, argparse, subprocess, socket, email.utils, datetime, time;
20 import logging as log;
21 from ftplib import FTP, FTP_TLS;
22 from base64 import b64encode;
23 try:
24 from cgi import parse_qsl;
25 except ImportError:
26 try:
27 from urlparse import parse_qsl;
28 except ImportError:
29 from urllib.parse import parse_qsl;
30 except (DeprecationWarning, TypeError):
31 try:
32 from urlparse import parse_qsl;
33 except ImportError:
34 from urllib.parse import parse_qsl;
35 haverequests = False;
36 try:
37 import requests;
38 haverequests = True;
39 except ImportError:
40 haverequests = False;
41 havemechanize = False;
42 try:
43 import mechanize;
44 havemechanize = True;
45 except ImportError:
46 havemechanize = False;
47 havepycurl = False;
48 try:
49 import pycurl;
50 havepycurl = True;
51 except ImportError:
52 havepycurl = False;
53 haveparamiko = False;
54 try:
55 import paramiko;
56 haveparamiko = True;
57 except ImportError:
58 haveparamiko = False;
59 havepysftp = False;
60 try:
61 import pysftp;
62 havepysftp = True;
63 except ImportError:
64 havepysftp = False;
65 haveurllib3 = False;
66 try:
67 import urllib3;
68 haveurllib3 = True;
69 except ImportError:
70 haveurllib3 = False;
71 havehttplib2 = False;
72 try:
73 from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
74 havehttplib2 = True;
75 except ImportError:
76 havehttplib2 = False;
77 havehttpx = False;
78 try:
79 import httpx;
80 havehttpx = True;
81 except ImportError:
82 havehttpx = False;
83 havehttpcore = False;
84 try:
85 import httpcore;
86 havehttpcore = True;
87 except ImportError:
88 havehttpcore = False;
89 haveaiohttp = False;
90 try:
91 import aiohttp;
92 haveaiohttp = True;
93 except ImportError:
94 haveaiohttp = False;
95 havebrotli = False;
96 try:
97 import brotli;
98 havebrotli = True;
99 except ImportError:
100 havebrotli = False;
101 havezstd = False;
102 try:
103 import zstandard;
104 havezstd = True;
105 except ImportError:
106 havezstd = False;
107 havelzma = False;
108 try:
109 import lzma;
110 havelzma = True;
111 except ImportError:
112 havelzma = False;
113 if(sys.version[0]=="2"):
114 try:
115 from io import StringIO, BytesIO;
116 except ImportError:
117 try:
118 from cStringIO import StringIO;
119 from cStringIO import StringIO as BytesIO;
120 except ImportError:
121 from StringIO import StringIO;
122 from StringIO import StringIO as BytesIO;
123 # From http://python-future.org/compatible_idioms.html
124 from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
125 from urllib import urlencode;
126 from urllib import urlopen as urlopenalt;
127 from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
128 import urlparse, cookielib;
129 from httplib import HTTPConnection, HTTPSConnection;
130 if(sys.version[0]>="3"):
131 from io import StringIO, BytesIO;
132 # From http://python-future.org/compatible_idioms.html
133 from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
134 from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
135 from urllib.error import HTTPError, URLError;
136 import urllib.parse as urlparse;
137 import http.cookiejar as cookielib;
138 from http.client import HTTPConnection, HTTPSConnection;
139
140 __program_name__ = "PyWWW-Get";
141 __program_alt_name__ = "PyWWWGet";
142 __program_small_name__ = "wwwget";
143 __project__ = __program_name__;
144 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
145 __version_info__ = (2, 0, 2, "RC 1", 1);
146 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
147 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
148 __revision__ = __version_info__[3];
149 __revision_id__ = "$Id$";
150 if(__version_info__[4] is not None):
151 __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
152 if(__version_info__[4] is None):
153 __version_date_plusrc__ = __version_date__;
154 if(__version_info__[3] is not None):
155 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
156 if(__version_info__[3] is None):
157 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
158
159 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
160 tmpfilesuffix = "-";
161 pytempdir = tempfile.gettempdir();
162
163 PyBitness = platform.architecture();
164 if(PyBitness=="32bit" or PyBitness=="32"):
165 PyBitness = "32";
166 elif(PyBitness=="64bit" or PyBitness=="64"):
167 PyBitness = "64";
168 else:
169 PyBitness = "32";
170
171 compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2'];
172 if(havebrotli):
173 compression_supported_list.append('br');
174 if(havezstd):
175 compression_supported_list.append('zstd');
176 if(havelzma):
177 compression_supported_list.append('lzma');
178 compression_supported_list.append('xz');
179 compression_supported = ', '.join(compression_supported_list);
180
181 geturls_cj = cookielib.CookieJar();
182 windowsNT4_ua_string = "Windows NT 4.0";
183 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
184 windows2k_ua_string = "Windows NT 5.0";
185 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
186 windowsXP_ua_string = "Windows NT 5.1";
187 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
188 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
189 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
190 windows7_ua_string = "Windows NT 6.1; Win64; x64";
191 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
192 windows8_ua_string = "Windows NT 6.2; Win64; x64";
193 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
194 windows81_ua_string = "Windows NT 6.3; Win64; x64";
195 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
196 windows10_ua_string = "Windows NT 10.0; Win64; x64";
197 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
198 windows11_ua_string = "Windows NT 11.0; Win64; x64";
199 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
200 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
201 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
202 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
203 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
204 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
205 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
206 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
207 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
208 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
209 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
210 if(platform.python_implementation()!=""):
211 py_implementation = platform.python_implementation();
212 if(platform.python_implementation()==""):
213 py_implementation = "Python";
214 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
215 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
216 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
217 geturls_ua = geturls_ua_firefox_windows7;
218 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
219 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
220 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
221 geturls_headers_chrome_windows7.update(windows7_ua_addon);
222 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
223 geturls_headers_chromium_windows7.update(windows7_ua_addon);
224 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
226 geturls_headers_opera_windows7.update(windows7_ua_addon);
227 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
228 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
229 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
230 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
231 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
232 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
233 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
234 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
235 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
236 geturls_headers = geturls_headers_firefox_windows7;
237 geturls_download_sleep = 0;
238
239 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
240 if(outtype=="print" and dbgenable):
241 print(dbgtxt);
242 return True;
243 elif(outtype=="log" and dbgenable):
244 logging.info(dbgtxt);
245 return True;
246 elif(outtype=="warning" and dbgenable):
247 logging.warning(dbgtxt);
248 return True;
249 elif(outtype=="error" and dbgenable):
250 logging.error(dbgtxt);
251 return True;
252 elif(outtype=="critical" and dbgenable):
253 logging.critical(dbgtxt);
254 return True;
255 elif(outtype=="exception" and dbgenable):
256 logging.exception(dbgtxt);
257 return True;
258 elif(outtype=="logalt" and dbgenable):
259 logging.log(dgblevel, dbgtxt);
260 return True;
261 elif(outtype=="debug" and dbgenable):
262 logging.debug(dbgtxt);
263 return True;
264 elif(not dbgenable):
265 return True;
266 else:
267 return False;
268 return False;
269
270 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
271 dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
272 if(not dbgout):
273 return False;
274 return dbgtxt;
275
276 def add_url_param(url, **params):
277 n=3;
278 parts = list(urlparse.urlsplit(url));
279 d = dict(parse_qsl(parts[n])); # use cgi.parse_qs for list values
280 d.update(params);
281 parts[n]=urlencode(d);
282 return urlparse.urlunsplit(parts);
283
284 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
285 def which_exec(execfile):
286 for path in os.environ["PATH"].split(":"):
287 if os.path.exists(path + "/" + execfile):
288 return path + "/" + execfile;
289
290 def listize(varlist):
291 il = 0;
292 ix = len(varlist);
293 ilx = 1;
294 newlistreg = {};
295 newlistrev = {};
296 newlistfull = {};
297 while(il < ix):
298 newlistreg.update({ilx: varlist[il]});
299 newlistrev.update({varlist[il]: ilx});
300 ilx = ilx + 1;
301 il = il + 1;
302 newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
303 return newlistfull;
304
305 def twolistize(varlist):
306 il = 0;
307 ix = len(varlist);
308 ilx = 1;
309 newlistnamereg = {};
310 newlistnamerev = {};
311 newlistdescreg = {};
312 newlistdescrev = {};
313 newlistfull = {};
314 while(il < ix):
315 newlistnamereg.update({ilx: varlist[il][0].strip()});
316 newlistnamerev.update({varlist[il][0].strip(): ilx});
317 newlistdescreg.update({ilx: varlist[il][1].strip()});
318 newlistdescrev.update({varlist[il][1].strip(): ilx});
319 ilx = ilx + 1;
320 il = il + 1;
321 newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
322 newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
323 newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
324 return newlistfull;
325
326 def arglistize(proexec, *varlist):
327 il = 0;
328 ix = len(varlist);
329 ilx = 1;
330 newarglist = [proexec];
331 while(il < ix):
332 if varlist[il][0] is not None:
333 newarglist.append(varlist[il][0]);
334 if varlist[il][1] is not None:
335 newarglist.append(varlist[il][1]);
336 il = il + 1;
337 return newarglist;
338
339 def fix_header_names(header_dict):
340 if(sys.version[0]=="2"):
341 header_dict = {k.title(): v for k, v in header_dict.iteritems()};
342 if(sys.version[0]>="3"):
343 header_dict = {k.title(): v for k, v in header_dict.items()};
344 return header_dict;
345
346 # hms_string by ArcGIS Python Recipes
347 # https://arcpy.wordpress.com/2012/04/20/146/
348 def hms_string(sec_elapsed):
349 h = int(sec_elapsed / (60 * 60));
350 m = int((sec_elapsed % (60 * 60)) / 60);
351 s = sec_elapsed % 60.0;
352 return "{}:{:>02}:{:>05.2f}".format(h, m, s);
353
354 # get_readable_size by Lipis
355 # http://stackoverflow.com/posts/14998888/revisions
356 def get_readable_size(bytes, precision=1, unit="IEC"):
357 unit = unit.upper();
358 if(unit!="IEC" and unit!="SI"):
359 unit = "IEC";
360 if(unit=="IEC"):
361 units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
362 unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
363 unitsize = 1024.0;
364 if(unit=="SI"):
365 units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
366 unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
367 unitsize = 1000.0;
368 return_val = {};
369 orgbytes = bytes;
370 for unit in units:
371 if abs(bytes) < unitsize:
372 strformat = "%3."+str(precision)+"f%s";
373 pre_return_val = (strformat % (bytes, unit));
374 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
375 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
376 alt_return_val = pre_return_val.split();
377 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
378 return return_val;
379 bytes /= unitsize;
380 strformat = "%."+str(precision)+"f%s";
381 pre_return_val = (strformat % (bytes, "YiB"));
382 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
383 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
384 alt_return_val = pre_return_val.split();
385 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
386 return return_val;
387
388 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
389 unit = unit.upper();
390 usehashtypes = usehashtypes.lower();
391 getfilesize = os.path.getsize(infile);
392 return_val = get_readable_size(getfilesize, precision, unit);
393 if(usehashes):
394 hashtypelist = usehashtypes.split(",");
395 openfile = open(infile, "rb");
396 filecontents = openfile.read();
397 openfile.close();
398 listnumcount = 0;
399 listnumend = len(hashtypelist);
400 while(listnumcount < listnumend):
401 hashtypelistlow = hashtypelist[listnumcount].strip();
402 hashtypelistup = hashtypelistlow.upper();
403 filehash = hashlib.new(hashtypelistup);
404 filehash.update(filecontents);
405 filegethash = filehash.hexdigest();
406 return_val.update({hashtypelistup: filegethash});
407 listnumcount += 1;
408 return return_val;
409
410 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
411 unit = unit.upper();
412 usehashtypes = usehashtypes.lower();
413 getfilesize = len(instring);
414 return_val = get_readable_size(getfilesize, precision, unit);
415 if(usehashes):
416 hashtypelist = usehashtypes.split(",");
417 listnumcount = 0;
418 listnumend = len(hashtypelist);
419 while(listnumcount < listnumend):
420 hashtypelistlow = hashtypelist[listnumcount].strip();
421 hashtypelistup = hashtypelistlow.upper();
422 filehash = hashlib.new(hashtypelistup);
423 if(sys.version[0]=="2"):
424 filehash.update(instring);
425 if(sys.version[0]>="3"):
426 filehash.update(instring.encode('utf-8'));
427 filegethash = filehash.hexdigest();
428 return_val.update({hashtypelistup: filegethash});
429 listnumcount += 1;
430 return return_val;
431
432 def http_status_to_reason(code):
433 reasons = {
434 100: 'Continue',
435 101: 'Switching Protocols',
436 102: 'Processing',
437 200: 'OK',
438 201: 'Created',
439 202: 'Accepted',
440 203: 'Non-Authoritative Information',
441 204: 'No Content',
442 205: 'Reset Content',
443 206: 'Partial Content',
444 207: 'Multi-Status',
445 208: 'Already Reported',
446 226: 'IM Used',
447 300: 'Multiple Choices',
448 301: 'Moved Permanently',
449 302: 'Found',
450 303: 'See Other',
451 304: 'Not Modified',
452 305: 'Use Proxy',
453 307: 'Temporary Redirect',
454 308: 'Permanent Redirect',
455 400: 'Bad Request',
456 401: 'Unauthorized',
457 402: 'Payment Required',
458 403: 'Forbidden',
459 404: 'Not Found',
460 405: 'Method Not Allowed',
461 406: 'Not Acceptable',
462 407: 'Proxy Authentication Required',
463 408: 'Request Timeout',
464 409: 'Conflict',
465 410: 'Gone',
466 411: 'Length Required',
467 412: 'Precondition Failed',
468 413: 'Payload Too Large',
469 414: 'URI Too Long',
470 415: 'Unsupported Media Type',
471 416: 'Range Not Satisfiable',
472 417: 'Expectation Failed',
473 421: 'Misdirected Request',
474 422: 'Unprocessable Entity',
475 423: 'Locked',
476 424: 'Failed Dependency',
477 426: 'Upgrade Required',
478 428: 'Precondition Required',
479 429: 'Too Many Requests',
480 431: 'Request Header Fields Too Large',
481 451: 'Unavailable For Legal Reasons',
482 500: 'Internal Server Error',
483 501: 'Not Implemented',
484 502: 'Bad Gateway',
485 503: 'Service Unavailable',
486 504: 'Gateway Timeout',
487 505: 'HTTP Version Not Supported',
488 506: 'Variant Also Negotiates',
489 507: 'Insufficient Storage',
490 508: 'Loop Detected',
491 510: 'Not Extended',
492 511: 'Network Authentication Required'
493 }
494 return reasons.get(code, 'Unknown Status Code');
495
496 def ftp_status_to_reason(code):
497 reasons = {
498 110: 'Restart marker reply',
499 120: 'Service ready in nnn minutes',
500 125: 'Data connection already open; transfer starting',
501 150: 'File status okay; about to open data connection',
502 200: 'Command okay',
503 202: 'Command not implemented, superfluous at this site',
504 211: 'System status, or system help reply',
505 212: 'Directory status',
506 213: 'File status',
507 214: 'Help message',
508 215: 'NAME system type',
509 220: 'Service ready for new user',
510 221: 'Service closing control connection',
511 225: 'Data connection open; no transfer in progress',
512 226: 'Closing data connection',
513 227: 'Entering Passive Mode',
514 230: 'User logged in, proceed',
515 250: 'Requested file action okay, completed',
516 257: '"PATHNAME" created',
517 331: 'User name okay, need password',
518 332: 'Need account for login',
519 350: 'Requested file action pending further information',
520 421: 'Service not available, closing control connection',
521 425: 'Can\'t open data connection',
522 426: 'Connection closed; transfer aborted',
523 450: 'Requested file action not taken',
524 451: 'Requested action aborted. Local error in processing',
525 452: 'Requested action not taken. Insufficient storage space in system',
526 500: 'Syntax error, command unrecognized',
527 501: 'Syntax error in parameters or arguments',
528 502: 'Command not implemented',
529 503: 'Bad sequence of commands',
530 504: 'Command not implemented for that parameter',
531 530: 'Not logged in',
532 532: 'Need account for storing files',
533 550: 'Requested action not taken. File unavailable',
534 551: 'Requested action aborted. Page type unknown',
535 552: 'Requested file action aborted. Exceeded storage allocation',
536 553: 'Requested action not taken. File name not allowed'
537 };
538 return reasons.get(code, 'Unknown Status Code');
539
540 def sftp_status_to_reason(code):
541 reasons = {
542 0: 'SSH_FX_OK',
543 1: 'SSH_FX_EOF',
544 2: 'SSH_FX_NO_SUCH_FILE',
545 3: 'SSH_FX_PERMISSION_DENIED',
546 4: 'SSH_FX_FAILURE',
547 5: 'SSH_FX_BAD_MESSAGE',
548 6: 'SSH_FX_NO_CONNECTION',
549 7: 'SSH_FX_CONNECTION_LOST',
550 8: 'SSH_FX_OP_UNSUPPORTED'
551 };
552 return reasons.get(code, 'Unknown Status Code');
553
554 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
555 if isinstance(headers, dict):
556 returnval = [];
557 if(sys.version[0]=="2"):
558 for headkey, headvalue in headers.iteritems():
559 returnval.append((headkey, headvalue));
560 if(sys.version[0]>="3"):
561 for headkey, headvalue in headers.items():
562 returnval.append((headkey, headvalue));
563 elif isinstance(headers, list):
564 returnval = headers;
565 else:
566 returnval = False;
567 return returnval;
568
569 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
570 if isinstance(headers, dict):
571 returnval = [];
572 if(sys.version[0]=="2"):
573 for headkey, headvalue in headers.iteritems():
574 returnval.append(headkey+": "+headvalue);
575 if(sys.version[0]>="3"):
576 for headkey, headvalue in headers.items():
577 returnval.append(headkey+": "+headvalue);
578 elif isinstance(headers, list):
579 returnval = headers;
580 else:
581 returnval = False;
582 return returnval;
583
584 def make_http_headers_from_pycurl_to_dict(headers):
585 header_dict = {};
586 headers = headers.strip().split('\r\n');
587 for header in headers:
588 parts = header.split(': ', 1)
589 if(len(parts) == 2):
590 key, value = parts;
591 header_dict[key.title()] = value;
592 return header_dict;
593
594 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
595 if isinstance(headers, list):
596 returnval = {};
597 mli = 0;
598 mlil = len(headers);
599 while(mli<mlil):
600 returnval.update({headers[mli][0]: headers[mli][1]});
601 mli = mli + 1;
602 elif isinstance(headers, dict):
603 returnval = headers;
604 else:
605 returnval = False;
606 return returnval;
607
608 def get_httplib_support(checkvalue=None):
609 global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
610 returnval = [];
611 returnval.append("ftp");
612 returnval.append("httplib");
613 if(havehttplib2):
614 returnval.append("httplib2");
615 returnval.append("urllib");
616 if(haveurllib3):
617 returnval.append("urllib3");
618 returnval.append("request3");
619 returnval.append("request");
620 if(haverequests):
621 returnval.append("requests");
622 if(haveaiohttp):
623 returnval.append("aiohttp");
624 if(havehttpx):
625 returnval.append("httpx");
626 returnval.append("httpx2");
627 if(havemechanize):
628 returnval.append("mechanize");
629 if(havepycurl):
630 returnval.append("pycurl");
631 if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
632 returnval.append("pycurl2");
633 if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
634 returnval.append("pycurl3");
635 if(haveparamiko):
636 returnval.append("sftp");
637 if(havepysftp):
638 returnval.append("pysftp");
639 if(not checkvalue is None):
640 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
641 checkvalue = "urllib";
642 if(checkvalue=="httplib1"):
643 checkvalue = "httplib";
644 if(checkvalue in returnval):
645 returnval = True;
646 else:
647 returnval = False;
648 return returnval;
649
650 def check_httplib_support(checkvalue="urllib"):
651 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
652 checkvalue = "urllib";
653 if(checkvalue=="httplib1"):
654 checkvalue = "httplib";
655 returnval = get_httplib_support(checkvalue);
656 return returnval;
657
658 def get_httplib_support_list():
659 returnval = get_httplib_support(None);
660 return returnval;
661
662 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
663 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
664 if(sleep<0):
665 sleep = geturls_download_sleep;
666 if(timeout<=0):
667 timeout = 10;
668 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
669 httplibuse = "urllib";
670 if(httplibuse=="httplib1"):
671 httplibuse = "httplib";
672 if(not haverequests and httplibuse=="requests"):
673 httplibuse = "urllib";
674 if(not haveaiohttp and httplibuse=="aiohttp"):
675 httplibuse = "urllib";
676 if(not havehttpx and httplibuse=="httpx"):
677 httplibuse = "urllib";
678 if(not havehttpx and httplibuse=="httpx2"):
679 httplibuse = "urllib";
680 if(not havehttpcore and httplibuse=="httpcore"):
681 httplibuse = "urllib";
682 if(not havehttpcore and httplibuse=="httpcore2"):
683 httplibuse = "urllib";
684 if(not havemechanize and httplibuse=="mechanize"):
685 httplibuse = "urllib";
686 if(not havepycurl and httplibuse=="pycurl"):
687 httplibuse = "urllib";
688 if(not havepycurl and httplibuse=="pycurl2"):
689 httplibuse = "urllib";
690 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
691 httplibuse = "pycurl";
692 if(not havepycurl and httplibuse=="pycurl3"):
693 httplibuse = "urllib";
694 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
695 httplibuse = "pycurl2";
696 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
697 httplibuse = "pycurl";
698 if(not havehttplib2 and httplibuse=="httplib2"):
699 httplibuse = "httplib";
700 if(not haveparamiko and httplibuse=="sftp"):
701 httplibuse = "ftp";
702 if(not havepysftp and httplibuse=="pysftp"):
703 httplibuse = "ftp";
704 urlparts = urlparse.urlparse(httpurl);
705 if(isinstance(httpheaders, list)):
706 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
707 httpheaders = fix_header_names(httpheaders);
708 if(httpuseragent is not None):
709 if('User-Agent' in httpheaders):
710 httpheaders['User-Agent'] = httpuseragent;
711 else:
712 httpuseragent.update({'User-Agent': httpuseragent});
713 if(httpreferer is not None):
714 if('Referer' in httpheaders):
715 httpheaders['Referer'] = httpreferer;
716 else:
717 httpuseragent.update({'Referer': httpreferer});
718 if(urlparts.username is not None or urlparts.password is not None):
719 if(sys.version[0]=="2"):
720 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
721 if(sys.version[0]>="3"):
722 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
723 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
724 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
725 if(httplibuse=="urllib" or httplibuse=="mechanize"):
726 if(isinstance(httpheaders, dict)):
727 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
728 if(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
729 if(isinstance(httpheaders, dict)):
730 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
731 geturls_opener.addheaders = httpheaders;
732 time.sleep(sleep);
733 if(postdata is not None and not isinstance(postdata, dict)):
734 postdata = urlencode(postdata);
735 if(httplibuse=="urllib" or httplibuse=="request"):
736 geturls_request = Request(httpurl);
737 try:
738 if(httpmethod=="GET"):
739 geturls_text = geturls_opener.open(geturls_request);
740 elif(httpmethod=="POST"):
741 geturls_text = geturls_opener.open(geturls_request, data=postdata);
742 else:
743 geturls_text = geturls_opener.open(geturls_request);
744 except HTTPError as geturls_text_error:
745 geturls_text = geturls_text_error;
746 log.info("Error With URL "+httpurl);
747 except URLError:
748 log.info("Error With URL "+httpurl);
749 return False;
750 except socket.timeout:
751 log.info("Error With URL "+httpurl);
752 return False;
753 httpcodeout = geturls_text.getcode();
754 try:
755 httpcodereason = geturls_text.reason;
756 except AttributeError:
757 httpcodereason = http_status_to_reason(geturls_text.getcode());
758 try:
759 httpversionout = geturls_text.version;
760 except AttributeError:
761 httpversionout = "1.1";
762 httpmethodout = geturls_request.get_method();
763 httpurlout = geturls_text.geturl();
764 httpheaderout = geturls_text.info();
765 httpheadersentout = httpheaders;
766 elif(httplibuse=="httplib"):
767 if(urlparts[0]=="http"):
768 httpconn = HTTPConnection(urlparts[1], timeout=timeout);
769 elif(urlparts[0]=="https"):
770 httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
771 else:
772 return False;
773 if(postdata is not None and not isinstance(postdata, dict)):
774 postdata = urlencode(postdata);
775 try:
776 if(httpmethod=="GET"):
777 httpconn.request("GET", urlparts[2], headers=httpheaders);
778 elif(httpmethod=="POST"):
779 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
780 else:
781 httpconn.request("GET", urlparts[2], headers=httpheaders);
782 except socket.timeout:
783 log.info("Error With URL "+httpurl);
784 return False;
785 except socket.gaierror:
786 log.info("Error With URL "+httpurl);
787 return False;
788 except BlockingIOError:
789 log.info("Error With URL "+httpurl);
790 return False;
791 geturls_text = httpconn.getresponse();
792 httpcodeout = geturls_text.status;
793 httpcodereason = geturls_text.reason;
794 if(geturls_text.version=="10"):
795 httpversionout = "1.0";
796 else:
797 httpversionout = "1.1";
798 httpmethodout = geturls_text._method;
799 httpurlout = httpurl;
800 httpheaderout = geturls_text.getheaders();
801 httpheadersentout = httpheaders;
802 elif(httplibuse=="httplib2"):
803 if(urlparts[0]=="http"):
804 httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
805 elif(urlparts[0]=="https"):
806 httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
807 else:
808 return False;
809 if(postdata is not None and not isinstance(postdata, dict)):
810 postdata = urlencode(postdata);
811 try:
812 if(httpmethod=="GET"):
813 httpconn.request("GET", urlparts[2], headers=httpheaders);
814 elif(httpmethod=="POST"):
815 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
816 else:
817 httpconn.request("GET", urlparts[2], headers=httpheaders);
818 except socket.timeout:
819 log.info("Error With URL "+httpurl);
820 return False;
821 except socket.gaierror:
822 log.info("Error With URL "+httpurl);
823 return False;
824 except BlockingIOError:
825 log.info("Error With URL "+httpurl);
826 return False;
827 geturls_text = httpconn.getresponse();
828 httpcodeout = geturls_text.status;
829 httpcodereason = geturls_text.reason;
830 if(geturls_text.version=="10"):
831 httpversionout = "1.0";
832 else:
833 httpversionout = "1.1";
834 httpmethodout = httpmethod;
835 httpurlout = httpurl;
836 httpheaderout = geturls_text.getheaders();
837 httpheadersentout = httpheaders;
838 elif(httplibuse=="urllib3" or httplibuse=="request3"):
839 timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
840 urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
841 try:
842 if(httpmethod=="GET"):
843 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
844 elif(httpmethod=="POST"):
845 geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
846 else:
847 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
848 except urllib3.exceptions.ConnectTimeoutError:
849 log.info("Error With URL "+httpurl);
850 return False;
851 except urllib3.exceptions.ConnectError:
852 log.info("Error With URL "+httpurl);
853 return False;
854 except urllib3.exceptions.MaxRetryError:
855 log.info("Error With URL "+httpurl);
856 return False;
857 except socket.timeout:
858 log.info("Error With URL "+httpurl);
859 return False;
860 except ValueError:
861 log.info("Error With URL "+httpurl);
862 return False;
863 httpcodeout = geturls_text.status;
864 httpcodereason = geturls_text.reason;
865 if(geturls_text.version=="10"):
866 httpversionout = "1.0";
867 else:
868 httpversionout = "1.1";
869 httpmethodout = httpmethod;
870 httpurlout = geturls_text.geturl();
871 httpheaderout = geturls_text.info();
872 httpheadersentout = httpheaders;
873 elif(httplibuse=="requests"):
874 try:
875 reqsession = requests.Session();
876 if(httpmethod=="GET"):
877 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
878 elif(httpmethod=="POST"):
879 geturls_text = reqsession.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
880 else:
881 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
882 except requests.exceptions.ConnectTimeout:
883 log.info("Error With URL "+httpurl);
884 return False;
885 except requests.exceptions.ConnectError:
886 log.info("Error With URL "+httpurl);
887 return False;
888 except socket.timeout:
889 log.info("Error With URL "+httpurl);
890 return False;
891 httpcodeout = geturls_text.status_code;
892 httpcodereason = geturls_text.reason;
893 if(geturls_text.raw.version=="10"):
894 httpversionout = "1.0";
895 else:
896 httpversionout = "1.1";
897 httpmethodout = httpmethod;
898 httpurlout = geturls_text.url;
899 httpheaderout = geturls_text.headers;
900 httpheadersentout = geturls_text.request.headers;
901 elif(httplibuse=="aiohttp"):
902 try:
903 reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize);
904 if(httpmethod=="GET"):
905 geturls_text = reqsession.get(httpurl);
906 elif(httpmethod=="POST"):
907 geturls_text = reqsession.post(httpurl, data=postdata);
908 else:
909 geturls_text = reqsession.get(httpurl);
910 except aiohttp.exceptions.ConnectTimeout:
911 log.info("Error With URL "+httpurl);
912 return False;
913 except aiohttp.exceptions.ConnectError:
914 log.info("Error With URL "+httpurl);
915 return False;
916 except socket.timeout:
917 log.info("Error With URL "+httpurl);
918 return False;
919 httpcodeout = geturls_text.status;
920 httpcodereason = geturls_text.reason;
921 httpversionout = geturls_text.version;
922 httpmethodout = geturls_text.method;
923 httpurlout = geturls_text.url;
924 httpheaderout = geturls_text.headers;
925 httpheadersentout = geturls_text.request_info.headers;
926 elif(httplibuse=="httpx"):
927 try:
928 if(httpmethod=="GET"):
929 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
930 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
931 elif(httpmethod=="POST"):
932 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
933 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
934 else:
935 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
936 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
937 except httpx.ConnectTimeout:
938 log.info("Error With URL "+httpurl);
939 return False;
940 except httpx.ConnectError:
941 log.info("Error With URL "+httpurl);
942 return False;
943 except socket.timeout:
944 log.info("Error With URL "+httpurl);
945 return False;
946 httpcodeout = geturls_text.status_code;
947 try:
948 httpcodereason = geturls_text.reason_phrase;
949 except:
950 httpcodereason = http_status_to_reason(geturls_text.status_code);
951 httpversionout = geturls_text.http_version;
952 httpmethodout = httpmethod;
953 httpurlout = str(geturls_text.url);
954 httpheaderout = geturls_text.headers;
955 httpheadersentout = geturls_text.request.headers;
956 elif(httplibuse=="httpx2"):
957 try:
958 if(httpmethod=="GET"):
959 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
960 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
961 elif(httpmethod=="POST"):
962 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
963 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
964 else:
965 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
966 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
967 except httpx.ConnectTimeout:
968 log.info("Error With URL "+httpurl);
969 return False;
970 except httpx.ConnectError:
971 log.info("Error With URL "+httpurl);
972 return False;
973 except socket.timeout:
974 log.info("Error With URL "+httpurl);
975 return False;
976 httpcodeout = geturls_text.status_code;
977 try:
978 httpcodereason = geturls_text.reason_phrase;
979 except:
980 httpcodereason = http_status_to_reason(geturls_text.status_code);
981 httpversionout = geturls_text.http_version;
982 httpmethodout = httpmethod;
983 httpurlout = str(geturls_text.url);
984 httpheaderout = geturls_text.headers;
985 httpheadersentout = geturls_text.request.headers;
986 elif(httplibuse=="httpcore"):
987 try:
988 if(httpmethod=="GET"):
989 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
990 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
991 elif(httpmethod=="POST"):
992 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
993 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
994 else:
995 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
996 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
997 except httpcore.ConnectTimeout:
998 log.info("Error With URL "+httpurl);
999 return False;
1000 except httpcore.ConnectError:
1001 log.info("Error With URL "+httpurl);
1002 return False;
1003 except socket.timeout:
1004 log.info("Error With URL "+httpurl);
1005 return False;
1006 httpcodeout = geturls_text.status;
1007 httpcodereason = http_status_to_reason(geturls_text.status);
1008 httpversionout = "1.1";
1009 httpmethodout = httpmethod;
1010 httpurlout = str(httpurl);
1011 httpheaderout = geturls_text.headers;
1012 httpheadersentout = httpheaders;
1013 elif(httplibuse=="httpcore2"):
1014 try:
1015 if(httpmethod=="GET"):
1016 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1017 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1018 elif(httpmethod=="POST"):
1019 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1020 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1021 else:
1022 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1023 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1024 except httpcore.ConnectTimeout:
1025 log.info("Error With URL "+httpurl);
1026 return False;
1027 except httpcore.ConnectError:
1028 log.info("Error With URL "+httpurl);
1029 return False;
1030 except socket.timeout:
1031 log.info("Error With URL "+httpurl);
1032 return False;
1033 httpcodeout = geturls_text.status;
1034 httpcodereason = http_status_to_reason(geturls_text.status);
1035 httpversionout = "1.1";
1036 httpmethodout = httpmethod;
1037 httpurlout = str(httpurl);
1038 httpheaderout = geturls_text.headers;
1039 httpheadersentout = httpheaders;
1040 elif(httplibuse=="mechanize"):
1041 geturls_opener = mechanize.Browser();
1042 if(isinstance(httpheaders, dict)):
1043 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1044 time.sleep(sleep);
1045 geturls_opener.addheaders = httpheaders;
1046 geturls_opener.set_cookiejar(httpcookie);
1047 geturls_opener.set_handle_robots(False);
1048 if(postdata is not None and not isinstance(postdata, dict)):
1049 postdata = urlencode(postdata);
1050 try:
1051 if(httpmethod=="GET"):
1052 geturls_text = geturls_opener.open(httpurl);
1053 elif(httpmethod=="POST"):
1054 geturls_text = geturls_opener.open(httpurl, data=postdata);
1055 else:
1056 geturls_text = geturls_opener.open(httpurl);
1057 except mechanize.HTTPError as geturls_text_error:
1058 geturls_text = geturls_text_error;
1059 log.info("Error With URL "+httpurl);
1060 except URLError:
1061 log.info("Error With URL "+httpurl);
1062 return False;
1063 except socket.timeout:
1064 log.info("Error With URL "+httpurl);
1065 return False;
1066 httpcodeout = geturls_text.code;
1067 httpcodereason = geturls_text.msg;
1068 httpversionout = "1.1";
1069 httpmethodout = httpmethod;
1070 httpurlout = geturls_text.geturl();
1071 httpheaderout = geturls_text.info();
1072 reqhead = geturls_opener.request;
1073 httpheadersentout = reqhead.header_items();
1074 elif(httplibuse=="pycurl"):
1075 retrieved_body = BytesIO();
1076 retrieved_headers = BytesIO();
1077 try:
1078 if(httpmethod=="GET"):
1079 geturls_text = pycurl.Curl();
1080 geturls_text.setopt(geturls_text.URL, httpurl);
1081 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1082 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1083 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1084 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1085 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1086 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1087 geturls_text.perform();
1088 elif(httpmethod=="POST"):
1089 geturls_text = pycurl.Curl();
1090 geturls_text.setopt(geturls_text.URL, httpurl);
1091 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1092 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1093 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1094 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1095 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1096 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1097 geturls_text.setopt(geturls_text.POST, True);
1098 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1099 geturls_text.perform();
1100 else:
1101 geturls_text = pycurl.Curl();
1102 geturls_text.setopt(geturls_text.URL, httpurl);
1103 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1104 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1105 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1106 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1107 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1108 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1109 geturls_text.perform();
1110 retrieved_headers.seek(0);
1111 if(sys.version[0]=="2"):
1112 pycurlhead = retrieved_headers.read();
1113 if(sys.version[0]>="3"):
1114 pycurlhead = retrieved_headers.read().decode('UTF-8');
1115 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1116 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1117 retrieved_body.seek(0);
1118 except socket.timeout:
1119 log.info("Error With URL "+httpurl);
1120 return False;
1121 except socket.gaierror:
1122 log.info("Error With URL "+httpurl);
1123 return False;
1124 except ValueError:
1125 log.info("Error With URL "+httpurl);
1126 return False;
1127 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1128 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1129 httpversionout = pyhttpverinfo[0];
1130 httpmethodout = httpmethod;
1131 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1132 httpheaderout = pycurlheadersout;
1133 httpheadersentout = httpheaders;
1134 elif(httplibuse=="pycurl2"):
1135 retrieved_body = BytesIO();
1136 retrieved_headers = BytesIO();
1137 try:
1138 if(httpmethod=="GET"):
1139 geturls_text = pycurl.Curl();
1140 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1141 geturls_text.setopt(geturls_text.URL, httpurl);
1142 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1143 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1144 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1145 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1146 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1147 geturls_text.perform();
1148 elif(httpmethod=="POST"):
1149 geturls_text = pycurl.Curl();
1150 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1151 geturls_text.setopt(geturls_text.URL, httpurl);
1152 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1153 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1154 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1155 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1156 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1157 geturls_text.setopt(geturls_text.POST, True);
1158 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1159 geturls_text.perform();
1160 else:
1161 geturls_text = pycurl.Curl();
1162 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1163 geturls_text.setopt(geturls_text.URL, httpurl);
1164 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1165 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1166 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1167 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1168 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1169 geturls_text.perform();
1170 retrieved_headers.seek(0);
1171 if(sys.version[0]=="2"):
1172 pycurlhead = retrieved_headers.read();
1173 if(sys.version[0]>="3"):
1174 pycurlhead = retrieved_headers.read().decode('UTF-8');
1175 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
1176 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1177 retrieved_body.seek(0);
1178 except socket.timeout:
1179 log.info("Error With URL "+httpurl);
1180 return False;
1181 except socket.gaierror:
1182 log.info("Error With URL "+httpurl);
1183 return False;
1184 except ValueError:
1185 log.info("Error With URL "+httpurl);
1186 return False;
1187 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1188 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1189 httpversionout = pyhttpverinfo[0];
1190 httpmethodout = httpmethod;
1191 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1192 httpheaderout = pycurlheadersout;
1193 httpheadersentout = httpheaders;
1194 elif(httplibuse=="pycurl3"):
1195 retrieved_body = BytesIO();
1196 retrieved_headers = BytesIO();
1197 try:
1198 if(httpmethod=="GET"):
1199 geturls_text = pycurl.Curl();
1200 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1201 geturls_text.setopt(geturls_text.URL, httpurl);
1202 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1203 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1204 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1205 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1206 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1207 geturls_text.perform();
1208 elif(httpmethod=="POST"):
1209 geturls_text = pycurl.Curl();
1210 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1211 geturls_text.setopt(geturls_text.URL, httpurl);
1212 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1213 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1214 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1215 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1216 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1217 geturls_text.setopt(geturls_text.POST, True);
1218 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1219 geturls_text.perform();
1220 else:
1221 geturls_text = pycurl.Curl();
1222 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1223 geturls_text.setopt(geturls_text.URL, httpurl);
1224 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1225 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1226 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1227 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1228 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1229 geturls_text.perform();
1230 retrieved_headers.seek(0);
1231 if(sys.version[0]=="2"):
1232 pycurlhead = retrieved_headers.read();
1233 if(sys.version[0]>="3"):
1234 pycurlhead = retrieved_headers.read().decode('UTF-8');
1235 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1236 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1237 retrieved_body.seek(0);
1238 except socket.timeout:
1239 log.info("Error With URL "+httpurl);
1240 return False;
1241 except socket.gaierror:
1242 log.info("Error With URL "+httpurl);
1243 return False;
1244 except ValueError:
1245 log.info("Error With URL "+httpurl);
1246 return False;
1247 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1248 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1249 httpversionout = pyhttpverinfo[0];
1250 httpmethodout = httpmethod;
1251 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1252 httpheaderout = pycurlheadersout;
1253 httpheadersentout = httpheaders;
1254 elif(httplibuse=="ftp"):
1255 geturls_text = download_file_from_ftp_file(httpurl);
1256 if(not geturls_text):
1257 return False;
1258 downloadsize = None;
1259 if(downloadsize is not None):
1260 downloadsize = int(downloadsize);
1261 if downloadsize is None: downloadsize = 0;
1262 fulldatasize = 0;
1263 prevdownsize = 0;
1264 log.info("Downloading URL "+httpurl);
1265 with BytesIO() as strbuf:
1266 while True:
1267 databytes = geturls_text.read(buffersize);
1268 if not databytes: break;
1269 datasize = len(databytes);
1270 fulldatasize = datasize + fulldatasize;
1271 percentage = "";
1272 if(downloadsize>0):
1273 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1274 downloaddiff = fulldatasize - prevdownsize;
1275 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1276 prevdownsize = fulldatasize;
1277 strbuf.write(databytes);
1278 strbuf.seek(0);
1279 returnval_content = strbuf.read();
1280 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1281 geturls_text.close();
1282 elif(httplibuse=="sftp"):
1283 geturls_text = download_file_from_sftp_file(httpurl);
1284 if(not geturls_text):
1285 return False;
1286 downloadsize = None;
1287 if(downloadsize is not None):
1288 downloadsize = int(downloadsize);
1289 if downloadsize is None: downloadsize = 0;
1290 fulldatasize = 0;
1291 prevdownsize = 0;
1292 log.info("Downloading URL "+httpurl);
1293 with BytesIO() as strbuf:
1294 while True:
1295 databytes = geturls_text.read(buffersize);
1296 if not databytes: break;
1297 datasize = len(databytes);
1298 fulldatasize = datasize + fulldatasize;
1299 percentage = "";
1300 if(downloadsize>0):
1301 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1302 downloaddiff = fulldatasize - prevdownsize;
1303 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1304 prevdownsize = fulldatasize;
1305 strbuf.write(databytes);
1306 strbuf.seek(0);
1307 returnval_content = strbuf.read();
1308 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1309 geturls_text.close();
1310 return returnval;
1311 elif(httplibuse=="pysftp"):
1312 geturls_text = download_file_from_pysftp_file(httpurl);
1313 if(not geturls_text):
1314 return False;
1315 downloadsize = None;
1316 if(downloadsize is not None):
1317 downloadsize = int(downloadsize);
1318 if downloadsize is None: downloadsize = 0;
1319 fulldatasize = 0;
1320 prevdownsize = 0;
1321 log.info("Downloading URL "+httpurl);
1322 with BytesIO() as strbuf:
1323 while True:
1324 databytes = geturls_text.read(buffersize);
1325 if not databytes: break;
1326 datasize = len(databytes);
1327 fulldatasize = datasize + fulldatasize;
1328 percentage = "";
1329 if(downloadsize>0):
1330 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1331 downloaddiff = fulldatasize - prevdownsize;
1332 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1333 prevdownsize = fulldatasize;
1334 strbuf.write(databytes);
1335 strbuf.seek(0);
1336 returnval_content = strbuf.read();
1337 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1338 geturls_text.close();
1339 return returnval;
1340 else:
1341 returnval = False;
1342 if(isinstance(httpheaderout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1343 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1344 if(isinstance(httpheaderout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1345 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
1346 if(sys.version[0]=="2"):
1347 try:
1348 prehttpheaderout = httpheaderout;
1349 httpheaderkeys = httpheaderout.keys();
1350 imax = len(httpheaderkeys);
1351 ic = 0;
1352 httpheaderout = {};
1353 while(ic < imax):
1354 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1355 ic += 1;
1356 except AttributeError:
1357 pass;
1358 httpheaderout = fix_header_names(httpheaderout);
1359 if(isinstance(httpheadersentout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1360 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1361 if(isinstance(httpheadersentout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1362 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
1363 httpheadersentout = fix_header_names(httpheadersentout);
1364 log.info("Downloading URL "+httpurl);
1365 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"):
1366 downloadsize = httpheaderout.get('Content-Length');
1367 if(downloadsize is not None):
1368 downloadsize = int(downloadsize);
1369 if downloadsize is None: downloadsize = 0;
1370 fulldatasize = 0;
1371 prevdownsize = 0;
1372 log.info("Downloading URL "+httpurl);
1373 with BytesIO() as strbuf:
1374 while True:
1375 databytes = geturls_text.read(buffersize);
1376 if not databytes: break;
1377 datasize = len(databytes);
1378 fulldatasize = datasize + fulldatasize;
1379 percentage = "";
1380 if(downloadsize>0):
1381 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1382 downloaddiff = fulldatasize - prevdownsize;
1383 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1384 prevdownsize = fulldatasize;
1385 strbuf.write(databytes);
1386 strbuf.seek(0);
1387 returnval_content = strbuf.read();
1388 geturls_text.close();
1389 if(httpheaderout.get("Content-Encoding")=="gzip"):
1390 try:
1391 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1392 except zlib.error:
1393 pass;
1394 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1395 try:
1396 returnval_content = zlib.decompress(returnval_content);
1397 except zlib.error:
1398 pass;
1399 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1400 try:
1401 returnval_content = brotli.decompress(returnval_content);
1402 except brotli.error:
1403 pass;
1404 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1405 try:
1406 returnval_content = zstandard.decompress(returnval_content);
1407 except zstandard.error:
1408 pass;
1409 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1410 try:
1411 returnval_content = lzma.decompress(returnval_content);
1412 except zstandard.error:
1413 pass;
1414 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1415 try:
1416 returnval_content = bz2.decompress(returnval_content);
1417 except zstandard.error:
1418 pass;
1419 elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1420 downloadsize = httpheaderout.get('Content-Length');
1421 if(downloadsize is not None):
1422 downloadsize = int(downloadsize);
1423 if downloadsize is None: downloadsize = 0;
1424 fulldatasize = 0;
1425 prevdownsize = 0;
1426 log.info("Downloading URL "+httpurl);
1427 with BytesIO() as strbuf:
1428 while True:
1429 databytes = geturls_text.read();
1430 if not databytes: break;
1431 datasize = len(databytes);
1432 fulldatasize = datasize + fulldatasize;
1433 percentage = "";
1434 if(downloadsize>0):
1435 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1436 downloaddiff = fulldatasize - prevdownsize;
1437 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1438 prevdownsize = fulldatasize;
1439 strbuf.write(databytes);
1440 break;
1441 strbuf.seek(0);
1442 returnval_content = strbuf.read();
1443 geturls_text.close();
1444 if(httpheaderout.get("Content-Encoding")=="gzip"):
1445 try:
1446 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1447 except zlib.error:
1448 pass;
1449 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1450 try:
1451 returnval_content = zlib.decompress(returnval_content);
1452 except zlib.error:
1453 pass;
1454 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1455 try:
1456 returnval_content = brotli.decompress(returnval_content);
1457 except brotli.error:
1458 pass;
1459 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1460 try:
1461 returnval_content = zstandard.decompress(returnval_content);
1462 except zstandard.error:
1463 pass;
1464 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1465 try:
1466 returnval_content = lzma.decompress(returnval_content);
1467 except zstandard.error:
1468 pass;
1469 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1470 try:
1471 returnval_content = bz2.decompress(returnval_content);
1472 except zstandard.error:
1473 pass;
1474 elif(httplibuse=="requests"):
1475 log.info("Downloading URL "+httpurl);
1476 downloadsize = httpheaderout.get('Content-Length');
1477 if(downloadsize is not None):
1478 downloadsize = int(downloadsize);
1479 if downloadsize is None: downloadsize = 0;
1480 fulldatasize = 0;
1481 prevdownsize = 0;
1482 log.info("Downloading URL "+httpurl);
1483 with BytesIO() as strbuf:
1484 while True:
1485 databytes = geturls_text.raw.read(buffersize);
1486 if not databytes: break;
1487 datasize = len(databytes);
1488 fulldatasize = datasize + fulldatasize;
1489 percentage = "";
1490 if(downloadsize>0):
1491 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1492 downloaddiff = fulldatasize - prevdownsize;
1493 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1494 prevdownsize = fulldatasize;
1495 strbuf.write(databytes);
1496 strbuf.seek(0);
1497 returnval_content = strbuf.read();
1498 geturls_text.close();
1499 if(httpheaderout.get("Content-Encoding")=="gzip"):
1500 try:
1501 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1502 except zlib.error:
1503 pass;
1504 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1505 try:
1506 returnval_content = zlib.decompress(returnval_content);
1507 except zlib.error:
1508 pass;
1509 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1510 try:
1511 returnval_content = brotli.decompress(returnval_content);
1512 except brotli.error:
1513 pass;
1514 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1515 try:
1516 returnval_content = zstandard.decompress(returnval_content);
1517 except zstandard.error:
1518 pass;
1519 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1520 try:
1521 returnval_content = lzma.decompress(returnval_content);
1522 except zstandard.error:
1523 pass;
1524 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1525 try:
1526 returnval_content = bz2.decompress(returnval_content);
1527 except zstandard.error:
1528 pass;
1529 elif(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
1530 log.info("Downloading URL "+httpurl);
1531 downloadsize = httpheaderout.get('Content-Length');
1532 if(downloadsize is not None):
1533 downloadsize = int(downloadsize);
1534 if downloadsize is None: downloadsize = 0;
1535 fulldatasize = 0;
1536 prevdownsize = 0;
1537 log.info("Downloading URL "+httpurl);
1538 with BytesIO() as strbuf:
1539 while True:
1540 databytes = retrieved_body.read(buffersize);
1541 if not databytes: break;
1542 datasize = len(databytes);
1543 fulldatasize = datasize + fulldatasize;
1544 percentage = "";
1545 if(downloadsize>0):
1546 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1547 downloaddiff = fulldatasize - prevdownsize;
1548 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1549 prevdownsize = fulldatasize;
1550 strbuf.write(databytes);
1551 strbuf.seek(0);
1552 returnval_content = strbuf.read();
1553 geturls_text.close();
1554 if(httpheaderout.get("Content-Encoding")=="gzip"):
1555 try:
1556 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1557 except zlib.error:
1558 pass;
1559 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1560 try:
1561 returnval_content = zlib.decompress(returnval_content);
1562 except zlib.error:
1563 pass;
1564 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1565 try:
1566 returnval_content = brotli.decompress(returnval_content);
1567 except brotli.error:
1568 pass;
1569 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1570 try:
1571 returnval_content = zstandard.decompress(returnval_content);
1572 except zstandard.error:
1573 pass;
1574 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1575 try:
1576 returnval_content = lzma.decompress(returnval_content);
1577 except zstandard.error:
1578 pass;
1579 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1580 try:
1581 returnval_content = bz2.decompress(returnval_content);
1582 except zstandard.error:
1583 pass;
1584 elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1585 pass;
1586 else:
1587 returnval = False;
1588 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse};
1589 return returnval;
1590
1591 def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
1592 if(isinstance(httpurl, list)):
1593 pass;
1594 elif(isinstance(httpurl, tuple)):
1595 pass;
1596 elif(isinstance(httpurl, dict)):
1597 httpurl = httpurl.values();
1598 else:
1599 httpurl = [httpurl];
1600 listsize = len(httpurl);
1601 listcount = 0;
1602 returnval = [];
1603 while(listcount<listsize):
1604 ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1605 returnval.append(ouputval);
1606 listcount += 1;
1607 return returnval;
1608
1609 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1610 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1611 exec_time_start = time.time();
1612 myhash = hashlib.new("sha1");
1613 if(sys.version[0]=="2"):
1614 myhash.update(httpurl);
1615 myhash.update(str(buffersize));
1616 myhash.update(str(exec_time_start));
1617 if(sys.version[0]>="3"):
1618 myhash.update(httpurl.encode('utf-8'));
1619 myhash.update(str(buffersize).encode('utf-8'));
1620 myhash.update(str(exec_time_start).encode('utf-8'));
1621 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1622 if(sleep<0):
1623 sleep = geturls_download_sleep;
1624 if(timeout<=0):
1625 timeout = 10;
1626 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1627 httplibuse = "urllib";
1628 if(httplibuse=="httplib1"):
1629 httplibuse = "httplib";
1630 if(not haverequests and httplibuse=="requests"):
1631 httplibuse = "urllib";
1632 if(not haveaiohttp and httplibuse=="aiohttp"):
1633 httplibuse = "urllib";
1634 if(not havehttpx and httplibuse=="httpx"):
1635 httplibuse = "urllib";
1636 if(not havehttpx and httplibuse=="httpx2"):
1637 httplibuse = "urllib";
1638 if(not havehttpcore and httplibuse=="httpcore"):
1639 httplibuse = "urllib";
1640 if(not havehttpcore and httplibuse=="httpcore2"):
1641 httplibuse = "urllib";
1642 if(not havemechanize and httplibuse=="mechanize"):
1643 httplibuse = "urllib";
1644 if(not havepycurl and httplibuse=="pycurl"):
1645 httplibuse = "urllib";
1646 if(not havepycurl and httplibuse=="pycurl2"):
1647 httplibuse = "urllib";
1648 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1649 httplibuse = "pycurl";
1650 if(not havepycurl and httplibuse=="pycurl3"):
1651 httplibuse = "urllib";
1652 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1653 httplibuse = "pycurl2";
1654 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1655 httplibuse = "pycurl";
1656 if(not havehttplib2 and httplibuse=="httplib2"):
1657 httplibuse = "httplib";
1658 if(not haveparamiko and httplibuse=="sftp"):
1659 httplibuse = "ftp";
1660 if(not haveparamiko and httplibuse=="pysftp"):
1661 httplibuse = "ftp";
1662 pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1663 if(not pretmpfilename):
1664 return False;
1665 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1666 tmpfilename = f.name;
1667 try:
1668 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1669 except AttributeError:
1670 try:
1671 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1672 except ValueError:
1673 pass;
1674 except ValueError:
1675 pass;
1676 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1677 f.write(pretmpfilename.get('Content'));
1678 f.close();
1679 exec_time_end = time.time();
1680 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1681 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1682 return returnval;
1683
1684 def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1685 if(isinstance(httpurl, list)):
1686 pass;
1687 elif(isinstance(httpurl, tuple)):
1688 pass;
1689 elif(isinstance(httpurl, dict)):
1690 httpurl = httpurl.values();
1691 else:
1692 httpurl = [httpurl];
1693 listsize = len(httpurl);
1694 listcount = 0;
1695 returnval = [];
1696 while(listcount<listsize):
1697 ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout);
1698 returnval.append(ouputval);
1699 listcount += 1;
1700 return returnval;
1701
1702 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1703 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1704 if(sleep<0):
1705 sleep = geturls_download_sleep;
1706 if(timeout<=0):
1707 timeout = 10;
1708 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1709 httplibuse = "urllib";
1710 if(httplibuse=="httplib1"):
1711 httplibuse = "httplib";
1712 if(not haverequests and httplibuse=="requests"):
1713 httplibuse = "urllib";
1714 if(not haveaiohttp and httplibuse=="aiohttp"):
1715 httplibuse = "urllib";
1716 if(not havehttpx and httplibuse=="httpx"):
1717 httplibuse = "urllib";
1718 if(not havehttpx and httplibuse=="httpx2"):
1719 httplibuse = "urllib";
1720 if(not havehttpcore and httplibuse=="httpcore"):
1721 httplibuse = "urllib";
1722 if(not havehttpcore and httplibuse=="httpcore2"):
1723 httplibuse = "urllib";
1724 if(not havemechanize and httplibuse=="mechanize"):
1725 httplibuse = "urllib";
1726 if(not havepycurl and httplibuse=="pycurl"):
1727 httplibuse = "urllib";
1728 if(not havepycurl and httplibuse=="pycurl2"):
1729 httplibuse = "urllib";
1730 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1731 httplibuse = "pycurl";
1732 if(not havepycurl and httplibuse=="pycurl3"):
1733 httplibuse = "urllib";
1734 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1735 httplibuse = "pycurl2";
1736 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1737 httplibuse = "pycurl";
1738 if(not havehttplib2 and httplibuse=="httplib2"):
1739 httplibuse = "httplib";
1740 if(not haveparamiko and httplibuse=="sftp"):
1741 httplibuse = "ftp";
1742 if(not havepysftp and httplibuse=="pysftp"):
1743 httplibuse = "ftp";
1744 if(not outfile=="-"):
1745 outpath = outpath.rstrip(os.path.sep);
1746 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1747 if(not os.path.exists(outpath)):
1748 os.makedirs(outpath);
1749 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1750 return False;
1751 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1752 return False;
1753 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1754 if(not pretmpfilename):
1755 return False;
1756 tmpfilename = pretmpfilename.get('Filename');
1757 downloadsize = int(os.path.getsize(tmpfilename));
1758 fulldatasize = 0;
1759 log.info("Moving file "+tmpfilename+" to "+filepath);
1760 exec_time_start = time.time();
1761 shutil.move(tmpfilename, filepath);
1762 try:
1763 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1764 except AttributeError:
1765 try:
1766 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1767 except ValueError:
1768 pass;
1769 except ValueError:
1770 pass;
1771 exec_time_end = time.time();
1772 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1773 if(os.path.exists(tmpfilename)):
1774 os.remove(tmpfilename);
1775 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1776 if(outfile=="-"):
1777 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1778 tmpfilename = pretmpfilename.get('Filename');
1779 downloadsize = int(os.path.getsize(tmpfilename));
1780 fulldatasize = 0;
1781 prevdownsize = 0;
1782 exec_time_start = time.time();
1783 with open(tmpfilename, 'rb') as ft:
1784 f = BytesIO();
1785 while True:
1786 databytes = ft.read(buffersize[1]);
1787 if not databytes: break;
1788 datasize = len(databytes);
1789 fulldatasize = datasize + fulldatasize;
1790 percentage = "";
1791 if(downloadsize>0):
1792 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1793 downloaddiff = fulldatasize - prevdownsize;
1794 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1795 prevdownsize = fulldatasize;
1796 f.write(databytes);
1797 f.seek(0);
1798 fdata = f.getvalue();
1799 f.close();
1800 ft.close();
1801 os.remove(tmpfilename);
1802 exec_time_end = time.time();
1803 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1804 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse};
1805 return returnval;
1806
1807 def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1808 if(isinstance(httpurl, list)):
1809 pass;
1810 elif(isinstance(httpurl, tuple)):
1811 pass;
1812 elif(isinstance(httpurl, dict)):
1813 httpurl = httpurl.values();
1814 else:
1815 httpurl = [httpurl];
1816 listsize = len(httpurl);
1817 listcount = 0;
1818 returnval = [];
1819 while(listcount<listsize):
1820 ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout);
1821 returnval.append(ouputval);
1822 listcount += 1;
1823 return returnval;
1824
1825 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1826 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1827 return returnval;
1828
1829 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1830 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1831 return returnval;
1832
1833 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1834 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout);
1835 return returnval;
1836
1837 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1838 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout);
1839 return returnval;
1840
1841 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1842 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout);
1843 return returnval;
1844
1845 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1846 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout);
1847 return returnval;
1848
1849 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1850 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout);
1851 return returnval;
1852
1853 def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1854 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout);
1855 return returnval;
1856
1857 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1858 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout);
1859 return returnval;
1860
1861 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1862 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout);
1863 return returnval;
1864
1865 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1866 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout);
1867 return returnval;
1868
1869 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1870 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout);
1871 return returnval;
1872
1873 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1874 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout);
1875 return returnval;
1876
1877 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1878 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout);
1879 return returnval;
1880
1881 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1882 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout);
1883 return returnval;
1884
1885 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1886 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout);
1887 return returnval;
1888
1889 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1890 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout);
1891 return returnval;
1892
1893 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1894 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout);
1895 return returnval;
1896
1897 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1898 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout);
1899 return returnval;
1900
1901 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1902 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1903 return returnval;
1904
1905 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1906 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1907 return returnval;
1908
1909 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1910 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout);
1911 return returnval;
1912
1913 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1914 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout);
1915 return returnval;
1916
1917 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1918 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout);
1919 return returnval;
1920
1921 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1922 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout);
1923 return returnval;
1924
1925 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1926 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout);
1927 return returnval;
1928
1929 def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1930 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout);
1931 return returnval;
1932
1933 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1934 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout);
1935 return returnval;
1936
1937 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1938 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout);
1939 return returnval;
1940
1941 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1942 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout);
1943 return returnval;
1944
1945 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1946 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout);
1947 return returnval;
1948
1949 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1950 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout);
1951 return returnval;
1952
1953 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1954 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout);
1955 return returnval;
1956
1957 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1958 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout);
1959 return returnval;
1960
1961 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1962 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout);
1963 return returnval;
1964
1965 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1966 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout);
1967 return returnval;
1968
1969 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1970 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout);
1971 return returnval;
1972
1973 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1974 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout);
1975 return returnval;
1976
1977 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1978 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1979 return returnval;
1980
1981 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1982 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout);
1983 return returnval;
1984
1985 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1986 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1987 return returnval;
1988
1989 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1990 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout);
1991 return returnval;
1992
1993 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1994 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout);
1995 return returnval;
1996
1997 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1998 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout);
1999 return returnval;
2000
2001 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2002 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout);
2003 return returnval;
2004
2005 def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2006 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout);
2007 return returnval;
2008
2009 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2010 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout);
2011 return returnval;
2012
2013 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2014 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout);
2015 return returnval;
2016
2017 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2018 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout);
2019 return returnval;
2020
2021 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2022 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout);
2023 return returnval;
2024
2025 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2026 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout);
2027 return returnval;
2028
2029 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2030 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout);
2031 return returnval;
2032
2033 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2034 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout);
2035 return returnval;
2036
2037 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2038 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout);
2039 return returnval;
2040
2041 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2042 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2043 return returnval;
2044
2045 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2046 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2047 return returnval;
2048
2049 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2050 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2051 return returnval;
2052
2053 def download_file_from_ftp_file(url):
2054 urlparts = urlparse.urlparse(url);
2055 file_name = os.path.basename(urlparts.path);
2056 file_dir = os.path.dirname(urlparts.path);
2057 if(urlparts.username is not None):
2058 ftp_username = urlparts.username;
2059 else:
2060 ftp_username = "anonymous";
2061 if(urlparts.password is not None):
2062 ftp_password = urlparts.password;
2063 elif(urlparts.password is None and urlparts.username=="anonymous"):
2064 ftp_password = "anonymous";
2065 else:
2066 ftp_password = "";
2067 if(urlparts.scheme=="ftp"):
2068 ftp = FTP();
2069 elif(urlparts.scheme=="ftps"):
2070 ftp = FTP_TLS();
2071 else:
2072 return False;
2073 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2074 return False;
2075 ftp_port = urlparts.port;
2076 if(urlparts.port is None):
2077 ftp_port = 21;
2078 try:
2079 ftp.connect(urlparts.hostname, ftp_port);
2080 except socket.gaierror:
2081 log.info("Error With URL "+httpurl);
2082 return False;
2083 except socket.timeout:
2084 log.info("Error With URL "+httpurl);
2085 return False;
2086 ftp.login(urlparts.username, urlparts.password);
2087 if(urlparts.scheme=="ftps"):
2088 ftp.prot_p();
2089 ftpfile = BytesIO();
2090 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2091 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2092 ftp.close();
2093 ftpfile.seek(0, 0);
2094 return ftpfile;
2095
2096 def download_file_from_ftp_string(url):
2097 ftpfile = download_file_from_ftp_file(url);
2098 return ftpfile.read();
2099
2100 def upload_file_to_ftp_file(ftpfile, url):
2101 urlparts = urlparse.urlparse(url);
2102 file_name = os.path.basename(urlparts.path);
2103 file_dir = os.path.dirname(urlparts.path);
2104 if(urlparts.username is not None):
2105 ftp_username = urlparts.username;
2106 else:
2107 ftp_username = "anonymous";
2108 if(urlparts.password is not None):
2109 ftp_password = urlparts.password;
2110 elif(urlparts.password is None and urlparts.username=="anonymous"):
2111 ftp_password = "anonymous";
2112 else:
2113 ftp_password = "";
2114 if(urlparts.scheme=="ftp"):
2115 ftp = FTP();
2116 elif(urlparts.scheme=="ftps"):
2117 ftp = FTP_TLS();
2118 else:
2119 return False;
2120 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2121 return False;
2122 ftp_port = urlparts.port;
2123 if(urlparts.port is None):
2124 ftp_port = 21;
2125 try:
2126 ftp.connect(urlparts.hostname, ftp_port);
2127 except socket.gaierror:
2128 log.info("Error With URL "+httpurl);
2129 return False;
2130 except socket.timeout:
2131 log.info("Error With URL "+httpurl);
2132 return False;
2133 ftp.login(urlparts.username, urlparts.password);
2134 if(urlparts.scheme=="ftps"):
2135 ftp.prot_p();
2136 ftp.storbinary("STOR "+urlparts.path, ftpfile);
2137 ftp.close();
2138 ftpfile.seek(0, 0);
2139 return ftpfile;
2140
2141 def upload_file_to_ftp_string(ftpstring, url):
2142 ftpfileo = BytesIO(ftpstring);
2143 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2144 ftpfileo.close();
2145 return ftpfile;
2146
2147 if(haveparamiko):
2148 def download_file_from_sftp_file(url):
2149 urlparts = urlparse.urlparse(url);
2150 file_name = os.path.basename(urlparts.path);
2151 file_dir = os.path.dirname(urlparts.path);
2152 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2153 return False;
2154 sftp_port = urlparts.port;
2155 if(urlparts.port is None):
2156 sftp_port = 22;
2157 else:
2158 sftp_port = urlparts.port;
2159 if(urlparts.username is not None):
2160 sftp_username = urlparts.username;
2161 else:
2162 sftp_username = "anonymous";
2163 if(urlparts.password is not None):
2164 sftp_password = urlparts.password;
2165 elif(urlparts.password is None and urlparts.username=="anonymous"):
2166 sftp_password = "anonymous";
2167 else:
2168 sftp_password = "";
2169 if(urlparts.scheme!="sftp"):
2170 return False;
2171 ssh = paramiko.SSHClient();
2172 ssh.load_system_host_keys();
2173 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2174 try:
2175 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2176 except paramiko.ssh_exception.SSHException:
2177 return False;
2178 except socket.gaierror:
2179 log.info("Error With URL "+httpurl);
2180 return False;
2181 except socket.timeout:
2182 log.info("Error With URL "+httpurl);
2183 return False;
2184 sftp = ssh.open_sftp();
2185 sftpfile = BytesIO();
2186 sftp.getfo(urlparts.path, sftpfile);
2187 sftp.close();
2188 ssh.close();
2189 sftpfile.seek(0, 0);
2190 return sftpfile;
2191 else:
2192 def download_file_from_sftp_file(url):
2193 return False;
2194
2195 if(haveparamiko):
2196 def download_file_from_sftp_string(url):
2197 sftpfile = download_file_from_sftp_file(url);
2198 return sftpfile.read();
2199 else:
2200 def download_file_from_ftp_string(url):
2201 return False;
2202
2203 if(haveparamiko):
2204 def upload_file_to_sftp_file(sftpfile, url):
2205 urlparts = urlparse.urlparse(url);
2206 file_name = os.path.basename(urlparts.path);
2207 file_dir = os.path.dirname(urlparts.path);
2208 sftp_port = urlparts.port;
2209 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2210 return False;
2211 if(urlparts.port is None):
2212 sftp_port = 22;
2213 else:
2214 sftp_port = urlparts.port;
2215 if(urlparts.username is not None):
2216 sftp_username = urlparts.username;
2217 else:
2218 sftp_username = "anonymous";
2219 if(urlparts.password is not None):
2220 sftp_password = urlparts.password;
2221 elif(urlparts.password is None and urlparts.username=="anonymous"):
2222 sftp_password = "anonymous";
2223 else:
2224 sftp_password = "";
2225 if(urlparts.scheme!="sftp"):
2226 return False;
2227 ssh = paramiko.SSHClient();
2228 ssh.load_system_host_keys();
2229 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2230 try:
2231 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2232 except paramiko.ssh_exception.SSHException:
2233 return False;
2234 except socket.gaierror:
2235 log.info("Error With URL "+httpurl);
2236 return False;
2237 except socket.timeout:
2238 log.info("Error With URL "+httpurl);
2239 return False;
2240 sftp = ssh.open_sftp();
2241 sftp.putfo(sftpfile, urlparts.path);
2242 sftp.close();
2243 ssh.close();
2244 sftpfile.seek(0, 0);
2245 return sftpfile;
2246 else:
2247 def upload_file_to_sftp_file(sftpfile, url):
2248 return False;
2249
2250 if(haveparamiko):
2251 def upload_file_to_sftp_string(sftpstring, url):
2252 sftpfileo = BytesIO(sftpstring);
2253 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2254 sftpfileo.close();
2255 return sftpfile;
2256 else:
2257 def upload_file_to_sftp_string(url):
2258 return False;
2259
2260
2261 if(havepysftp):
2262 def download_file_from_pysftp_file(url):
2263 urlparts = urlparse.urlparse(url);
2264 file_name = os.path.basename(urlparts.path);
2265 file_dir = os.path.dirname(urlparts.path);
2266 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2267 return False;
2268 sftp_port = urlparts.port;
2269 if(urlparts.port is None):
2270 sftp_port = 22;
2271 else:
2272 sftp_port = urlparts.port;
2273 if(urlparts.username is not None):
2274 sftp_username = urlparts.username;
2275 else:
2276 sftp_username = "anonymous";
2277 if(urlparts.password is not None):
2278 sftp_password = urlparts.password;
2279 elif(urlparts.password is None and urlparts.username=="anonymous"):
2280 sftp_password = "anonymous";
2281 else:
2282 sftp_password = "";
2283 if(urlparts.scheme!="sftp"):
2284 return False;
2285 try:
2286 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2287 except paramiko.ssh_exception.SSHException:
2288 return False;
2289 except socket.gaierror:
2290 log.info("Error With URL "+httpurl);
2291 return False;
2292 except socket.timeout:
2293 log.info("Error With URL "+httpurl);
2294 return False;
2295 sftp = ssh.open_sftp();
2296 sftpfile = BytesIO();
2297 sftp.getfo(urlparts.path, sftpfile);
2298 sftp.close();
2299 ssh.close();
2300 sftpfile.seek(0, 0);
2301 return sftpfile;
2302 else:
2303 def download_file_from_pysftp_file(url):
2304 return False;
2305
2306 if(havepysftp):
2307 def download_file_from_pysftp_string(url):
2308 sftpfile = download_file_from_pysftp_file(url);
2309 return sftpfile.read();
2310 else:
2311 def download_file_from_ftp_string(url):
2312 return False;
2313
2314 if(havepysftp):
2315 def upload_file_to_pysftp_file(sftpfile, url):
2316 urlparts = urlparse.urlparse(url);
2317 file_name = os.path.basename(urlparts.path);
2318 file_dir = os.path.dirname(urlparts.path);
2319 sftp_port = urlparts.port;
2320 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2321 return False;
2322 if(urlparts.port is None):
2323 sftp_port = 22;
2324 else:
2325 sftp_port = urlparts.port;
2326 if(urlparts.username is not None):
2327 sftp_username = urlparts.username;
2328 else:
2329 sftp_username = "anonymous";
2330 if(urlparts.password is not None):
2331 sftp_password = urlparts.password;
2332 elif(urlparts.password is None and urlparts.username=="anonymous"):
2333 sftp_password = "anonymous";
2334 else:
2335 sftp_password = "";
2336 if(urlparts.scheme!="sftp"):
2337 return False;
2338 try:
2339 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2340 except paramiko.ssh_exception.SSHException:
2341 return False;
2342 except socket.gaierror:
2343 log.info("Error With URL "+httpurl);
2344 return False;
2345 except socket.timeout:
2346 log.info("Error With URL "+httpurl);
2347 return False;
2348 sftp = ssh.open_sftp();
2349 sftp.putfo(sftpfile, urlparts.path);
2350 sftp.close();
2351 ssh.close();
2352 sftpfile.seek(0, 0);
2353 return sftpfile;
2354 else:
2355 def upload_file_to_pysftp_file(sftpfile, url):
2356 return False;
2357
2358 if(havepysftp):
2359 def upload_file_to_pysftp_string(sftpstring, url):
2360 sftpfileo = BytesIO(sftpstring);
2361 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2362 sftpfileo.close();
2363 return sftpfile;
2364 else:
2365 def upload_file_to_pysftp_string(url):
2366 return False;
python libary/module to download files.