pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, bz2, time, argparse, subprocess, socket, email.utils, datetime, time;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 try:
  24  from cgi import parse_qsl;
  25 except ImportError:
  26  from urlparse import parse_qsl;
  27 haverequests = False;
  28 try:
  29  import requests;
  30  haverequests = True;
  31 except ImportError:
  32  haverequests = False;
  33 havemechanize = False;
  34 try:
  35  import mechanize;
  36  havemechanize = True;
  37 except ImportError:
  38  havemechanize = False;
  39 havepycurl = False;
  40 try:
  41  import pycurl;
  42  havepycurl = True;
  43 except ImportError:
  44  havepycurl = False;
  45 haveparamiko = False;
  46 try:
  47  import paramiko;
  48  haveparamiko = True;
  49 except ImportError:
  50  haveparamiko = False;
  51 havepysftp = False;
  52 try:
  53  import pysftp;
  54  havepysftp = True;
  55 except ImportError:
  56  havepysftp = False;
  57 haveurllib3 = False;
  58 try:
  59  import urllib3;
  60  haveurllib3 = True;
  61 except ImportError:
  62  haveurllib3 = False;
  63 havehttplib2 = False;
  64 try:
  65  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  66  havehttplib2 = True;
  67 except ImportError:
  68  havehttplib2 = False;
  69 havehttpx = False;
  70 try:
  71  import httpx;
  72  havehttpx = True;
  73 except ImportError:
  74  havehttpx = False;
  75 havehttpcore = False;
  76 try:
  77  import httpcore;
  78  havehttpcore = True;
  79 except ImportError:
  80  havehttpcore = False;
  81 haveaiohttp = False;
  82 try:
  83  import aiohttp;
  84  haveaiohttp = True;
  85 except ImportError:
  86  haveaiohttp = False;
  87 havebrotli = False;
  88 try:
  89  import brotli;
  90  havebrotli = True;
  91 except ImportError:
  92  havebrotli = False;
  93 havezstd = False;
  94 try:
  95  import zstandard;
  96  havezstd = True;
  97 except ImportError:
  98  havezstd = False;
  99 havelzma = False;
 100 try:
 101  import lzma;
 102  havelzma = True;
 103 except ImportError:
 104  havelzma = False;
 105 if(sys.version[0]=="2"):
 106  try:
 107   from io import StringIO, BytesIO;
 108  except ImportError:
 109   try:
 110    from cStringIO import StringIO;
 111    from cStringIO import StringIO as BytesIO;
 112   except ImportError:
 113    from StringIO import StringIO;
 114    from StringIO import StringIO as BytesIO;
 115  # From http://python-future.org/compatible_idioms.html
 116  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
 117  from urllib import urlencode;
 118  from urllib import urlopen as urlopenalt;
 119  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
 120  import urlparse, cookielib;
 121  from httplib import HTTPConnection, HTTPSConnection;
 122 if(sys.version[0]>="3"):
 123  from io import StringIO, BytesIO;
 124  # From http://python-future.org/compatible_idioms.html
 125  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
 126  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
 127  from urllib.error import HTTPError, URLError;
 128  import urllib.parse as urlparse;
 129  import http.cookiejar as cookielib;
 130  from http.client import HTTPConnection, HTTPSConnection;
 131
 132 __program_name__ = "PyWWW-Get";
 133 __program_alt_name__ = "PyWWWGet";
 134 __program_small_name__ = "wwwget";
 135 __project__ = __program_name__;
 136 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
 137 __version_info__ = (2, 0, 2, "RC 1", 1);
 138 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
 139 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
 140 __revision__ = __version_info__[3];
 141 __revision_id__ = "$Id$";
 142 if(__version_info__[4] is not None):
 143  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
 144 if(__version_info__[4] is None):
 145  __version_date_plusrc__ = __version_date__;
 146 if(__version_info__[3] is not None):
 147  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 148 if(__version_info__[3] is None):
 149  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 150
 151 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 152 tmpfilesuffix = "-";
 153 pytempdir = tempfile.gettempdir();
 154
 155 PyBitness = platform.architecture();
 156 if(PyBitness=="32bit" or PyBitness=="32"):
 157  PyBitness = "32";
 158 elif(PyBitness=="64bit" or PyBitness=="64"):
 159  PyBitness = "64";
 160 else:
 161  PyBitness = "32";
 162
 163 compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2'];
 164 if(havebrotli):
 165  compression_supported_list.append('br');
 166 if(havezstd):
 167  compression_supported_list.append('zstd');
 168 if(havelzma):
 169  compression_supported_list.append('lzma');
 170  compression_supported_list.append('xz');
 171 compression_supported = ', '.join(compression_supported_list);
 172
 173 geturls_cj = cookielib.CookieJar();
 174 windowsNT4_ua_string = "Windows NT 4.0";
 175 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 176 windows2k_ua_string = "Windows NT 5.0";
 177 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 178 windowsXP_ua_string = "Windows NT 5.1";
 179 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 180 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 181 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 182 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 183 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 184 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 185 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 186 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 187 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 188 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 189 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 190 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 191 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 192 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 193 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 194 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 195 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 196 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 197 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 198 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 199 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 200 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 201 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 202 if(platform.python_implementation()!=""):
 203  py_implementation = platform.python_implementation();
 204 if(platform.python_implementation()==""):
 205  py_implementation = "Python";
 206 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 207 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 208 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 209 geturls_ua = geturls_ua_firefox_windows7;
 210 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 211 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 212 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 213 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 214 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 215 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 216 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 217 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 218 geturls_headers_opera_windows7.update(windows7_ua_addon);
 219 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 220 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 221 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 222 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 223 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 224 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 225 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 226 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 227 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 228 geturls_headers = geturls_headers_firefox_windows7;
 229 geturls_download_sleep = 0;
 230
 231 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 232  if(outtype=="print" and dbgenable):
 233   print(dbgtxt);
 234   return True;
 235  elif(outtype=="log" and dbgenable):
 236   logging.info(dbgtxt);
 237   return True;
 238  elif(outtype=="warning" and dbgenable):
 239   logging.warning(dbgtxt);
 240   return True;
 241  elif(outtype=="error" and dbgenable):
 242   logging.error(dbgtxt);
 243   return True;
 244  elif(outtype=="critical" and dbgenable):
 245   logging.critical(dbgtxt);
 246   return True;
 247  elif(outtype=="exception" and dbgenable):
 248   logging.exception(dbgtxt);
 249   return True;
 250  elif(outtype=="logalt" and dbgenable):
 251   logging.log(dgblevel, dbgtxt);
 252   return True;
 253  elif(outtype=="debug" and dbgenable):
 254   logging.debug(dbgtxt);
 255   return True;
 256  elif(not dbgenable):
 257   return True;
 258  else:
 259   return False;
 260  return False;
 261
 262 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 263  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 264  if(not dbgout):
 265   return False;
 266  return dbgtxt;
 267
 268 def add_url_param(url, **params):
 269  n=3;
 270  parts = list(urlparse.urlsplit(url));
 271  d = dict(parse_qsl(parts[n])); # use cgi.parse_qs for list values
 272  d.update(params);
 273  parts[n]=urlencode(d);
 274  return urlparse.urlunsplit(parts);
 275
 276 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 277 def which_exec(execfile):
 278  for path in os.environ["PATH"].split(":"):
 279   if os.path.exists(path + "/" + execfile):
 280    return path + "/" + execfile;
 281
 282 def listize(varlist):
 283  il = 0;
 284  ix = len(varlist);
 285  ilx = 1;
 286  newlistreg = {};
 287  newlistrev = {};
 288  newlistfull = {};
 289  while(il < ix):
 290   newlistreg.update({ilx: varlist[il]});
 291   newlistrev.update({varlist[il]: ilx});
 292   ilx = ilx + 1;
 293   il = il + 1;
 294  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 295  return newlistfull;
 296
 297 def twolistize(varlist):
 298  il = 0;
 299  ix = len(varlist);
 300  ilx = 1;
 301  newlistnamereg = {};
 302  newlistnamerev = {};
 303  newlistdescreg = {};
 304  newlistdescrev = {};
 305  newlistfull = {};
 306  while(il < ix):
 307   newlistnamereg.update({ilx: varlist[il][0].strip()});
 308   newlistnamerev.update({varlist[il][0].strip(): ilx});
 309   newlistdescreg.update({ilx: varlist[il][1].strip()});
 310   newlistdescrev.update({varlist[il][1].strip(): ilx});
 311   ilx = ilx + 1;
 312   il = il + 1;
 313  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 314  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 315  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 316  return newlistfull;
 317
 318 def arglistize(proexec, *varlist):
 319  il = 0;
 320  ix = len(varlist);
 321  ilx = 1;
 322  newarglist = [proexec];
 323  while(il < ix):
 324   if varlist[il][0] is not None:
 325    newarglist.append(varlist[il][0]);
 326   if varlist[il][1] is not None:
 327    newarglist.append(varlist[il][1]);
 328   il = il + 1;
 329  return newarglist;
 330
 331 def fix_header_names(header_dict):
 332  if(sys.version[0]=="2"):
 333   header_dict =  {k.title(): v for k, v in header_dict.iteritems()};
 334  if(sys.version[0]>="3"):
 335   header_dict =  {k.title(): v for k, v in header_dict.items()};
 336  return header_dict;
 337
 338 # hms_string by ArcGIS Python Recipes
 339 # https://arcpy.wordpress.com/2012/04/20/146/
 340 def hms_string(sec_elapsed):
 341  h = int(sec_elapsed / (60 * 60));
 342  m = int((sec_elapsed % (60 * 60)) / 60);
 343  s = sec_elapsed % 60.0;
 344  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 345
 346 # get_readable_size by Lipis
 347 # http://stackoverflow.com/posts/14998888/revisions
 348 def get_readable_size(bytes, precision=1, unit="IEC"):
 349  unit = unit.upper();
 350  if(unit!="IEC" and unit!="SI"):
 351   unit = "IEC";
 352  if(unit=="IEC"):
 353   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 354   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 355   unitsize = 1024.0;
 356  if(unit=="SI"):
 357   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 358   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 359   unitsize = 1000.0;
 360  return_val = {};
 361  orgbytes = bytes;
 362  for unit in units:
 363   if abs(bytes) < unitsize:
 364    strformat = "%3."+str(precision)+"f%s";
 365    pre_return_val = (strformat % (bytes, unit));
 366    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 367    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 368    alt_return_val = pre_return_val.split();
 369    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 370    return return_val;
 371   bytes /= unitsize;
 372  strformat = "%."+str(precision)+"f%s";
 373  pre_return_val = (strformat % (bytes, "YiB"));
 374  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 375  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 376  alt_return_val = pre_return_val.split();
 377  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 378  return return_val;
 379
 380 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 381  unit = unit.upper();
 382  usehashtypes = usehashtypes.lower();
 383  getfilesize = os.path.getsize(infile);
 384  return_val = get_readable_size(getfilesize, precision, unit);
 385  if(usehashes):
 386   hashtypelist = usehashtypes.split(",");
 387   openfile = open(infile, "rb");
 388   filecontents = openfile.read();
 389   openfile.close();
 390   listnumcount = 0;
 391   listnumend = len(hashtypelist);
 392   while(listnumcount < listnumend):
 393    hashtypelistlow = hashtypelist[listnumcount].strip();
 394    hashtypelistup = hashtypelistlow.upper();
 395    filehash = hashlib.new(hashtypelistup);
 396    filehash.update(filecontents);
 397    filegethash = filehash.hexdigest();
 398    return_val.update({hashtypelistup: filegethash});
 399    listnumcount += 1;
 400  return return_val;
 401
 402 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 403  unit = unit.upper();
 404  usehashtypes = usehashtypes.lower();
 405  getfilesize = len(instring);
 406  return_val = get_readable_size(getfilesize, precision, unit);
 407  if(usehashes):
 408   hashtypelist = usehashtypes.split(",");
 409   listnumcount = 0;
 410   listnumend = len(hashtypelist);
 411   while(listnumcount < listnumend):
 412    hashtypelistlow = hashtypelist[listnumcount].strip();
 413    hashtypelistup = hashtypelistlow.upper();
 414    filehash = hashlib.new(hashtypelistup);
 415    if(sys.version[0]=="2"):
 416     filehash.update(instring);
 417    if(sys.version[0]>="3"):
 418     filehash.update(instring.encode('utf-8'));
 419    filegethash = filehash.hexdigest();
 420    return_val.update({hashtypelistup: filegethash});
 421    listnumcount += 1;
 422  return return_val;
 423
 424 def http_status_to_reason(code):
 425     reasons = {
 426         100: 'Continue',
 427         101: 'Switching Protocols',
 428         102: 'Processing',
 429         200: 'OK',
 430         201: 'Created',
 431         202: 'Accepted',
 432         203: 'Non-Authoritative Information',
 433         204: 'No Content',
 434         205: 'Reset Content',
 435         206: 'Partial Content',
 436         207: 'Multi-Status',
 437         208: 'Already Reported',
 438         226: 'IM Used',
 439         300: 'Multiple Choices',
 440         301: 'Moved Permanently',
 441         302: 'Found',
 442         303: 'See Other',
 443         304: 'Not Modified',
 444         305: 'Use Proxy',
 445         307: 'Temporary Redirect',
 446         308: 'Permanent Redirect',
 447         400: 'Bad Request',
 448         401: 'Unauthorized',
 449         402: 'Payment Required',
 450         403: 'Forbidden',
 451         404: 'Not Found',
 452         405: 'Method Not Allowed',
 453         406: 'Not Acceptable',
 454         407: 'Proxy Authentication Required',
 455         408: 'Request Timeout',
 456         409: 'Conflict',
 457         410: 'Gone',
 458         411: 'Length Required',
 459         412: 'Precondition Failed',
 460         413: 'Payload Too Large',
 461         414: 'URI Too Long',
 462         415: 'Unsupported Media Type',
 463         416: 'Range Not Satisfiable',
 464         417: 'Expectation Failed',
 465         421: 'Misdirected Request',
 466         422: 'Unprocessable Entity',
 467         423: 'Locked',
 468         424: 'Failed Dependency',
 469         426: 'Upgrade Required',
 470         428: 'Precondition Required',
 471         429: 'Too Many Requests',
 472         431: 'Request Header Fields Too Large',
 473         451: 'Unavailable For Legal Reasons',
 474         500: 'Internal Server Error',
 475         501: 'Not Implemented',
 476         502: 'Bad Gateway',
 477         503: 'Service Unavailable',
 478         504: 'Gateway Timeout',
 479         505: 'HTTP Version Not Supported',
 480         506: 'Variant Also Negotiates',
 481         507: 'Insufficient Storage',
 482         508: 'Loop Detected',
 483         510: 'Not Extended',
 484         511: 'Network Authentication Required'
 485     }
 486     return reasons.get(code, 'Unknown Status Code');
 487
 488 def ftp_status_to_reason(code):
 489     reasons = {
 490         110: 'Restart marker reply',
 491         120: 'Service ready in nnn minutes',
 492         125: 'Data connection already open; transfer starting',
 493         150: 'File status okay; about to open data connection',
 494         200: 'Command okay',
 495         202: 'Command not implemented, superfluous at this site',
 496         211: 'System status, or system help reply',
 497         212: 'Directory status',
 498         213: 'File status',
 499         214: 'Help message',
 500         215: 'NAME system type',
 501         220: 'Service ready for new user',
 502         221: 'Service closing control connection',
 503         225: 'Data connection open; no transfer in progress',
 504         226: 'Closing data connection',
 505         227: 'Entering Passive Mode',
 506         230: 'User logged in, proceed',
 507         250: 'Requested file action okay, completed',
 508         257: '"PATHNAME" created',
 509         331: 'User name okay, need password',
 510         332: 'Need account for login',
 511         350: 'Requested file action pending further information',
 512         421: 'Service not available, closing control connection',
 513         425: 'Can\'t open data connection',
 514         426: 'Connection closed; transfer aborted',
 515         450: 'Requested file action not taken',
 516         451: 'Requested action aborted. Local error in processing',
 517         452: 'Requested action not taken. Insufficient storage space in system',
 518         500: 'Syntax error, command unrecognized',
 519         501: 'Syntax error in parameters or arguments',
 520         502: 'Command not implemented',
 521         503: 'Bad sequence of commands',
 522         504: 'Command not implemented for that parameter',
 523         530: 'Not logged in',
 524         532: 'Need account for storing files',
 525         550: 'Requested action not taken. File unavailable',
 526         551: 'Requested action aborted. Page type unknown',
 527         552: 'Requested file action aborted. Exceeded storage allocation',
 528         553: 'Requested action not taken. File name not allowed'
 529     };
 530     return reasons.get(code, 'Unknown Status Code');
 531
 532 def sftp_status_to_reason(code):
 533     reasons = {
 534         0: 'SSH_FX_OK',
 535         1: 'SSH_FX_EOF',
 536         2: 'SSH_FX_NO_SUCH_FILE',
 537         3: 'SSH_FX_PERMISSION_DENIED',
 538         4: 'SSH_FX_FAILURE',
 539         5: 'SSH_FX_BAD_MESSAGE',
 540         6: 'SSH_FX_NO_CONNECTION',
 541         7: 'SSH_FX_CONNECTION_LOST',
 542         8: 'SSH_FX_OP_UNSUPPORTED'
 543     };
 544     return reasons.get(code, 'Unknown Status Code');
 545
 546 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 547  if isinstance(headers, dict):
 548   returnval = [];
 549   if(sys.version[0]=="2"):
 550    for headkey, headvalue in headers.iteritems():
 551     returnval.append((headkey, headvalue));
 552   if(sys.version[0]>="3"):
 553    for headkey, headvalue in headers.items():
 554     returnval.append((headkey, headvalue));
 555  elif isinstance(headers, list):
 556   returnval = headers;
 557  else:
 558   returnval = False;
 559  return returnval;
 560
 561 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 562  if isinstance(headers, dict):
 563   returnval = [];
 564   if(sys.version[0]=="2"):
 565    for headkey, headvalue in headers.iteritems():
 566     returnval.append(headkey+": "+headvalue);
 567   if(sys.version[0]>="3"):
 568    for headkey, headvalue in headers.items():
 569     returnval.append(headkey+": "+headvalue);
 570  elif isinstance(headers, list):
 571   returnval = headers;
 572  else:
 573   returnval = False;
 574  return returnval;
 575
 576 def make_http_headers_from_pycurl_to_dict(headers):
 577  header_dict = {};
 578  headers = headers.strip().split('\r\n');
 579  for header in headers:
 580   parts = header.split(': ', 1)
 581   if(len(parts) == 2):
 582    key, value = parts;
 583    header_dict[key.title()] = value;
 584  return header_dict;
 585
 586 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 587  if isinstance(headers, list):
 588   returnval = {};
 589   mli = 0;
 590   mlil = len(headers);
 591   while(mli<mlil):
 592    returnval.update({headers[mli][0]: headers[mli][1]});
 593    mli = mli + 1;
 594  elif isinstance(headers, dict):
 595   returnval = headers;
 596  else:
 597   returnval = False;
 598  return returnval;
 599
 600 def get_httplib_support(checkvalue=None):
 601  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 602  returnval = [];
 603  returnval.append("ftp");
 604  returnval.append("httplib");
 605  if(havehttplib2):
 606   returnval.append("httplib2");
 607  returnval.append("urllib");
 608  if(haveurllib3):
 609   returnval.append("urllib3");
 610   returnval.append("request3");
 611  returnval.append("request");
 612  if(haverequests):
 613   returnval.append("requests");
 614  if(haveaiohttp):
 615   returnval.append("aiohttp");
 616  if(havehttpx):
 617   returnval.append("httpx");
 618   returnval.append("httpx2");
 619  if(havemechanize):
 620   returnval.append("mechanize");
 621  if(havepycurl):
 622   returnval.append("pycurl");
 623   if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 624    returnval.append("pycurl2");
 625   if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
 626    returnval.append("pycurl3");
 627  if(haveparamiko):
 628   returnval.append("sftp");
 629  if(havepysftp):
 630   returnval.append("pysftp");
 631  if(not checkvalue is None):
 632   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 633    checkvalue = "urllib";
 634   if(checkvalue=="httplib1"):
 635    checkvalue = "httplib";
 636   if(checkvalue in returnval):
 637    returnval = True;
 638   else:
 639    returnval = False;
 640  return returnval;
 641
 642 def check_httplib_support(checkvalue="urllib"):
 643  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 644   checkvalue = "urllib";
 645  if(checkvalue=="httplib1"):
 646   checkvalue = "httplib";
 647  returnval = get_httplib_support(checkvalue);
 648  return returnval;
 649
 650 def get_httplib_support_list():
 651  returnval = get_httplib_support(None);
 652  return returnval;
 653
 654 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
 655  global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 656  if(sleep<0):
 657   sleep = geturls_download_sleep;
 658  if(timeout<=0):
 659   timeout = 10;
 660  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
 661   httplibuse = "urllib";
 662  if(httplibuse=="httplib1"):
 663   httplibuse = "httplib";
 664  if(not haverequests and httplibuse=="requests"):
 665   httplibuse = "urllib";
 666  if(not haveaiohttp and httplibuse=="aiohttp"):
 667   httplibuse = "urllib";
 668  if(not havehttpx and httplibuse=="httpx"):
 669   httplibuse = "urllib";
 670  if(not havehttpx and httplibuse=="httpx2"):
 671   httplibuse = "urllib";
 672  if(not havehttpcore and httplibuse=="httpcore"):
 673   httplibuse = "urllib";
 674  if(not havehttpcore and httplibuse=="httpcore2"):
 675   httplibuse = "urllib";
 676  if(not havemechanize and httplibuse=="mechanize"):
 677   httplibuse = "urllib";
 678  if(not havepycurl and httplibuse=="pycurl"):
 679   httplibuse = "urllib";
 680  if(not havepycurl and httplibuse=="pycurl2"):
 681   httplibuse = "urllib";
 682  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 683   httplibuse = "pycurl";
 684  if(not havepycurl and httplibuse=="pycurl3"):
 685   httplibuse = "urllib";
 686  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 687   httplibuse = "pycurl2";
 688  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 689   httplibuse = "pycurl";
 690  if(not havehttplib2 and httplibuse=="httplib2"):
 691   httplibuse = "httplib";
 692  if(not haveparamiko and httplibuse=="sftp"):
 693   httplibuse = "ftp";
 694  if(not havepysftp and httplibuse=="pysftp"):
 695   httplibuse = "ftp";
 696  urlparts = urlparse.urlparse(httpurl);
 697  if(isinstance(httpheaders, list)):
 698   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 699  httpheaders = fix_header_names(httpheaders);
 700  if(httpuseragent is not None):
 701   if('User-Agent' in httpheaders):
 702    httpheaders['User-Agent'] = httpuseragent;
 703   else:
 704    httpuseragent.update({'User-Agent': httpuseragent});
 705  if(httpreferer is not None):
 706   if('Referer' in httpheaders):
 707    httpheaders['Referer'] = httpreferer;
 708   else:
 709    httpuseragent.update({'Referer': httpreferer});
 710  if(urlparts.username is not None or urlparts.password is not None):
 711   if(sys.version[0]=="2"):
 712    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
 713   if(sys.version[0]>="3"):
 714    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 715   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 716  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 717  if(httplibuse=="urllib" or httplibuse=="mechanize"):
 718   if(isinstance(httpheaders, dict)):
 719    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 720  if(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
 721   if(isinstance(httpheaders, dict)):
 722    httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
 723  geturls_opener.addheaders = httpheaders;
 724  time.sleep(sleep);
 725  if(postdata is not None and not isinstance(postdata, dict)):
 726   postdata = urlencode(postdata);
 727  if(httplibuse=="urllib" or httplibuse=="request"):
 728   geturls_request = Request(httpurl);
 729   try:
 730    if(httpmethod=="GET"):
 731     geturls_text = geturls_opener.open(geturls_request);
 732    elif(httpmethod=="POST"):
 733     geturls_text = geturls_opener.open(geturls_request, data=postdata);
 734    else:
 735     geturls_text = geturls_opener.open(geturls_request);
 736   except HTTPError as geturls_text_error:
 737    geturls_text = geturls_text_error;
 738    log.info("Error With URL "+httpurl);
 739   except URLError:
 740    log.info("Error With URL "+httpurl);
 741    return False;
 742   except socket.timeout:
 743    log.info("Error With URL "+httpurl);
 744    return False;
 745   httpcodeout = geturls_text.getcode();
 746   try:
 747    httpcodereason = geturls_text.reason;
 748   except AttributeError:
 749    httpcodereason = http_status_to_reason(geturls_text.getcode());
 750   try:
 751    httpversionout = geturls_text.version;
 752   except AttributeError:
 753    httpversionout = "1.1";
 754   httpmethodout = geturls_request.get_method();
 755   httpurlout = geturls_text.geturl();
 756   httpheaderout = geturls_text.info();
 757   httpheadersentout = httpheaders;
 758  elif(httplibuse=="httplib"):
 759   if(urlparts[0]=="http"):
 760    httpconn = HTTPConnection(urlparts[1], timeout=timeout);
 761   elif(urlparts[0]=="https"):
 762    httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
 763   else:
 764    return False;
 765   if(postdata is not None and not isinstance(postdata, dict)):
 766    postdata = urlencode(postdata);
 767   try:
 768    if(httpmethod=="GET"):
 769     httpconn.request("GET", urlparts[2], headers=httpheaders);
 770    elif(httpmethod=="POST"):
 771     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 772    else:
 773     httpconn.request("GET", urlparts[2], headers=httpheaders);
 774   except socket.timeout:
 775    log.info("Error With URL "+httpurl);
 776    return False;
 777   except socket.gaierror:
 778    log.info("Error With URL "+httpurl);
 779    return False;
 780   except BlockingIOError:
 781    log.info("Error With URL "+httpurl);
 782    return False;
 783   geturls_text = httpconn.getresponse();
 784   httpcodeout = geturls_text.status;
 785   httpcodereason = geturls_text.reason;
 786   if(geturls_text.version=="10"):
 787    httpversionout = "1.0";
 788   else:
 789    httpversionout = "1.1";
 790   httpmethodout = geturls_text._method;
 791   httpurlout = httpurl;
 792   httpheaderout = geturls_text.getheaders();
 793   httpheadersentout = httpheaders;
 794  elif(httplibuse=="httplib2"):
 795   if(urlparts[0]=="http"):
 796    httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
 797   elif(urlparts[0]=="https"):
 798    httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
 799   else:
 800    return False;
 801   if(postdata is not None and not isinstance(postdata, dict)):
 802    postdata = urlencode(postdata);
 803   try:
 804    if(httpmethod=="GET"):
 805     httpconn.request("GET", urlparts[2], headers=httpheaders);
 806    elif(httpmethod=="POST"):
 807     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 808    else:
 809     httpconn.request("GET", urlparts[2], headers=httpheaders);
 810   except socket.timeout:
 811    log.info("Error With URL "+httpurl);
 812    return False;
 813   except socket.gaierror:
 814    log.info("Error With URL "+httpurl);
 815    return False;
 816   except BlockingIOError:
 817    log.info("Error With URL "+httpurl);
 818    return False;
 819   geturls_text = httpconn.getresponse();
 820   httpcodeout = geturls_text.status;
 821   httpcodereason = geturls_text.reason;
 822   if(geturls_text.version=="10"):
 823    httpversionout = "1.0";
 824   else:
 825    httpversionout = "1.1";
 826   httpmethodout = httpmethod;
 827   httpurlout = httpurl;
 828   httpheaderout = geturls_text.getheaders();
 829   httpheadersentout = httpheaders;
 830  elif(httplibuse=="urllib3" or httplibuse=="request3"):
 831   timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
 832   urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
 833   try:
 834    if(httpmethod=="GET"):
 835     geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 836    elif(httpmethod=="POST"):
 837     geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
 838    else:
 839     geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 840   except urllib3.exceptions.ConnectTimeoutError:
 841    log.info("Error With URL "+httpurl);
 842    return False;
 843   except urllib3.exceptions.ConnectError:
 844    log.info("Error With URL "+httpurl);
 845    return False;
 846   except urllib3.exceptions.MaxRetryError:
 847    log.info("Error With URL "+httpurl);
 848    return False;
 849   except socket.timeout:
 850    log.info("Error With URL "+httpurl);
 851    return False;
 852   except ValueError:
 853    log.info("Error With URL "+httpurl);
 854    return False;
 855   httpcodeout = geturls_text.status;
 856   httpcodereason = geturls_text.reason;
 857   if(geturls_text.version=="10"):
 858    httpversionout = "1.0";
 859   else:
 860    httpversionout = "1.1";
 861   httpmethodout = httpmethod;
 862   httpurlout = geturls_text.geturl();
 863   httpheaderout = geturls_text.info();
 864   httpheadersentout = httpheaders;
 865  elif(httplibuse=="requests"):
 866   try:
 867    reqsession = requests.Session();
 868    if(httpmethod=="GET"):
 869     geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 870    elif(httpmethod=="POST"):
 871     geturls_text = reqsession.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 872    else:
 873     geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 874   except requests.exceptions.ConnectTimeout:
 875    log.info("Error With URL "+httpurl);
 876    return False;
 877   except requests.exceptions.ConnectError:
 878    log.info("Error With URL "+httpurl);
 879    return False;
 880   except socket.timeout:
 881    log.info("Error With URL "+httpurl);
 882    return False;
 883   httpcodeout = geturls_text.status_code;
 884   httpcodereason = geturls_text.reason;
 885   if(geturls_text.raw.version=="10"):
 886    httpversionout = "1.0";
 887   else:
 888    httpversionout = "1.1";
 889   httpmethodout = httpmethod;
 890   httpurlout = geturls_text.url;
 891   httpheaderout = geturls_text.headers;
 892   httpheadersentout = geturls_text.request.headers;
 893  elif(httplibuse=="aiohttp"):
 894   try:
 895    reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize);
 896    if(httpmethod=="GET"):
 897     geturls_text = reqsession.get(httpurl);
 898    elif(httpmethod=="POST"):
 899     geturls_text = reqsession.post(httpurl, data=postdata);
 900    else:
 901     geturls_text = reqsession.get(httpurl);
 902   except aiohttp.exceptions.ConnectTimeout:
 903    log.info("Error With URL "+httpurl);
 904    return False;
 905   except aiohttp.exceptions.ConnectError:
 906    log.info("Error With URL "+httpurl);
 907    return False;
 908   except socket.timeout:
 909    log.info("Error With URL "+httpurl);
 910    return False;
 911   httpcodeout = geturls_text.status;
 912   httpcodereason = geturls_text.reason;
 913   httpversionout = geturls_text.version;
 914   httpmethodout = geturls_text.method;
 915   httpurlout = geturls_text.url;
 916   httpheaderout = geturls_text.headers;
 917   httpheadersentout = geturls_text.request_info.headers;
 918  elif(httplibuse=="httpx"):
 919   try:
 920    if(httpmethod=="GET"):
 921     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 922     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 923    elif(httpmethod=="POST"):
 924     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 925     geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 926    else:
 927     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 928     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 929   except httpx.ConnectTimeout:
 930    log.info("Error With URL "+httpurl);
 931    return False;
 932   except httpx.ConnectError:
 933    log.info("Error With URL "+httpurl);
 934    return False;
 935   except socket.timeout:
 936    log.info("Error With URL "+httpurl);
 937    return False;
 938   httpcodeout = geturls_text.status_code;
 939   try:
 940    httpcodereason = geturls_text.reason_phrase;
 941   except:
 942    httpcodereason = http_status_to_reason(geturls_text.status_code);
 943   httpversionout = geturls_text.http_version;
 944   httpmethodout = httpmethod;
 945   httpurlout = str(geturls_text.url);
 946   httpheaderout = geturls_text.headers;
 947   httpheadersentout = geturls_text.request.headers;
 948  elif(httplibuse=="httpx2"):
 949   try:
 950    if(httpmethod=="GET"):
 951     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 952     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 953    elif(httpmethod=="POST"):
 954     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 955     geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 956    else:
 957     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 958     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 959   except httpx.ConnectTimeout:
 960    log.info("Error With URL "+httpurl);
 961    return False;
 962   except httpx.ConnectError:
 963    log.info("Error With URL "+httpurl);
 964    return False;
 965   except socket.timeout:
 966    log.info("Error With URL "+httpurl);
 967    return False;
 968   httpcodeout = geturls_text.status_code;
 969   try:
 970    httpcodereason = geturls_text.reason_phrase;
 971   except:
 972    httpcodereason = http_status_to_reason(geturls_text.status_code);
 973   httpversionout = geturls_text.http_version;
 974   httpmethodout = httpmethod;
 975   httpurlout = str(geturls_text.url);
 976   httpheaderout = geturls_text.headers;
 977   httpheadersentout = geturls_text.request.headers;
 978  elif(httplibuse=="httpcore"):
 979   try:
 980    if(httpmethod=="GET"):
 981     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 982     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 983    elif(httpmethod=="POST"):
 984     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 985     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
 986    else:
 987     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 988     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 989   except httpcore.ConnectTimeout:
 990    log.info("Error With URL "+httpurl);
 991    return False;
 992   except httpcore.ConnectError:
 993    log.info("Error With URL "+httpurl);
 994    return False;
 995   except socket.timeout:
 996    log.info("Error With URL "+httpurl);
 997    return False;
 998   httpcodeout = geturls_text.status;
 999   httpcodereason = http_status_to_reason(geturls_text.status);
1000   httpversionout = "1.1";
1001   httpmethodout = httpmethod;
1002   httpurlout = str(httpurl);
1003   httpheaderout = geturls_text.headers;
1004   httpheadersentout = httpheaders;
1005  elif(httplibuse=="httpcore2"):
1006   try:
1007    if(httpmethod=="GET"):
1008     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1009     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1010    elif(httpmethod=="POST"):
1011     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1012     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1013    else:
1014     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1015     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1016   except httpcore.ConnectTimeout:
1017    log.info("Error With URL "+httpurl);
1018    return False;
1019   except httpcore.ConnectError:
1020    log.info("Error With URL "+httpurl);
1021    return False;
1022   except socket.timeout:
1023    log.info("Error With URL "+httpurl);
1024    return False;
1025   httpcodeout = geturls_text.status;
1026   httpcodereason = http_status_to_reason(geturls_text.status);
1027   httpversionout = "1.1";
1028   httpmethodout = httpmethod;
1029   httpurlout = str(httpurl);
1030   httpheaderout = geturls_text.headers;
1031   httpheadersentout = httpheaders;
1032  elif(httplibuse=="mechanize"):
1033   geturls_opener = mechanize.Browser();
1034   if(isinstance(httpheaders, dict)):
1035    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1036   time.sleep(sleep);
1037   geturls_opener.addheaders = httpheaders;
1038   geturls_opener.set_cookiejar(httpcookie);
1039   geturls_opener.set_handle_robots(False);
1040   if(postdata is not None and not isinstance(postdata, dict)):
1041    postdata = urlencode(postdata);
1042   try:
1043    if(httpmethod=="GET"):
1044     geturls_text = geturls_opener.open(httpurl);
1045    elif(httpmethod=="POST"):
1046     geturls_text = geturls_opener.open(httpurl, data=postdata);
1047    else:
1048     geturls_text = geturls_opener.open(httpurl);
1049   except mechanize.HTTPError as geturls_text_error:
1050    geturls_text = geturls_text_error;
1051    log.info("Error With URL "+httpurl);
1052   except URLError:
1053    log.info("Error With URL "+httpurl);
1054    return False;
1055   except socket.timeout:
1056    log.info("Error With URL "+httpurl);
1057    return False;
1058   httpcodeout = geturls_text.code;
1059   httpcodereason = geturls_text.msg;
1060   httpversionout = "1.1";
1061   httpmethodout = httpmethod;
1062   httpurlout = geturls_text.geturl();
1063   httpheaderout = geturls_text.info();
1064   reqhead = geturls_opener.request;
1065   httpheadersentout = reqhead.header_items();
1066  elif(httplibuse=="pycurl"):
1067   retrieved_body = BytesIO();
1068   retrieved_headers = BytesIO();
1069   try:
1070    if(httpmethod=="GET"):
1071     geturls_text = pycurl.Curl();
1072     geturls_text.setopt(geturls_text.URL, httpurl);
1073     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1074     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1075     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1076     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1077     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1078     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1079     geturls_text.perform();
1080    elif(httpmethod=="POST"):
1081     geturls_text = pycurl.Curl();
1082     geturls_text.setopt(geturls_text.URL, httpurl);
1083     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1084     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1085     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1086     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1087     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1088     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1089     geturls_text.setopt(geturls_text.POST, True);
1090     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1091     geturls_text.perform();
1092    else:
1093     geturls_text = pycurl.Curl();
1094     geturls_text.setopt(geturls_text.URL, httpurl);
1095     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1096     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1097     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1098     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1099     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1100     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1101     geturls_text.perform();
1102    retrieved_headers.seek(0);
1103    if(sys.version[0]=="2"):
1104     pycurlhead = retrieved_headers.read();
1105    if(sys.version[0]>="3"):
1106     pycurlhead = retrieved_headers.read().decode('UTF-8');
1107    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1108    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1109    retrieved_body.seek(0);
1110   except socket.timeout:
1111    log.info("Error With URL "+httpurl);
1112    return False;
1113   except socket.gaierror:
1114    log.info("Error With URL "+httpurl);
1115    return False;
1116   except ValueError:
1117    log.info("Error With URL "+httpurl);
1118    return False;
1119   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1120   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1121   httpversionout = pyhttpverinfo[0];
1122   httpmethodout = httpmethod;
1123   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1124   httpheaderout = pycurlheadersout;
1125   httpheadersentout = httpheaders;
1126  elif(httplibuse=="pycurl2"):
1127   retrieved_body = BytesIO();
1128   retrieved_headers = BytesIO();
1129   try:
1130    if(httpmethod=="GET"):
1131     geturls_text = pycurl.Curl();
1132     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1133     geturls_text.setopt(geturls_text.URL, httpurl);
1134     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1135     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1136     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1137     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1138     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1139     geturls_text.perform();
1140    elif(httpmethod=="POST"):
1141     geturls_text = pycurl.Curl();
1142     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1143     geturls_text.setopt(geturls_text.URL, httpurl);
1144     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1145     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1146     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1147     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1148     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1149     geturls_text.setopt(geturls_text.POST, True);
1150     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1151     geturls_text.perform();
1152    else:
1153     geturls_text = pycurl.Curl();
1154     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1155     geturls_text.setopt(geturls_text.URL, httpurl);
1156     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1157     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1158     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1159     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1160     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1161     geturls_text.perform();
1162    retrieved_headers.seek(0);
1163    if(sys.version[0]=="2"):
1164     pycurlhead = retrieved_headers.read();
1165    if(sys.version[0]>="3"):
1166     pycurlhead = retrieved_headers.read().decode('UTF-8');
1167    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
1168    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1169    retrieved_body.seek(0);
1170   except socket.timeout:
1171    log.info("Error With URL "+httpurl);
1172    return False;
1173   except socket.gaierror:
1174    log.info("Error With URL "+httpurl);
1175    return False;
1176   except ValueError:
1177    log.info("Error With URL "+httpurl);
1178    return False;
1179   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1180   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1181   httpversionout = pyhttpverinfo[0];
1182   httpmethodout = httpmethod;
1183   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1184   httpheaderout = pycurlheadersout;
1185   httpheadersentout = httpheaders;
1186  elif(httplibuse=="pycurl3"):
1187   retrieved_body = BytesIO();
1188   retrieved_headers = BytesIO();
1189   try:
1190    if(httpmethod=="GET"):
1191     geturls_text = pycurl.Curl();
1192     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1193     geturls_text.setopt(geturls_text.URL, httpurl);
1194     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1195     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1196     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1197     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1198     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1199     geturls_text.perform();
1200    elif(httpmethod=="POST"):
1201     geturls_text = pycurl.Curl();
1202     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1203     geturls_text.setopt(geturls_text.URL, httpurl);
1204     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1205     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1206     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1207     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1208     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1209     geturls_text.setopt(geturls_text.POST, True);
1210     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1211     geturls_text.perform();
1212    else:
1213     geturls_text = pycurl.Curl();
1214     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1215     geturls_text.setopt(geturls_text.URL, httpurl);
1216     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1217     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1218     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1219     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1220     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1221     geturls_text.perform();
1222    retrieved_headers.seek(0);
1223    if(sys.version[0]=="2"):
1224     pycurlhead = retrieved_headers.read();
1225    if(sys.version[0]>="3"):
1226     pycurlhead = retrieved_headers.read().decode('UTF-8');
1227    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1228    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1229    retrieved_body.seek(0);
1230   except socket.timeout:
1231    log.info("Error With URL "+httpurl);
1232    return False;
1233   except socket.gaierror:
1234    log.info("Error With URL "+httpurl);
1235    return False;
1236   except ValueError:
1237    log.info("Error With URL "+httpurl);
1238    return False;
1239   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1240   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1241   httpversionout = pyhttpverinfo[0];
1242   httpmethodout = httpmethod;
1243   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1244   httpheaderout = pycurlheadersout;
1245   httpheadersentout = httpheaders;
1246  elif(httplibuse=="ftp"):
1247   geturls_text = download_file_from_ftp_file(httpurl);
1248   if(not geturls_text):
1249    return False;
1250   downloadsize = None;
1251   if(downloadsize is not None):
1252    downloadsize = int(downloadsize);
1253   if downloadsize is None: downloadsize = 0;
1254   fulldatasize = 0;
1255   prevdownsize = 0;
1256   log.info("Downloading URL "+httpurl);
1257   with BytesIO() as strbuf:
1258    while True:
1259     databytes = geturls_text.read(buffersize);
1260     if not databytes: break;
1261     datasize = len(databytes);
1262     fulldatasize = datasize + fulldatasize;
1263     percentage = "";
1264     if(downloadsize>0):
1265      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1266     downloaddiff = fulldatasize - prevdownsize;
1267     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1268     prevdownsize = fulldatasize;
1269     strbuf.write(databytes);
1270    strbuf.seek(0);
1271    returnval_content = strbuf.read();
1272   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1273   geturls_text.close();
1274  elif(httplibuse=="sftp"):
1275   geturls_text = download_file_from_sftp_file(httpurl);
1276   if(not geturls_text):
1277    return False;
1278   downloadsize = None;
1279   if(downloadsize is not None):
1280    downloadsize = int(downloadsize);
1281   if downloadsize is None: downloadsize = 0;
1282   fulldatasize = 0;
1283   prevdownsize = 0;
1284   log.info("Downloading URL "+httpurl);
1285   with BytesIO() as strbuf:
1286    while True:
1287     databytes = geturls_text.read(buffersize);
1288     if not databytes: break;
1289     datasize = len(databytes);
1290     fulldatasize = datasize + fulldatasize;
1291     percentage = "";
1292     if(downloadsize>0):
1293      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1294     downloaddiff = fulldatasize - prevdownsize;
1295     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1296     prevdownsize = fulldatasize;
1297     strbuf.write(databytes);
1298    strbuf.seek(0);
1299    returnval_content = strbuf.read();
1300   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1301   geturls_text.close();
1302   return returnval;
1303  elif(httplibuse=="pysftp"):
1304   geturls_text = download_file_from_pysftp_file(httpurl);
1305   if(not geturls_text):
1306    return False;
1307   downloadsize = None;
1308   if(downloadsize is not None):
1309    downloadsize = int(downloadsize);
1310   if downloadsize is None: downloadsize = 0;
1311   fulldatasize = 0;
1312   prevdownsize = 0;
1313   log.info("Downloading URL "+httpurl);
1314   with BytesIO() as strbuf:
1315    while True:
1316     databytes = geturls_text.read(buffersize);
1317     if not databytes: break;
1318     datasize = len(databytes);
1319     fulldatasize = datasize + fulldatasize;
1320     percentage = "";
1321     if(downloadsize>0):
1322      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1323     downloaddiff = fulldatasize - prevdownsize;
1324     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1325     prevdownsize = fulldatasize;
1326     strbuf.write(databytes);
1327    strbuf.seek(0);
1328    returnval_content = strbuf.read();
1329   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1330   geturls_text.close();
1331   return returnval;
1332  else:
1333   returnval = False;
1334  if(isinstance(httpheaderout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1335   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1336  if(isinstance(httpheaderout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1337   httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
1338  if(sys.version[0]=="2"):
1339   try:
1340    prehttpheaderout = httpheaderout;
1341    httpheaderkeys = httpheaderout.keys();
1342    imax = len(httpheaderkeys);
1343    ic = 0;
1344    httpheaderout = {};
1345    while(ic < imax):
1346     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1347     ic += 1;
1348   except AttributeError:
1349    pass;
1350  httpheaderout = fix_header_names(httpheaderout);
1351  if(isinstance(httpheadersentout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1352   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1353  if(isinstance(httpheadersentout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1354   httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
1355  httpheadersentout = fix_header_names(httpheadersentout);
1356  log.info("Downloading URL "+httpurl);
1357  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"):
1358   downloadsize = httpheaderout.get('Content-Length');
1359   if(downloadsize is not None):
1360    downloadsize = int(downloadsize);
1361   if downloadsize is None: downloadsize = 0;
1362   fulldatasize = 0;
1363   prevdownsize = 0;
1364   log.info("Downloading URL "+httpurl);
1365   with BytesIO() as strbuf:
1366    while True:
1367     databytes = geturls_text.read(buffersize);
1368     if not databytes: break;
1369     datasize = len(databytes);
1370     fulldatasize = datasize + fulldatasize;
1371     percentage = "";
1372     if(downloadsize>0):
1373      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1374     downloaddiff = fulldatasize - prevdownsize;
1375     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1376     prevdownsize = fulldatasize;
1377     strbuf.write(databytes);
1378    strbuf.seek(0);
1379    returnval_content = strbuf.read();
1380   geturls_text.close();
1381   if(httpheaderout.get("Content-Encoding")=="gzip"):
1382    try:
1383     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1384    except zlib.error:
1385     pass;
1386   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1387    try:
1388     returnval_content = zlib.decompress(returnval_content);
1389    except zlib.error:
1390     pass;
1391   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1392    try:
1393     returnval_content = brotli.decompress(returnval_content);
1394    except brotli.error:
1395     pass;
1396   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1397    try:
1398     returnval_content = zstandard.decompress(returnval_content);
1399    except zstandard.error:
1400     pass;
1401   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1402    try:
1403     returnval_content = lzma.decompress(returnval_content);
1404    except zstandard.error:
1405     pass;
1406   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1407    try:
1408     returnval_content = bz2.decompress(returnval_content);
1409    except zstandard.error:
1410     pass;
1411  elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1412   downloadsize = httpheaderout.get('Content-Length');
1413   if(downloadsize is not None):
1414    downloadsize = int(downloadsize);
1415   if downloadsize is None: downloadsize = 0;
1416   fulldatasize = 0;
1417   prevdownsize = 0;
1418   log.info("Downloading URL "+httpurl);
1419   with BytesIO() as strbuf:
1420    while True:
1421     databytes = geturls_text.read();
1422     if not databytes: break;
1423     datasize = len(databytes);
1424     fulldatasize = datasize + fulldatasize;
1425     percentage = "";
1426     if(downloadsize>0):
1427      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1428     downloaddiff = fulldatasize - prevdownsize;
1429     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1430     prevdownsize = fulldatasize;
1431     strbuf.write(databytes);
1432     break;
1433    strbuf.seek(0);
1434    returnval_content = strbuf.read();
1435   geturls_text.close();
1436   if(httpheaderout.get("Content-Encoding")=="gzip"):
1437    try:
1438     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1439    except zlib.error:
1440     pass;
1441   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1442    try:
1443     returnval_content = zlib.decompress(returnval_content);
1444    except zlib.error:
1445     pass;
1446   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1447    try:
1448     returnval_content = brotli.decompress(returnval_content);
1449    except brotli.error:
1450     pass;
1451   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1452    try:
1453     returnval_content = zstandard.decompress(returnval_content);
1454    except zstandard.error:
1455     pass;
1456   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1457    try:
1458     returnval_content = lzma.decompress(returnval_content);
1459    except zstandard.error:
1460     pass;
1461   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1462    try:
1463     returnval_content = bz2.decompress(returnval_content);
1464    except zstandard.error:
1465     pass;
1466  elif(httplibuse=="requests"):
1467   log.info("Downloading URL "+httpurl);
1468   downloadsize = httpheaderout.get('Content-Length');
1469   if(downloadsize is not None):
1470    downloadsize = int(downloadsize);
1471   if downloadsize is None: downloadsize = 0;
1472   fulldatasize = 0;
1473   prevdownsize = 0;
1474   log.info("Downloading URL "+httpurl);
1475   with BytesIO() as strbuf:
1476    while True:
1477     databytes = geturls_text.raw.read(buffersize);
1478     if not databytes: break;
1479     datasize = len(databytes);
1480     fulldatasize = datasize + fulldatasize;
1481     percentage = "";
1482     if(downloadsize>0):
1483      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1484     downloaddiff = fulldatasize - prevdownsize;
1485     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1486     prevdownsize = fulldatasize;
1487     strbuf.write(databytes);
1488    strbuf.seek(0);
1489    returnval_content = strbuf.read();
1490   geturls_text.close();
1491   if(httpheaderout.get("Content-Encoding")=="gzip"):
1492    try:
1493     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1494    except zlib.error:
1495     pass;
1496   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1497    try:
1498     returnval_content = zlib.decompress(returnval_content);
1499    except zlib.error:
1500     pass;
1501   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1502    try:
1503     returnval_content = brotli.decompress(returnval_content);
1504    except brotli.error:
1505     pass;
1506   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1507    try:
1508     returnval_content = zstandard.decompress(returnval_content);
1509    except zstandard.error:
1510     pass;
1511   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1512    try:
1513     returnval_content = lzma.decompress(returnval_content);
1514    except zstandard.error:
1515     pass;
1516   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1517    try:
1518     returnval_content = bz2.decompress(returnval_content);
1519    except zstandard.error:
1520     pass;
1521  elif(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
1522   log.info("Downloading URL "+httpurl);
1523   downloadsize = httpheaderout.get('Content-Length');
1524   if(downloadsize is not None):
1525    downloadsize = int(downloadsize);
1526   if downloadsize is None: downloadsize = 0;
1527   fulldatasize = 0;
1528   prevdownsize = 0;
1529   log.info("Downloading URL "+httpurl);
1530   with BytesIO() as strbuf:
1531    while True:
1532     databytes = retrieved_body.read(buffersize);
1533     if not databytes: break;
1534     datasize = len(databytes);
1535     fulldatasize = datasize + fulldatasize;
1536     percentage = "";
1537     if(downloadsize>0):
1538      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1539     downloaddiff = fulldatasize - prevdownsize;
1540     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1541     prevdownsize = fulldatasize;
1542     strbuf.write(databytes);
1543    strbuf.seek(0);
1544    returnval_content = strbuf.read();
1545   geturls_text.close();
1546   if(httpheaderout.get("Content-Encoding")=="gzip"):
1547    try:
1548     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1549    except zlib.error:
1550     pass;
1551   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1552    try:
1553     returnval_content = zlib.decompress(returnval_content);
1554    except zlib.error:
1555     pass;
1556   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1557    try:
1558     returnval_content = brotli.decompress(returnval_content);
1559    except brotli.error:
1560     pass;
1561   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1562    try:
1563     returnval_content = zstandard.decompress(returnval_content);
1564    except zstandard.error:
1565     pass;
1566   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1567    try:
1568     returnval_content = lzma.decompress(returnval_content);
1569    except zstandard.error:
1570     pass;
1571   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1572    try:
1573     returnval_content = bz2.decompress(returnval_content);
1574    except zstandard.error:
1575     pass;
1576  elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1577   pass;
1578  else:
1579   returnval = False;
1580  returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse};
1581  return returnval;
1582
1583 def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
1584  if(isinstance(httpurl, list)):
1585   pass;
1586  elif(isinstance(httpurl, tuple)):
1587   pass;
1588  elif(isinstance(httpurl, dict)):
1589   httpurl = httpurl.values();
1590  else:
1591   httpurl = [httpurl];
1592  listsize = len(httpurl);
1593  listcount = 0;
1594  returnval = [];
1595  while(listcount<listsize):
1596   ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1597   returnval.append(ouputval);
1598   listcount += 1;
1599  return returnval;
1600
1601 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1602  global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1603  exec_time_start = time.time();
1604  myhash = hashlib.new("sha1");
1605  if(sys.version[0]=="2"):
1606   myhash.update(httpurl);
1607   myhash.update(str(buffersize));
1608   myhash.update(str(exec_time_start));
1609  if(sys.version[0]>="3"):
1610   myhash.update(httpurl.encode('utf-8'));
1611   myhash.update(str(buffersize).encode('utf-8'));
1612   myhash.update(str(exec_time_start).encode('utf-8'));
1613  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1614  if(sleep<0):
1615   sleep = geturls_download_sleep;
1616  if(timeout<=0):
1617   timeout = 10;
1618  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1619   httplibuse = "urllib";
1620  if(httplibuse=="httplib1"):
1621   httplibuse = "httplib";
1622  if(not haverequests and httplibuse=="requests"):
1623   httplibuse = "urllib";
1624  if(not haveaiohttp and httplibuse=="aiohttp"):
1625   httplibuse = "urllib";
1626  if(not havehttpx and httplibuse=="httpx"):
1627   httplibuse = "urllib";
1628  if(not havehttpx and httplibuse=="httpx2"):
1629   httplibuse = "urllib";
1630  if(not havehttpcore and httplibuse=="httpcore"):
1631   httplibuse = "urllib";
1632  if(not havehttpcore and httplibuse=="httpcore2"):
1633   httplibuse = "urllib";
1634  if(not havemechanize and httplibuse=="mechanize"):
1635   httplibuse = "urllib";
1636  if(not havepycurl and httplibuse=="pycurl"):
1637   httplibuse = "urllib";
1638  if(not havepycurl and httplibuse=="pycurl2"):
1639   httplibuse = "urllib";
1640  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1641   httplibuse = "pycurl";
1642  if(not havepycurl and httplibuse=="pycurl3"):
1643   httplibuse = "urllib";
1644  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1645   httplibuse = "pycurl2";
1646  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1647   httplibuse = "pycurl";
1648  if(not havehttplib2 and httplibuse=="httplib2"):
1649   httplibuse = "httplib";
1650  if(not haveparamiko and httplibuse=="sftp"):
1651   httplibuse = "ftp";
1652  if(not haveparamiko and httplibuse=="pysftp"):
1653   httplibuse = "ftp";
1654  pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1655  if(not pretmpfilename):
1656   return False;
1657  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1658   tmpfilename = f.name;
1659   try:
1660    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1661   except AttributeError:
1662    try:
1663     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1664    except ValueError:
1665     pass;
1666   except ValueError:
1667    pass;
1668   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1669   f.write(pretmpfilename.get('Content'));
1670   f.close();
1671  exec_time_end = time.time();
1672  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1673  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1674  return returnval;
1675
1676 def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1677  if(isinstance(httpurl, list)):
1678   pass;
1679  elif(isinstance(httpurl, tuple)):
1680   pass;
1681  elif(isinstance(httpurl, dict)):
1682   httpurl = httpurl.values();
1683  else:
1684   httpurl = [httpurl];
1685  listsize = len(httpurl);
1686  listcount = 0;
1687  returnval = [];
1688  while(listcount<listsize):
1689   ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout);
1690   returnval.append(ouputval);
1691   listcount += 1;
1692  return returnval;
1693
1694 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1695  global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1696  if(sleep<0):
1697   sleep = geturls_download_sleep;
1698  if(timeout<=0):
1699   timeout = 10;
1700  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1701   httplibuse = "urllib";
1702  if(httplibuse=="httplib1"):
1703   httplibuse = "httplib";
1704  if(not haverequests and httplibuse=="requests"):
1705   httplibuse = "urllib";
1706  if(not haveaiohttp and httplibuse=="aiohttp"):
1707   httplibuse = "urllib";
1708  if(not havehttpx and httplibuse=="httpx"):
1709   httplibuse = "urllib";
1710  if(not havehttpx and httplibuse=="httpx2"):
1711   httplibuse = "urllib";
1712  if(not havehttpcore and httplibuse=="httpcore"):
1713   httplibuse = "urllib";
1714  if(not havehttpcore and httplibuse=="httpcore2"):
1715   httplibuse = "urllib";
1716  if(not havemechanize and httplibuse=="mechanize"):
1717   httplibuse = "urllib";
1718  if(not havepycurl and httplibuse=="pycurl"):
1719   httplibuse = "urllib";
1720  if(not havepycurl and httplibuse=="pycurl2"):
1721   httplibuse = "urllib";
1722  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1723   httplibuse = "pycurl";
1724  if(not havepycurl and httplibuse=="pycurl3"):
1725   httplibuse = "urllib";
1726  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1727   httplibuse = "pycurl2";
1728  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1729   httplibuse = "pycurl";
1730  if(not havehttplib2 and httplibuse=="httplib2"):
1731   httplibuse = "httplib";
1732  if(not haveparamiko and httplibuse=="sftp"):
1733   httplibuse = "ftp";
1734  if(not havepysftp and httplibuse=="pysftp"):
1735   httplibuse = "ftp";
1736  if(not outfile=="-"):
1737   outpath = outpath.rstrip(os.path.sep);
1738   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1739   if(not os.path.exists(outpath)):
1740    os.makedirs(outpath);
1741   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1742    return False;
1743   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1744    return False;
1745   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1746   if(not pretmpfilename):
1747    return False;
1748   tmpfilename = pretmpfilename.get('Filename');
1749   downloadsize = int(os.path.getsize(tmpfilename));
1750   fulldatasize = 0;
1751   log.info("Moving file "+tmpfilename+" to "+filepath);
1752   exec_time_start = time.time();
1753   shutil.move(tmpfilename, filepath);
1754   try:
1755    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1756   except AttributeError:
1757    try:
1758     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1759    except ValueError:
1760     pass;
1761   except ValueError:
1762    pass;
1763   exec_time_end = time.time();
1764   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1765   if(os.path.exists(tmpfilename)):
1766    os.remove(tmpfilename);
1767   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1768  if(outfile=="-"):
1769   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1770   tmpfilename = pretmpfilename.get('Filename');
1771   downloadsize = int(os.path.getsize(tmpfilename));
1772   fulldatasize = 0;
1773   prevdownsize = 0;
1774   exec_time_start = time.time();
1775   with open(tmpfilename, 'rb') as ft:
1776    f = BytesIO();
1777    while True:
1778     databytes = ft.read(buffersize[1]);
1779     if not databytes: break;
1780     datasize = len(databytes);
1781     fulldatasize = datasize + fulldatasize;
1782     percentage = "";
1783     if(downloadsize>0):
1784      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1785     downloaddiff = fulldatasize - prevdownsize;
1786     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1787     prevdownsize = fulldatasize;
1788     f.write(databytes);
1789    f.seek(0);
1790    fdata = f.getvalue();
1791    f.close();
1792    ft.close();
1793    os.remove(tmpfilename);
1794    exec_time_end = time.time();
1795    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1796   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse};
1797  return returnval;
1798
1799 def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1800  if(isinstance(httpurl, list)):
1801   pass;
1802  elif(isinstance(httpurl, tuple)):
1803   pass;
1804  elif(isinstance(httpurl, dict)):
1805   httpurl = httpurl.values();
1806  else:
1807   httpurl = [httpurl];
1808  listsize = len(httpurl);
1809  listcount = 0;
1810  returnval = [];
1811  while(listcount<listsize):
1812   ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout);
1813   returnval.append(ouputval);
1814   listcount += 1;
1815  return returnval;
1816
1817 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1818   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1819   return returnval;
1820
1821 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1822   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1823   return returnval;
1824
1825 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1826   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout);
1827   return returnval;
1828
1829 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1830   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout);
1831   return returnval;
1832
1833 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1834   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout);
1835   return returnval;
1836
1837 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1838   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout);
1839   return returnval;
1840
1841 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1842   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout);
1843   return returnval;
1844
1845 def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1846   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout);
1847   return returnval;
1848
1849 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1850   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout);
1851   return returnval;
1852
1853 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1854   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout);
1855   return returnval;
1856
1857 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1858   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout);
1859   return returnval;
1860
1861 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1862   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout);
1863   return returnval;
1864
1865 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1866   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout);
1867   return returnval;
1868
1869 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1870   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout);
1871   return returnval;
1872
1873 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1874   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout);
1875   return returnval;
1876
1877 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1878   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout);
1879   return returnval;
1880
1881 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1882   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout);
1883   return returnval;
1884
1885 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1886   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout);
1887   return returnval;
1888
1889 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1890   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout);
1891   return returnval;
1892
1893 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1894   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1895   return returnval;
1896
1897 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1898   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1899   return returnval;
1900
1901 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1902   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout);
1903   return returnval;
1904
1905 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1906   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout);
1907   return returnval;
1908
1909 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1910   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout);
1911   return returnval;
1912
1913 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1914   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout);
1915   return returnval;
1916
1917 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1918   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout);
1919   return returnval;
1920
1921 def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1922   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout);
1923   return returnval;
1924
1925 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1926   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout);
1927   return returnval;
1928
1929 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1930   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout);
1931   return returnval;
1932
1933 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1934   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout);
1935   return returnval;
1936
1937 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1938   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout);
1939   return returnval;
1940
1941 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1942   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout);
1943   return returnval;
1944
1945 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1946   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout);
1947   return returnval;
1948
1949 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1950   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout);
1951   return returnval;
1952
1953 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1954   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout);
1955   return returnval;
1956
1957 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1958   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout);
1959   return returnval;
1960
1961 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1962   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout);
1963   return returnval;
1964
1965 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1966   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout);
1967   return returnval;
1968
1969 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1970   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1971   return returnval;
1972
1973 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1974   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout);
1975   return returnval;
1976
1977 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1978   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1979   return returnval;
1980
1981 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1982   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout);
1983   return returnval;
1984
1985 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1986   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout);
1987   return returnval;
1988
1989 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1990   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout);
1991   return returnval;
1992
1993 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1994   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout);
1995   return returnval;
1996
1997 def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1998   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout);
1999   return returnval;
2000
2001 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2002   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout);
2003   return returnval;
2004
2005 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2006   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout);
2007   return returnval;
2008
2009 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2010   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout);
2011   return returnval;
2012
2013 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2014   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout);
2015   return returnval;
2016
2017 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2018   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout);
2019   return returnval;
2020
2021 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2022   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout);
2023   return returnval;
2024
2025 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2026   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout);
2027   return returnval;
2028
2029 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2030   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout);
2031   return returnval;
2032
2033 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2034   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2035   return returnval;
2036
2037 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2038   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2039   return returnval;
2040
2041 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2042   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2043   return returnval;
2044
2045 def download_file_from_ftp_file(url):
2046  urlparts = urlparse.urlparse(url);
2047  file_name = os.path.basename(urlparts.path);
2048  file_dir = os.path.dirname(urlparts.path);
2049  if(urlparts.username is not None):
2050   ftp_username = urlparts.username;
2051  else:
2052   ftp_username = "anonymous";
2053  if(urlparts.password is not None):
2054   ftp_password = urlparts.password;
2055  elif(urlparts.password is None and urlparts.username=="anonymous"):
2056   ftp_password = "anonymous";
2057  else:
2058   ftp_password = "";
2059  if(urlparts.scheme=="ftp"):
2060   ftp = FTP();
2061  elif(urlparts.scheme=="ftps"):
2062   ftp = FTP_TLS();
2063  else:
2064   return False;
2065  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2066   return False;
2067  ftp_port = urlparts.port;
2068  if(urlparts.port is None):
2069   ftp_port = 21;
2070  try:
2071   ftp.connect(urlparts.hostname, ftp_port);
2072  except socket.gaierror:
2073   log.info("Error With URL "+httpurl);
2074   return False;
2075  except socket.timeout:
2076   log.info("Error With URL "+httpurl);
2077   return False;
2078  ftp.login(urlparts.username, urlparts.password);
2079  if(urlparts.scheme=="ftps"):
2080   ftp.prot_p();
2081  ftpfile = BytesIO();
2082  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2083  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2084  ftp.close();
2085  ftpfile.seek(0, 0);
2086  return ftpfile;
2087
2088 def download_file_from_ftp_string(url):
2089  ftpfile = download_file_from_ftp_file(url);
2090  return ftpfile.read();
2091
2092 def upload_file_to_ftp_file(ftpfile, url):
2093  urlparts = urlparse.urlparse(url);
2094  file_name = os.path.basename(urlparts.path);
2095  file_dir = os.path.dirname(urlparts.path);
2096  if(urlparts.username is not None):
2097   ftp_username = urlparts.username;
2098  else:
2099   ftp_username = "anonymous";
2100  if(urlparts.password is not None):
2101   ftp_password = urlparts.password;
2102  elif(urlparts.password is None and urlparts.username=="anonymous"):
2103   ftp_password = "anonymous";
2104  else:
2105   ftp_password = "";
2106  if(urlparts.scheme=="ftp"):
2107   ftp = FTP();
2108  elif(urlparts.scheme=="ftps"):
2109   ftp = FTP_TLS();
2110  else:
2111   return False;
2112  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2113   return False;
2114  ftp_port = urlparts.port;
2115  if(urlparts.port is None):
2116   ftp_port = 21;
2117  try:
2118   ftp.connect(urlparts.hostname, ftp_port);
2119  except socket.gaierror:
2120   log.info("Error With URL "+httpurl);
2121   return False;
2122  except socket.timeout:
2123   log.info("Error With URL "+httpurl);
2124   return False;
2125  ftp.login(urlparts.username, urlparts.password);
2126  if(urlparts.scheme=="ftps"):
2127   ftp.prot_p();
2128  ftp.storbinary("STOR "+urlparts.path, ftpfile);
2129  ftp.close();
2130  ftpfile.seek(0, 0);
2131  return ftpfile;
2132
2133 def upload_file_to_ftp_string(ftpstring, url):
2134  ftpfileo = BytesIO(ftpstring);
2135  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2136  ftpfileo.close();
2137  return ftpfile;
2138
2139 if(haveparamiko):
2140  def download_file_from_sftp_file(url):
2141   urlparts = urlparse.urlparse(url);
2142   file_name = os.path.basename(urlparts.path);
2143   file_dir = os.path.dirname(urlparts.path);
2144   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2145    return False;
2146   sftp_port = urlparts.port;
2147   if(urlparts.port is None):
2148    sftp_port = 22;
2149   else:
2150    sftp_port = urlparts.port;
2151   if(urlparts.username is not None):
2152    sftp_username = urlparts.username;
2153   else:
2154    sftp_username = "anonymous";
2155   if(urlparts.password is not None):
2156    sftp_password = urlparts.password;
2157   elif(urlparts.password is None and urlparts.username=="anonymous"):
2158    sftp_password = "anonymous";
2159   else:
2160    sftp_password = "";
2161   if(urlparts.scheme!="sftp"):
2162    return False;
2163   ssh = paramiko.SSHClient();
2164   ssh.load_system_host_keys();
2165   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2166   try:
2167    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2168   except paramiko.ssh_exception.SSHException:
2169    return False;
2170   except socket.gaierror:
2171    log.info("Error With URL "+httpurl);
2172    return False;
2173   except socket.timeout:
2174    log.info("Error With URL "+httpurl);
2175    return False;
2176   sftp = ssh.open_sftp();
2177   sftpfile = BytesIO();
2178   sftp.getfo(urlparts.path, sftpfile);
2179   sftp.close();
2180   ssh.close();
2181   sftpfile.seek(0, 0);
2182   return sftpfile;
2183 else:
2184  def download_file_from_sftp_file(url):
2185   return False;
2186
2187 if(haveparamiko):
2188  def download_file_from_sftp_string(url):
2189   sftpfile = download_file_from_sftp_file(url);
2190   return sftpfile.read();
2191 else:
2192  def download_file_from_ftp_string(url):
2193   return False;
2194
2195 if(haveparamiko):
2196  def upload_file_to_sftp_file(sftpfile, url):
2197   urlparts = urlparse.urlparse(url);
2198   file_name = os.path.basename(urlparts.path);
2199   file_dir = os.path.dirname(urlparts.path);
2200   sftp_port = urlparts.port;
2201   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2202    return False;
2203   if(urlparts.port is None):
2204    sftp_port = 22;
2205   else:
2206    sftp_port = urlparts.port;
2207   if(urlparts.username is not None):
2208    sftp_username = urlparts.username;
2209   else:
2210    sftp_username = "anonymous";
2211   if(urlparts.password is not None):
2212    sftp_password = urlparts.password;
2213   elif(urlparts.password is None and urlparts.username=="anonymous"):
2214    sftp_password = "anonymous";
2215   else:
2216    sftp_password = "";
2217   if(urlparts.scheme!="sftp"):
2218    return False;
2219   ssh = paramiko.SSHClient();
2220   ssh.load_system_host_keys();
2221   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2222   try:
2223    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2224   except paramiko.ssh_exception.SSHException:
2225    return False;
2226   except socket.gaierror:
2227    log.info("Error With URL "+httpurl);
2228    return False;
2229   except socket.timeout:
2230    log.info("Error With URL "+httpurl);
2231    return False;
2232   sftp = ssh.open_sftp();
2233   sftp.putfo(sftpfile, urlparts.path);
2234   sftp.close();
2235   ssh.close();
2236   sftpfile.seek(0, 0);
2237   return sftpfile;
2238 else:
2239  def upload_file_to_sftp_file(sftpfile, url):
2240   return False;
2241
2242 if(haveparamiko):
2243  def upload_file_to_sftp_string(sftpstring, url):
2244   sftpfileo = BytesIO(sftpstring);
2245   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2246   sftpfileo.close();
2247   return sftpfile;
2248 else:
2249  def upload_file_to_sftp_string(url):
2250   return False;
2251
2252
2253 if(havepysftp):
2254  def download_file_from_pysftp_file(url):
2255   urlparts = urlparse.urlparse(url);
2256   file_name = os.path.basename(urlparts.path);
2257   file_dir = os.path.dirname(urlparts.path);
2258   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2259    return False;
2260   sftp_port = urlparts.port;
2261   if(urlparts.port is None):
2262    sftp_port = 22;
2263   else:
2264    sftp_port = urlparts.port;
2265   if(urlparts.username is not None):
2266    sftp_username = urlparts.username;
2267   else:
2268    sftp_username = "anonymous";
2269   if(urlparts.password is not None):
2270    sftp_password = urlparts.password;
2271   elif(urlparts.password is None and urlparts.username=="anonymous"):
2272    sftp_password = "anonymous";
2273   else:
2274    sftp_password = "";
2275   if(urlparts.scheme!="sftp"):
2276    return False;
2277   try:
2278    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2279   except paramiko.ssh_exception.SSHException:
2280    return False;
2281   except socket.gaierror:
2282    log.info("Error With URL "+httpurl);
2283    return False;
2284   except socket.timeout:
2285    log.info("Error With URL "+httpurl);
2286    return False;
2287   sftp = ssh.open_sftp();
2288   sftpfile = BytesIO();
2289   sftp.getfo(urlparts.path, sftpfile);
2290   sftp.close();
2291   ssh.close();
2292   sftpfile.seek(0, 0);
2293   return sftpfile;
2294 else:
2295  def download_file_from_pysftp_file(url):
2296   return False;
2297
2298 if(havepysftp):
2299  def download_file_from_pysftp_string(url):
2300   sftpfile = download_file_from_pysftp_file(url);
2301   return sftpfile.read();
2302 else:
2303  def download_file_from_ftp_string(url):
2304   return False;
2305
2306 if(havepysftp):
2307  def upload_file_to_pysftp_file(sftpfile, url):
2308   urlparts = urlparse.urlparse(url);
2309   file_name = os.path.basename(urlparts.path);
2310   file_dir = os.path.dirname(urlparts.path);
2311   sftp_port = urlparts.port;
2312   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2313    return False;
2314   if(urlparts.port is None):
2315    sftp_port = 22;
2316   else:
2317    sftp_port = urlparts.port;
2318   if(urlparts.username is not None):
2319    sftp_username = urlparts.username;
2320   else:
2321    sftp_username = "anonymous";
2322   if(urlparts.password is not None):
2323    sftp_password = urlparts.password;
2324   elif(urlparts.password is None and urlparts.username=="anonymous"):
2325    sftp_password = "anonymous";
2326   else:
2327    sftp_password = "";
2328   if(urlparts.scheme!="sftp"):
2329    return False;
2330   try:
2331    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2332   except paramiko.ssh_exception.SSHException:
2333    return False;
2334   except socket.gaierror:
2335    log.info("Error With URL "+httpurl);
2336    return False;
2337   except socket.timeout:
2338    log.info("Error With URL "+httpurl);
2339    return False;
2340   sftp = ssh.open_sftp();
2341   sftp.putfo(sftpfile, urlparts.path);
2342   sftp.close();
2343   ssh.close();
2344   sftpfile.seek(0, 0);
2345   return sftpfile;
2346 else:
2347  def upload_file_to_pysftp_file(sftpfile, url):
2348   return False;
2349
2350 if(havepysftp):
2351  def upload_file_to_pysftp_string(sftpstring, url):
2352   sftpfileo = BytesIO(sftpstring);
2353   sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2354   sftpfileo.close();
2355   return sftpfile;
2356 else:
2357  def upload_file_to_pysftp_string(url):
2358   return False;