Add files via upload

[PyWWW-Get.git] / pywwwget.py

#!/usr/bin/env python

'''
    This program is free software; you can redistribute it and/or modify
    it under the terms of the Revised BSD License.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    Revised BSD License for more details.

    Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
    Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski

    $FileInfo: pywwwget.py - Last Update: 9/30/2023 Ver. 1.7.0 RC 1 - Author: cooldude2k $
'''

from __future__ import division, absolute_import, print_function;
import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket, email.utils, datetime, time;
import logging as log;
from ftplib import FTP, FTP_TLS;
from base64 import b64encode;
haverequests = False;
try:
 import requests;
 haverequests = True;
except ImportError:
 haverequests = False;
havemechanize = False;
try:
 import mechanize;
 havemechanize = True;
except ImportError:
 havemechanize = False;
havepycurl = False;
try:
 import pycurl;
 havepycurl = True;
except ImportError:
 havepycurl = False;
haveparamiko = False;
try:
 import paramiko;
 haveparamiko = True;
except ImportError:
 haveparamiko = False;
havepysftp = False;
try:
 import pysftp;
 havepysftp = True;
except ImportError:
 havepysftp = False;
haveurllib3 = False;
try:
 import urllib3;
 haveurllib3 = True;
except ImportError:
 haveurllib3 = False;
havehttplib2 = False;
try:
 from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
 havehttplib2 = True;
except ImportError:
 havehttplib2 = False;
havehttpx = False;
try:
 import httpx;
 havehttpx = True;
except ImportError:
 havehttpx = False;
havehttpcore = False;
try:
 import httpcore;
 havehttpcore = True;
except ImportError:
 havehttpcore = False;
havebrotli = False;
try:
 import brotli;
 havebrotli = True;
except ImportError:
 havebrotli = False;
havezstd = False;
try:
 import zstandard;
 havezstd = True;
except ImportError:
 havezstd = False;
if(sys.version[0]=="2"):
 try:
  from io import StringIO, BytesIO;
 except ImportError:
  try:
   from cStringIO import StringIO;
   from cStringIO import StringIO as BytesIO;
  except ImportError:
   from StringIO import StringIO;
   from StringIO import StringIO as BytesIO;
 # From http://python-future.org/compatible_idioms.html
 from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
 from urllib import urlencode;
 from urllib import urlopen as urlopenalt;
 from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
 import urlparse, cookielib;
 from httplib import HTTPConnection, HTTPSConnection;
if(sys.version[0]>="3"):
 from io import StringIO, BytesIO;
 # From http://python-future.org/compatible_idioms.html
 from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
 from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
 from urllib.error import HTTPError, URLError;
 import urllib.parse as urlparse;
 import http.cookiejar as cookielib;
 from http.client import HTTPConnection, HTTPSConnection;

__program_name__ = "PyWWW-Get";
__program_alt_name__ = "PyWWWGet";
__program_small_name__ = "wwwget";
__project__ = __program_name__;
__project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
__version_info__ = (1, 7, 0, "RC 1", 1);
__version_date_info__ = (2023, 9, 30, "RC 1", 1);
__version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
__revision__ = __version_info__[3];
__revision_id__ = "$Id$";
if(__version_info__[4] is not None):
 __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
if(__version_info__[4] is None):
 __version_date_plusrc__ = __version_date__;
if(__version_info__[3] is not None):
 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
if(__version_info__[3] is None):
 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);

tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
tmpfilesuffix = "-";
pytempdir = tempfile.gettempdir();

PyBitness = platform.architecture();
if(PyBitness=="32bit" or PyBitness=="32"):
 PyBitness = "32";
elif(PyBitness=="64bit" or PyBitness=="64"):
 PyBitness = "64";
else:
 PyBitness = "32";

compression_supported = "gzip, deflate";
if(havebrotli):
 compression_supported = "gzip, deflate, br";
else:
 compression_supported = "gzip, deflate";

geturls_cj = cookielib.CookieJar();
windowsNT4_ua_string = "Windows NT 4.0";
windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
windows2k_ua_string = "Windows NT 5.0";
windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
windowsXP_ua_string = "Windows NT 5.1";
windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
windows7_ua_string = "Windows NT 6.1; Win64; x64";
windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
windows8_ua_string = "Windows NT 6.2; Win64; x64";
windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
windows81_ua_string = "Windows NT 6.3; Win64; x64";
windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
windows10_ua_string = "Windows NT 10.0; Win64; x64";
windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
windows11_ua_string = "Windows NT 11.0; Win64; x64";
windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
if(platform.python_implementation()!=""):
 py_implementation = platform.python_implementation();
if(platform.python_implementation()==""):
 py_implementation = "Python";
geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
geturls_ua = geturls_ua_firefox_windows7;
geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
geturls_headers_chrome_windows7.update(windows7_ua_addon);
geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
geturls_headers_chromium_windows7.update(windows7_ua_addon);
geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
geturls_headers_opera_windows7.update(windows7_ua_addon);
geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
geturls_headers = geturls_headers_firefox_windows7;
geturls_download_sleep = 0;

def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 if(outtype=="print" and dbgenable):
  print(dbgtxt);
  return True;
 elif(outtype=="log" and dbgenable):
  logging.info(dbgtxt);
  return True;
 elif(outtype=="warning" and dbgenable):
  logging.warning(dbgtxt);
  return True;
 elif(outtype=="error" and dbgenable):
  logging.error(dbgtxt);
  return True;
 elif(outtype=="critical" and dbgenable):
  logging.critical(dbgtxt);
  return True;
 elif(outtype=="exception" and dbgenable):
  logging.exception(dbgtxt);
  return True;
 elif(outtype=="logalt" and dbgenable):
  logging.log(dgblevel, dbgtxt);
  return True;
 elif(outtype=="debug" and dbgenable):
  logging.debug(dbgtxt);
  return True;
 elif(not dbgenable):
  return True;
 else:
  return False;
 return False;

def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 if(not dbgout):
  return False;
 return dbgtxt;

def add_url_param(url, **params):
 n=3;
 parts = list(urlparse.urlsplit(url));
 d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 d.update(params);
 parts[n]=urlencode(d);
 return urlparse.urlunsplit(parts);

os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
def which_exec(execfile):
 for path in os.environ["PATH"].split(":"):
  if os.path.exists(path + "/" + execfile):
   return path + "/" + execfile;

def listize(varlist):
 il = 0;
 ix = len(varlist);
 ilx = 1;
 newlistreg = {};
 newlistrev = {};
 newlistfull = {};
 while(il < ix):
  newlistreg.update({ilx: varlist[il]});
  newlistrev.update({varlist[il]: ilx});
  ilx = ilx + 1;
  il = il + 1;
 newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 return newlistfull;

def twolistize(varlist):
 il = 0;
 ix = len(varlist);
 ilx = 1;
 newlistnamereg = {};
 newlistnamerev = {};
 newlistdescreg = {};
 newlistdescrev = {};
 newlistfull = {};
 while(il < ix):
  newlistnamereg.update({ilx: varlist[il][0].strip()});
  newlistnamerev.update({varlist[il][0].strip(): ilx});
  newlistdescreg.update({ilx: varlist[il][1].strip()});
  newlistdescrev.update({varlist[il][1].strip(): ilx});
  ilx = ilx + 1;
  il = il + 1;
 newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 return newlistfull;

def arglistize(proexec, *varlist):
 il = 0;
 ix = len(varlist);
 ilx = 1;
 newarglist = [proexec];
 while(il < ix):
  if varlist[il][0] is not None:
   newarglist.append(varlist[il][0]);
  if varlist[il][1] is not None:
   newarglist.append(varlist[il][1]);
  il = il + 1;
 return newarglist;

def fix_header_names(header_dict):
 header_dict =  {k.title(): v for k, v in header_dict.items()};
 return header_dict;

# hms_string by ArcGIS Python Recipes
# https://arcpy.wordpress.com/2012/04/20/146/
def hms_string(sec_elapsed):
 h = int(sec_elapsed / (60 * 60));
 m = int((sec_elapsed % (60 * 60)) / 60);
 s = sec_elapsed % 60.0;
 return "{}:{:>02}:{:>05.2f}".format(h, m, s);

# get_readable_size by Lipis
# http://stackoverflow.com/posts/14998888/revisions
def get_readable_size(bytes, precision=1, unit="IEC"):
 unit = unit.upper();
 if(unit!="IEC" and unit!="SI"):
  unit = "IEC";
 if(unit=="IEC"):
  units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
  unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
  unitsize = 1024.0;
 if(unit=="SI"):
  units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
  unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
  unitsize = 1000.0;
 return_val = {};
 orgbytes = bytes;
 for unit in units:
  if abs(bytes) < unitsize:
   strformat = "%3."+str(precision)+"f%s";
   pre_return_val = (strformat % (bytes, unit));
   pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
   pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
   alt_return_val = pre_return_val.split();
   return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
   return return_val;
  bytes /= unitsize;
 strformat = "%."+str(precision)+"f%s";
 pre_return_val = (strformat % (bytes, "YiB"));
 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 alt_return_val = pre_return_val.split();
 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 return return_val;

def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 unit = unit.upper();
 usehashtypes = usehashtypes.lower();
 getfilesize = os.path.getsize(infile);
 return_val = get_readable_size(getfilesize, precision, unit);
 if(usehashes):
  hashtypelist = usehashtypes.split(",");
  openfile = open(infile, "rb");
  filecontents = openfile.read();
  openfile.close();
  listnumcount = 0;
  listnumend = len(hashtypelist);
  while(listnumcount < listnumend):
   hashtypelistlow = hashtypelist[listnumcount].strip();
   hashtypelistup = hashtypelistlow.upper();
   filehash = hashlib.new(hashtypelistup);
   filehash.update(filecontents);
   filegethash = filehash.hexdigest();
   return_val.update({hashtypelistup: filegethash});
   listnumcount += 1;
 return return_val;

def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 unit = unit.upper();
 usehashtypes = usehashtypes.lower();
 getfilesize = len(instring);
 return_val = get_readable_size(getfilesize, precision, unit);
 if(usehashes):
  hashtypelist = usehashtypes.split(",");
  listnumcount = 0;
  listnumend = len(hashtypelist);
  while(listnumcount < listnumend):
   hashtypelistlow = hashtypelist[listnumcount].strip();
   hashtypelistup = hashtypelistlow.upper();
   filehash = hashlib.new(hashtypelistup);
   if(sys.version[0]=="2"):
    filehash.update(instring);
   if(sys.version[0]>="3"):
    filehash.update(instring.encode('utf-8'));
   filegethash = filehash.hexdigest();
   return_val.update({hashtypelistup: filegethash});
   listnumcount += 1;
 return return_val;

def http_status_to_reason(code):
    reasons = {
        100: 'Continue',
        101: 'Switching Protocols',
        102: 'Processing',
        200: 'OK',
        201: 'Created',
        202: 'Accepted',
        203: 'Non-Authoritative Information',
        204: 'No Content',
        205: 'Reset Content',
        206: 'Partial Content',
        207: 'Multi-Status',
        208: 'Already Reported',
        226: 'IM Used',
        300: 'Multiple Choices',
        301: 'Moved Permanently',
        302: 'Found',
        303: 'See Other',
        304: 'Not Modified',
        305: 'Use Proxy',
        307: 'Temporary Redirect',
        308: 'Permanent Redirect',
        400: 'Bad Request',
        401: 'Unauthorized',
        402: 'Payment Required',
        403: 'Forbidden',
        404: 'Not Found',
        405: 'Method Not Allowed',
        406: 'Not Acceptable',
        407: 'Proxy Authentication Required',
        408: 'Request Timeout',
        409: 'Conflict',
        410: 'Gone',
        411: 'Length Required',
        412: 'Precondition Failed',
        413: 'Payload Too Large',
        414: 'URI Too Long',
        415: 'Unsupported Media Type',
        416: 'Range Not Satisfiable',
        417: 'Expectation Failed',
        421: 'Misdirected Request',
        422: 'Unprocessable Entity',
        423: 'Locked',
        424: 'Failed Dependency',
        426: 'Upgrade Required',
        428: 'Precondition Required',
        429: 'Too Many Requests',
        431: 'Request Header Fields Too Large',
        451: 'Unavailable For Legal Reasons',
        500: 'Internal Server Error',
        501: 'Not Implemented',
        502: 'Bad Gateway',
        503: 'Service Unavailable',
        504: 'Gateway Timeout',
        505: 'HTTP Version Not Supported',
        506: 'Variant Also Negotiates',
        507: 'Insufficient Storage',
        508: 'Loop Detected',
        510: 'Not Extended',
        511: 'Network Authentication Required'
    }
    return reasons.get(code, 'Unknown Status Code');

def ftp_status_to_reason(code):
    reasons = {
        110: 'Restart marker reply',
        120: 'Service ready in nnn minutes',
        125: 'Data connection already open; transfer starting',
        150: 'File status okay; about to open data connection',
        200: 'Command okay',
        202: 'Command not implemented, superfluous at this site',
        211: 'System status, or system help reply',
        212: 'Directory status',
        213: 'File status',
        214: 'Help message',
        215: 'NAME system type',
        220: 'Service ready for new user',
        221: 'Service closing control connection',
        225: 'Data connection open; no transfer in progress',
        226: 'Closing data connection',
        227: 'Entering Passive Mode',
        230: 'User logged in, proceed',
        250: 'Requested file action okay, completed',
        257: '"PATHNAME" created',
        331: 'User name okay, need password',
        332: 'Need account for login',
        350: 'Requested file action pending further information',
        421: 'Service not available, closing control connection',
        425: 'Can\'t open data connection',
        426: 'Connection closed; transfer aborted',
        450: 'Requested file action not taken',
        451: 'Requested action aborted. Local error in processing',
        452: 'Requested action not taken. Insufficient storage space in system',
        500: 'Syntax error, command unrecognized',
        501: 'Syntax error in parameters or arguments',
        502: 'Command not implemented',
        503: 'Bad sequence of commands',
        504: 'Command not implemented for that parameter',
        530: 'Not logged in',
        532: 'Need account for storing files',
        550: 'Requested action not taken. File unavailable',
        551: 'Requested action aborted. Page type unknown',
        552: 'Requested file action aborted. Exceeded storage allocation',
        553: 'Requested action not taken. File name not allowed'
    };
    return reasons.get(code, 'Unknown Status Code');

def sftp_status_to_reason(code):
    reasons = {
        0: 'SSH_FX_OK',
        1: 'SSH_FX_EOF',
        2: 'SSH_FX_NO_SUCH_FILE',
        3: 'SSH_FX_PERMISSION_DENIED',
        4: 'SSH_FX_FAILURE',
        5: 'SSH_FX_BAD_MESSAGE',
        6: 'SSH_FX_NO_CONNECTION',
        7: 'SSH_FX_CONNECTION_LOST',
        8: 'SSH_FX_OP_UNSUPPORTED'
    };
    return reasons.get(code, 'Unknown Status Code');

def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 if isinstance(headers, dict):
  returnval = [];
  if(sys.version[0]=="2"):
   for headkey, headvalue in headers.iteritems():
    returnval.append((headkey, headvalue));
  if(sys.version[0]>="3"):
   for headkey, headvalue in headers.items():
    returnval.append((headkey, headvalue));
 elif isinstance(headers, list):
  returnval = headers;
 else:
  returnval = False;
 return returnval;

def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 if isinstance(headers, dict):
  returnval = [];
  if(sys.version[0]=="2"):
   for headkey, headvalue in headers.iteritems():
    returnval.append(headkey+": "+headvalue);
  if(sys.version[0]>="3"):
   for headkey, headvalue in headers.items():
    returnval.append(headkey+": "+headvalue);
 elif isinstance(headers, list):
  returnval = headers;
 else:
  returnval = False;
 return returnval;

def make_http_headers_from_pycurl_to_dict(headers):
 header_dict = {};
 headers = headers.strip().split('\r\n');
 for header in headers:
  parts = header.split(': ', 1)
  if(len(parts) == 2):
   key, value = parts;
   header_dict[key.title()] = value;
 return header_dict;

def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 if isinstance(headers, list):
  returnval = {};
  mli = 0;
  mlil = len(headers);
  while(mli<mlil):
   returnval.update({headers[mli][0]: headers[mli][1]});
   mli = mli + 1;
 elif isinstance(headers, dict):
  returnval = headers;
 else:
  returnval = False;
 return returnval;

def get_httplib_support(checkvalue=None):
 global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 returnval = [];
 returnval.append("ftp");
 returnval.append("httplib");
 if(havehttplib2):
  returnval.append("httplib2");
 returnval.append("urllib");
 if(haveurllib3):
  returnval.append("urllib3");
  returnval.append("request3");
 returnval.append("request");
 if(haverequests):
  returnval.append("requests");
 if(havehttpx):
  returnval.append("httpx");
  returnval.append("httpx2");
 if(havemechanize):
  returnval.append("mechanize");
 if(havepycurl):
  returnval.append("pycurl");
 if(haveparamiko):
  returnval.append("sftp");
 if(havepysftp):
  returnval.append("pysftp");
 if(not checkvalue is None):
  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
   checkvalue = "urllib";
  if(checkvalue=="httplib1"):
   checkvalue = "httplib";
  if(checkvalue in returnval):
   returnval = True;
  else:
   returnval = False;
 return returnval;

def check_httplib_support(checkvalue="urllib"):
 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
  checkvalue = "urllib";
 if(checkvalue=="httplib1"):
  checkvalue = "httplib";
 returnval = get_httplib_support(checkvalue);
 return returnval;

def get_httplib_support_list():
 returnval = get_httplib_support(None);
 return returnval;

def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 global geturls_download_sleep, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 if(sleep<0):
  sleep = geturls_download_sleep;
 if(httplibuse=="urllib1" or httplibuse=="urllib2"):
  httplibuse = "urllib";
 if(httplibuse=="httplib1"):
  httplibuse = "httplib";
 if(not haverequests and httplibuse=="requests"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx2"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore2"):
  httplibuse = "urllib";
 if(not havemechanize and httplibuse=="mechanize"):
  httplibuse = "urllib";
 if(not havepycurl and httplibuse=="pycurl"):
  httplibuse = "urllib";
 if(not havehttplib2 and httplibuse=="httplib2"):
  httplibuse = "httplib";
 if(not haveparamiko and httplibuse=="sftp"):
  httplibuse = "ftp";
 if(not havepysftp and httplibuse=="pysftp"):
  httplibuse = "ftp";
 urlparts = urlparse.urlparse(httpurl);
 if(isinstance(httpheaders, list)):
  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 httpheaders = fix_header_names(httpheaders);
 if(httpuseragent is not None):
  if('User-Agent' in httpheaders):
   httpheaders['User-Agent'] = httpuseragent;
  else:
   httpuseragent.update({'User-Agent': httpuseragent});
 if(httpreferer is not None):
  if('Referer' in httpheaders):
   httpheaders['Referer'] = httpreferer;
  else:
   httpuseragent.update({'Referer': httpreferer});
 if(urlparts.username is not None or urlparts.password is not None):
  inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
  httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="mechanize"):
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 if(httplibuse=="pycurl"):
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
 geturls_opener.addheaders = httpheaders;
 time.sleep(sleep);
 if(postdata is not None and not isinstance(postdata, dict)):
  postdata = urlencode(postdata);
 if(httplibuse=="urllib"):
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_opener.open(httpurl);
   elif(httpmethod=="POST"):
    geturls_text = geturls_opener.open(httpurl, data=postdata);
   else:
    geturls_text = geturls_opener.open(httpurl);
  except HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getcode();
  httpcodereason = geturls_text.reason;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="request"):
  try:
   if(httpmethod=="GET"):
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request);
   elif(httpmethod=="POST"):
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request, data=postdata);
   else:
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request);
  except HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getcode();
  httpcodereason = geturls_text.reason;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="request3"):
  urllib_pool = urllib3.PoolManager(headers=httpheaders);
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
   elif(httpmethod=="POST"):
    geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
   else:
    geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
  except urllib3.exceptions.ConnectTimeoutError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.MaxRetryError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="httplib"):
  if(urlparts[0]=="http"):
   httpconn = HTTPConnection(urlparts[1]);
  elif(urlparts[0]=="https"):
   httpconn = HTTPSConnection(urlparts[1]);
  else:
   return False;
  if(postdata is not None and not isinstance(postdata, dict)):
   postdata = urlencode(postdata);
  try:
   if(httpmethod=="GET"):
    httpconn.request("GET", urlparts[2], headers=httpheaders);
   elif(httpmethod=="POST"):
    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
   else:
    httpconn.request("GET", urlparts[2], headers=httpheaders);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  geturls_text = httpconn.getresponse();
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = httpurl;
  httpheaderout = geturls_text.getheaders();
  httpheadersentout = httpheaders;
 elif(httplibuse=="httplib2"):
  if(urlparts[0]=="http"):
   httpconn = HTTPConnectionWithTimeout(urlparts[1]);
  elif(urlparts[0]=="https"):
   httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
  else:
   return False;
  if(postdata is not None and not isinstance(postdata, dict)):
   postdata = urlencode(postdata);
  try:
   if(httpmethod=="GET"):
    httpconn.request("GET", urlparts[2], headers=httpheaders);
   elif(httpmethod=="POST"):
    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
   else:
    httpconn.request("GET", urlparts[2], headers=httpheaders);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  geturls_text = httpconn.getresponse();
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = httpurl;
  httpheaderout = geturls_text.getheaders();
  httpheadersentout = httpheaders;
 elif(httplibuse=="urllib3"):
  urllib_pool = urllib3.PoolManager(headers=httpheaders);
  try:
   if(httpmethod=="GET"):
    geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
   elif(httpmethod=="POST"):
    geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
   else:
    geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
  except urllib3.exceptions.ConnectTimeoutError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.MaxRetryError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="requests"):
  try:
   reqsession = requests.Session();
   if(httpmethod=="GET"):
    geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie);
   elif(httpmethod=="POST"):
    geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
   else:
    geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie);
  except requests.exceptions.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except requests.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason;
  if(geturls_text.raw.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.url;
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpx"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
   elif(httpmethod=="POST"):
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
   else:
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
  except httpx.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpx.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason_phrase;
  httpversionout = geturls_text.http_version;
  httpmethodout = httpmethod;
  httpurlout = str(geturls_text.url);
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpx2"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
   elif(httpmethod=="POST"):
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
   else:
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
  except httpx.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpx.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason;
  httpversionout = geturls_text.http_version;
  httpmethodout = httpmethod;
  httpurlout = str(geturls_text.url);
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpcore"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
   elif(httpmethod=="POST"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
   else:
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
  except httpcore.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpcore.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = http_status_to_reason(geturls_text.status);
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = str(httpurl);
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="httpcore2"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
   elif(httpmethod=="POST"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
   else:
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
  except httpcore.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpcore.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = str(httpurl);
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="mechanize"):
  geturls_opener = mechanize.Browser();
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
  time.sleep(sleep);
  geturls_opener.addheaders = httpheaders;
  geturls_opener.set_cookiejar(httpcookie);
  geturls_opener.set_handle_robots(False);
  if(postdata is not None and not isinstance(postdata, dict)):
   postdata = urlencode(postdata);
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_opener.open(httpurl);
   elif(httpmethod=="POST"):
    geturls_text = geturls_opener.open(httpurl, data=postdata);
   else:
    geturls_text = geturls_opener.open(httpurl);
  except mechanize.HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.code;
  httpcodereason = http_status_to_reason(geturls_text.code);
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  reqhead = geturls_opener.request;
  httpheadersentout = reqhead.header_items();
 elif(httplibuse=="pycurl"):
  retrieved_body = BytesIO();
  retrieved_headers = BytesIO();
  try:
   if(httpmethod=="GET"):
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   elif(httpmethod=="POST"):
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   else:
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   retrieved_headers.seek(0);
   pycurlhead = retrieved_headers.read().decode('UTF-8');
   pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+) ([A-Za-z\s]+)$', pycurlhead.splitlines()[0])[0];
   pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
   retrieved_body.seek(0);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
  httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
  httpversionout = pyhttpverinfo[0];
  httpmethodout = httpmethod;
  httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
  httpheaderout = pycurlheadersout;
  httpheadersentout = httpheaders;
 elif(httplibuse=="ftp"):
  geturls_text = download_file_from_ftp_file(httpurl);
  if(not geturls_text):
   return False;
  log.info("Downloading URL "+httpurl);
  returnval_content = geturls_text.read()[:];
  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
  geturls_text.close();
 elif(httplibuse=="sftp"):
  geturls_text = download_file_from_sftp_file(httpurl);
  if(not geturls_text):
   return False;
  log.info("Downloading URL "+httpurl);
  returnval_content = geturls_text.read()[:];
  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
  geturls_text.close();
  return returnval;
 elif(httplibuse=="pysftp"):
  geturls_text = download_file_from_pysftp_file(httpurl);
  if(not geturls_text):
   return False;
  log.info("Downloading URL "+httpurl);
  returnval_content = geturls_text.read()[:];
  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
  geturls_text.close();
  return returnval;
 else:
  returnval = False;
 if(isinstance(httpheaderout, list) and httplibuse!="pycurl"):
  httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
 if(isinstance(httpheaderout, list) and httplibuse=="pycurl"):
  httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
 if(sys.version[0]=="2"):
  try:
   prehttpheaderout = httpheaderout;
   httpheaderkeys = httpheaderout.keys();
   imax = len(httpheaderkeys);
   ic = 0;
   httpheaderout = {};
   while(ic < imax):
    httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
    ic += 1;
  except AttributeError:
   pass;
 httpheaderout = fix_header_names(httpheaderout);
 if(isinstance(httpheadersentout, list) and httplibuse!="pycurl"):
  httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
 if(isinstance(httpheadersentout, list) and httplibuse=="pycurl"):
  httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
 httpheadersentout = fix_header_names(httpheadersentout);
 log.info("Downloading URL "+httpurl);
 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
   if(sys.version[0]=="2"):
    strbuf = StringIO(geturls_text.read());
   if(sys.version[0]>="3"):
    strbuf = BytesIO(geturls_text.read());
   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
   returnval_content = gzstrbuf.read()[:];
  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
   returnval_content = geturls_text.read()[:];
  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
   returnval_content = geturls_text.read()[:];
   returnval_content = brotli.decompress(returnval_content);
  geturls_text.close();
 elif(httplibuse=="requests"):
  log.info("Downloading URL "+httpurl);
  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
   if(sys.version[0]=="2"):
    strbuf = StringIO(geturls_text.raw.read());
   if(sys.version[0]>="3"):
    strbuf = BytesIO(geturls_text.raw.read());
   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
   returnval_content = gzstrbuf.read()[:];
  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
   returnval_content = geturls_text.raw.read()[:];
  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
   returnval_content = geturls_text.raw.read()[:];
   returnval_content = brotli.decompress(returnval_content);
  geturls_text.close();
 elif(httplibuse=="pycurl"):
  log.info("Downloading URL "+httpurl);
  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
   if(sys.version[0]=="2"):
    strbuf = StringIO(retrieved_body.read());
   if(sys.version[0]>="3"):
    strbuf = BytesIO(retrieved_body.read());
   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
   returnval_content = gzstrbuf.read()[:];
  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
   returnval_content = retrieved_body.read()[:];
  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
   returnval_content = retrieved_body.read()[:];
   returnval_content = brotli.decompress(returnval_content);
  geturls_text.close();
 elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
  pass;
 else:
  returnval = False;
 returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
 return returnval;

def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
 global geturls_download_sleep, tmpfileprefix, tmpfilesuffix, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 exec_time_start = time.time();
 myhash = hashlib.new("sha1");
 if(sys.version[0]=="2"):
  myhash.update(httpurl);
  myhash.update(str(buffersize));
  myhash.update(str(exec_time_start));
 if(sys.version[0]>="3"):
  myhash.update(httpurl.encode('utf-8'));
  myhash.update(str(buffersize).encode('utf-8'));
  myhash.update(str(exec_time_start).encode('utf-8'));
 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 if(sleep<0):
  sleep = geturls_download_sleep;
 if(httplibuse=="urllib1" or httplibuse=="urllib2"):
  httplibuse = "urllib";
 if(httplibuse=="httplib1"):
  httplibuse = "httplib";
 if(not haverequests and httplibuse=="requests"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx2"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore2"):
  httplibuse = "urllib";
 if(not havemechanize and httplibuse=="mechanize"):
  httplibuse = "urllib";
 if(not havepycurl and httplibuse=="pycurl"):
  httplibuse = "urllib";
 if(not havehttplib2 and httplibuse=="httplib2"):
  httplibuse = "httplib";
 if(not haveparamiko and httplibuse=="sftp"):
  httplibuse = "ftp";
 if(not haveparamiko and httplibuse=="pysftp"):
  httplibuse = "ftp";
 urlparts = urlparse.urlparse(httpurl);
 if(isinstance(httpheaders, list)):
  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 httpheaders = fix_header_names(httpheaders);
 if(httpuseragent is not None):
  if('User-Agent' in httpheaders):
   httpheaders['User-Agent'] = httpuseragent;
  else:
   httpuseragent.update({'User-Agent': httpuseragent});
 if(httpreferer is not None):
  if('Referer' in httpheaders):
   httpheaders['Referer'] = httpreferer;
  else:
   httpuseragent.update({'Referer': httpreferer});
 if(urlparts.username is not None or urlparts.password is not None):
  inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
  httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="mechanize"):
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 if(httplibuse=="pycurl"):
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
 geturls_opener.addheaders = httpheaders;
 time.sleep(sleep);
 if(httplibuse=="urllib"):
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_opener.open(httpurl);
   elif(httpmethod=="POST"):
    geturls_text = geturls_opener.open(httpurl, data=postdata);
   else:
    geturls_text = geturls_opener.open(httpurl);
  except HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getcode();
  httpcodereason = geturls_text.reason;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="request"):
  try:
   if(httpmethod=="GET"):
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request);
   elif(httpmethod=="POST"):
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request, data=postdata);
   else:
    geturls_request = Request(httpurl, headers=httpheaders);
    geturls_text = urlopen(geturls_request);
  except HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getcode();
  httpcodereason = geturls_text.reason;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="request3"):
  urllib_pool = urllib3.PoolManager(headers=httpheaders);
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
   elif(httpmethod=="POST"):
    geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
   else:
    geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
  except urllib3.exceptions.ConnectTimeoutError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.MaxRetryError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="httplib"):
  if(urlparts[0]=="http"):
   httpconn = HTTPConnection(urlparts[1]);
  elif(urlparts[0]=="https"):
   httpconn = HTTPSConnection(urlparts[1]);
  else:
   return False;
  if(postdata is not None and not isinstance(postdata, dict)):
   postdata = urlencode(postdata);
  try:
   if(httpmethod=="GET"):
    httpconn.request("GET", urlparts[2], headers=httpheaders);
   elif(httpmethod=="POST"):
    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
   else:
    httpconn.request("GET", urlparts[2], headers=httpheaders);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  geturls_text = httpconn.getresponse();
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = httpurl;
  httpheaderout = geturls_text.getheaders();
  httpheadersentout = httpheaders;
 elif(httplibuse=="httplib2"):
  try:
   if(httpmethod=="GET"):
    httpconn.request("GET", urlparts[2], headers=httpheaders);
   elif(httpmethod=="POST"):
    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
   else:
    httpconn.request("GET", urlparts[2], headers=httpheaders);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  geturls_text = httpconn.getresponse();
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = httpurl;
  httpheaderout = geturls_text.getheaders();
  httpheadersentout = httpheaders;
 elif(httplibuse=="urllib3"):
  urllib_pool = urllib3.PoolManager(headers=httpheaders);
  try:
   if(httpmethod=="GET"):
    geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
   elif(httpmethod=="POST"):
    geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
   else:
    geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
  except urllib3.exceptions.ConnectTimeoutError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except urllib3.exceptions.MaxRetryError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason;
  if(geturls_text.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  httpheadersentout = httpheaders;
 elif(httplibuse=="requests"):
  try:
   reqsession = requests.Session();
   if(httpmethod=="GET"):
    geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
   elif(httpmethod=="POST"):
    geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie, stream=True);
   else:
    geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
  except requests.exceptions.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except requests.exceptions.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason;
  if(geturls_text.raw.version=="10"):
   httpversionout = "1.0";
  else:
   httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.url;
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpx"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
   elif(httpmethod=="POST"):
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
   else:
    httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
  except httpx.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpx.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason_phrase;
  httpversionout = geturls_text.http_version;
  httpmethodout = httpmethod;
  httpurlout = str(geturls_text.url);
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpx2"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
   elif(httpmethod=="POST"):
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
   else:
    httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
    geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
  except httpx.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpx.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status_code;
  httpcodereason = geturls_text.reason_phrase;
  httpversionout = geturls_text.http_version;
  httpmethodout = httpmethod;
  httpurlout = str(geturls_text.url);
  httpheaderout = geturls_text.headers;
  httpheadersentout = geturls_text.request.headers;
 elif(httplibuse=="httpcore"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
   elif(httpmethod=="POST"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
   else:
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
  except httpcore.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpcore.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = http_status_to_reason(geturls_text.status);
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = str(httpurl);
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="httpcore2"):
  try:
   if(httpmethod=="GET"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
   elif(httpmethod=="POST"):
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
   else:
    httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
    geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
  except httpcore.ConnectTimeout:
   log.info("Error With URL "+httpurl);
   return False;
  except httpcore.ConnectError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.status;
  httpcodereason = geturls_text.reason_phrase;
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = str(httpurl);
  httpheaderout = geturls_text.headers;
  httpheadersentout = httpheaders;
 elif(httplibuse=="mechanize"):
  geturls_opener = mechanize.Browser();
  if(isinstance(httpheaders, dict)):
   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
  time.sleep(sleep);
  geturls_opener.addheaders = httpheaders;
  geturls_opener.set_cookiejar(httpcookie);
  geturls_opener.set_handle_robots(False);
  if(postdata is not None and not isinstance(postdata, dict)):
   postdata = urlencode(postdata);
  try:
   if(httpmethod=="GET"):
    geturls_text = geturls_opener.open(httpurl);
   elif(httpmethod=="POST"):
    geturls_text = geturls_opener.open(httpurl, data=postdata);
   else:
    geturls_text = geturls_opener.open(httpurl);
  except mechanize.HTTPError as geturls_text_error:
   geturls_text = geturls_text_error;
   log.info("Error With URL "+httpurl);
  except URLError:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.code;
  httpcodereason = http_status_to_reason(geturls_text.code);
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.geturl();
  httpheaderout = geturls_text.info();
  reqhead = geturls_opener.request;
  httpheadersentout = reqhead.header_items();
 elif(httplibuse=="pycurl"):
  retrieved_body = BytesIO();
  retrieved_headers = BytesIO();
  try:
   if(httpmethod=="GET"):
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   elif(httpmethod=="POST"):
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   else:
    geturls_text = pycurl.Curl();
    geturls_text.setopt(geturls_text.URL, httpurl);
    geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
    geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
    geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
    geturls_text.perform();
   retrieved_headers.seek(0);
   pycurlhead = retrieved_headers.read().decode('UTF-8');
   pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+) ([A-Za-z\s]+)$', pycurlhead.splitlines()[0])[0];
   pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
   retrieved_body.seek(0);
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
  httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
  httpversionout = "1.1";
  httpmethodout = httpmethod;
  httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
  httpheaderout = pycurlheadersout;
  httpheadersentout = httpheaders;
 elif(httplibuse=="ftp"):
  geturls_text = download_file_from_ftp_file(httpurl);
  if(not geturls_text):
   return False;
  geturls_text.seek(0, 2);
  downloadsize = geturls_text.tell();
  geturls_text.seek(0, 0);
 elif(httplibuse=="sftp"):
  geturls_text = download_file_from_sftp_file(httpurl);
  if(not geturls_text):
   return False;
  geturls_text.seek(0, 2);
  downloadsize = geturls_text.tell();
  geturls_text.seek(0, 0);
  if(downloadsize is not None):
   downloadsize = int(downloadsize);
  if downloadsize is None: downloadsize = 0;
  fulldatasize = 0;
  prevdownsize = 0;
 elif(httplibuse=="pysftp"):
  geturls_text = download_file_from_pysftp_file(httpurl);
  if(not geturls_text):
   return False;
  geturls_text.seek(0, 2);
  downloadsize = geturls_text.tell();
  geturls_text.seek(0, 0);
 else:
  returnval = False;
 if(isinstance(httpheaderout, list) and httplibuse!="pycurl"):
  httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
 if(isinstance(httpheaderout, list) and httplibuse=="pycurl"):
  httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
 if(sys.version[0]=="2"):
  try:
   prehttpheaderout = httpheaderout;
   httpheaderkeys = httpheaderout.keys();
   imax = len(httpheaderkeys);
   ic = 0;
   httpheaderout = {};
   while(ic < imax):
    httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
    ic += 1;
  except AttributeError:
   pass;
 httpheaderout = fix_header_names(httpheaderout);
 if(isinstance(httpheadersentout, list) and httplibuse!="pycurl"):
  httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
 if(isinstance(httpheadersentout, list) and httplibuse=="pycurl"):
  httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
 httpheadersentout = fix_header_names(httpheadersentout);
 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="requests" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
  downloadsize = httpheaderout.get('Content-Length');
 if(downloadsize is not None):
  downloadsize = int(downloadsize);
 if downloadsize is None: downloadsize = 0;
 fulldatasize = 0;
 prevdownsize = 0;
 log.info("Downloading URL "+httpurl);
 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2" or httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
   tmpfilename = f.name;
   try:
    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
   except AttributeError:
    try:
     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
    except ValueError:
     pass;
   except ValueError:
    pass;
   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
   while True:
    databytes = geturls_text.read(buffersize);
    if not databytes: break;
    datasize = len(databytes);
    fulldatasize = datasize + fulldatasize;
    percentage = "";
    if(downloadsize>0):
     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
    downloaddiff = fulldatasize - prevdownsize;
    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
    prevdownsize = fulldatasize;
    f.write(databytes);
   f.close();
 elif(httplibuse=="requests"):
  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
   tmpfilename = f.name;
   try:
    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
   except AttributeError:
    try:
     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
    except ValueError:
     pass;
   except ValueError:
    pass;
   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
   for databytes in geturls_text.iter_content(chunk_size=buffersize):
    datasize = len(databytes);
    fulldatasize = datasize + fulldatasize;
    percentage = "";
    if(downloadsize>0):
     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
    downloaddiff = fulldatasize - prevdownsize;
    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
    prevdownsize = fulldatasize;
    f.write(databytes);
   f.close();
 elif(httplibuse=="pycurl"):
  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
   tmpfilename = f.name;
   try:
    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
   except AttributeError:
    try:
     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
    except ValueError:
     pass;
   except ValueError:
    pass;
   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
   while True:
    databytes = retrieved_body.read(buffersize);
    if not databytes: break;
    datasize = len(databytes);
    fulldatasize = datasize + fulldatasize;
    percentage = "";
    if(downloadsize>0):
     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
    downloaddiff = fulldatasize - prevdownsize;
    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
    prevdownsize = fulldatasize;
    f.write(databytes);
   f.close();
 else:
  pass;
 geturls_text.close();
 exec_time_end = time.time();
 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 return returnval;

def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 global geturls_download_sleep, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 if(sleep<0):
  sleep = geturls_download_sleep;
 if(httplibuse=="urllib1" or httplibuse=="urllib2"):
  httplibuse = "urllib";
 if(httplibuse=="httplib1"):
  httplibuse = "httplib";
 if(not haverequests and httplibuse=="requests"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx"):
  httplibuse = "urllib";
 if(not havehttpx and httplibuse=="httpx2"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore"):
  httplibuse = "urllib";
 if(not havehttpcore and httplibuse=="httpcore2"):
  httplibuse = "urllib";
 if(not havemechanize and httplibuse=="mechanize"):
  httplibuse = "urllib";
 if(not havepycurl and httplibuse=="pycurl"):
  httplibuse = "urllib";
 if(not havehttplib2 and httplibuse=="httplib2"):
  httplibuse = "httplib";
 if(not haveparamiko and httplibuse=="sftp"):
  httplibuse = "ftp";
 if(not havepysftp and httplibuse=="pysftp"):
  httplibuse = "ftp";
 if(not outfile=="-"):
  outpath = outpath.rstrip(os.path.sep);
  filepath = os.path.realpath(outpath+os.path.sep+outfile);
  if(not os.path.exists(outpath)):
   os.makedirs(outpath);
  if(os.path.exists(outpath) and os.path.isfile(outpath)):
   return False;
  if(os.path.exists(filepath) and os.path.isdir(filepath)):
   return False;
  pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
  if(not pretmpfilename):
   return False;
  tmpfilename = pretmpfilename['Filename'];
  downloadsize = os.path.getsize(tmpfilename);
  fulldatasize = 0;
  log.info("Moving file "+tmpfilename+" to "+filepath);
  exec_time_start = time.time();
  shutil.move(tmpfilename, filepath);
  try:
   os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
  except AttributeError:
   try:
    os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
   except ValueError:
    pass;
  except ValueError:
   pass;
  exec_time_end = time.time();
  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
  if(os.path.exists(tmpfilename)):
   os.remove(tmpfilename);
  returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
 if(outfile=="-" and sys.version[0]=="2"):
  pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
  if(not pretmpfilename):
   return False;
  tmpfilename = pretmpfilename['Filename'];
  downloadsize = os.path.getsize(tmpfilename);
  fulldatasize = 0;
  prevdownsize = 0;
  exec_time_start = time.time();
  with open(tmpfilename, 'rb') as ft:
   f = StringIO();
   while True:
    databytes = ft.read(buffersize[1]);
    if not databytes: break;
    datasize = len(databytes);
    fulldatasize = datasize + fulldatasize;
    percentage = "";
    if(downloadsize>0):
     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
    downloaddiff = fulldatasize - prevdownsize;
    log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
    prevdownsize = fulldatasize;
    f.write(databytes);
   f.seek(0);
   fdata = f.getvalue();
   f.close();
   ft.close();
   os.remove(tmpfilename);
   exec_time_end = time.time();
   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
  returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
 if(outfile=="-" and sys.version[0]>="3"):
  pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
  tmpfilename = pretmpfilename['Filename'];
  downloadsize = os.path.getsize(tmpfilename);
  fulldatasize = 0;
  prevdownsize = 0;
  exec_time_start = time.time();
  with open(tmpfilename, 'rb') as ft:
   f = BytesIO();
   while True:
    databytes = ft.read(buffersize[1]);
    if not databytes: break;
    datasize = len(databytes);
    fulldatasize = datasize + fulldatasize;
    percentage = "";
    if(downloadsize>0):
     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
    downloaddiff = fulldatasize - prevdownsize;
    log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
    prevdownsize = fulldatasize;
    f.write(databytes);
   f.seek(0);
   fdata = f.getvalue();
   f.close();
   ft.close();
   os.remove(tmpfilename);
   exec_time_end = time.time();
   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
  returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
 return returnval;

def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", sleep);
  return returnval;

def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", sleep);
  return returnval;

def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", sleep);
  return returnval;

def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", sleep);
  return returnval;

def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", sleep);
  return returnval;

def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", sleep);
  return returnval;

def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", sleep);
  return returnval;

def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", sleep);
  return returnval;

def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", sleep);
  return returnval;

def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", sleep);
  return returnval;

def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", sleep);
  return returnval;

def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", sleep);
  return returnval;

def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", sleep);
  return returnval;

def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", sleep);
  return returnval;

def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", sleep);
  return returnval;

def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
  returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", sleep);
  return returnval;

def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep);
  return returnval;

def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", buffersize, sleep);
  return returnval;

def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep);
  return returnval;

def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep);
  return returnval;

def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep);
  return returnval;

def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep);
  return returnval;

def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep);
  return returnval;

def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep);
  return returnval;

def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep);
  return returnval;

def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep);
  return returnval;

def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, buffersize, sleep);
  return returnval;

def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
  returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, buffersize, sleep);
  return returnval;

def download_file_from_ftp_file(url):
 urlparts = urlparse.urlparse(url);
 file_name = os.path.basename(urlparts.path);
 file_dir = os.path.dirname(urlparts.path);
 if(urlparts.username is not None):
  ftp_username = urlparts.username;
 else:
  ftp_username = "anonymous";
 if(urlparts.password is not None):
  ftp_password = urlparts.password;
 elif(urlparts.password is None and urlparts.username=="anonymous"):
  ftp_password = "anonymous";
 else:
  ftp_password = "";
 if(urlparts.scheme=="ftp"):
  ftp = FTP();
 elif(urlparts.scheme=="ftps"):
  ftp = FTP_TLS();
 else:
  return False;
 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
  return False;
 ftp_port = urlparts.port;
 if(urlparts.port is None):
  ftp_port = 21;
 try:
  ftp.connect(urlparts.hostname, ftp_port);
 except socket.gaierror:
  log.info("Error With URL "+httpurl);
  return False;
 except socket.timeout:
  log.info("Error With URL "+httpurl);
  return False;
 ftp.login(urlparts.username, urlparts.password);
 if(urlparts.scheme=="ftps"):
  ftp.prot_p();
 ftpfile = BytesIO();
 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
 ftp.close();
 ftpfile.seek(0, 0);
 return ftpfile;

def download_file_from_ftp_string(url):
 ftpfile = download_file_from_ftp_file(url);
 return ftpfile.read();

def upload_file_to_ftp_file(ftpfile, url):
 urlparts = urlparse.urlparse(url);
 file_name = os.path.basename(urlparts.path);
 file_dir = os.path.dirname(urlparts.path);
 if(urlparts.username is not None):
  ftp_username = urlparts.username;
 else:
  ftp_username = "anonymous";
 if(urlparts.password is not None):
  ftp_password = urlparts.password;
 elif(urlparts.password is None and urlparts.username=="anonymous"):
  ftp_password = "anonymous";
 else:
  ftp_password = "";
 if(urlparts.scheme=="ftp"):
  ftp = FTP();
 elif(urlparts.scheme=="ftps"):
  ftp = FTP_TLS();
 else:
  return False;
 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
  return False;
 ftp_port = urlparts.port;
 if(urlparts.port is None):
  ftp_port = 21;
 try:
  ftp.connect(urlparts.hostname, ftp_port);
 except socket.gaierror:
  log.info("Error With URL "+httpurl);
  return False;
 except socket.timeout:
  log.info("Error With URL "+httpurl);
  return False;
 ftp.login(urlparts.username, urlparts.password);
 if(urlparts.scheme=="ftps"):
  ftp.prot_p();
 ftp.storbinary("STOR "+urlparts.path, ftpfile);
 ftp.close();
 ftpfile.seek(0, 0);
 return ftpfile;

def upload_file_to_ftp_string(ftpstring, url):
 ftpfileo = BytesIO(ftpstring);
 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
 ftpfileo.close();
 return ftpfile;

if(haveparamiko):
 def download_file_from_sftp_file(url):
  urlparts = urlparse.urlparse(url);
  file_name = os.path.basename(urlparts.path);
  file_dir = os.path.dirname(urlparts.path);
  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
   return False;
  sftp_port = urlparts.port;
  if(urlparts.port is None):
   sftp_port = 22;
  else:
   sftp_port = urlparts.port;
  if(urlparts.username is not None):
   sftp_username = urlparts.username;
  else:
   sftp_username = "anonymous";
  if(urlparts.password is not None):
   sftp_password = urlparts.password;
  elif(urlparts.password is None and urlparts.username=="anonymous"):
   sftp_password = "anonymous";
  else:
   sftp_password = "";
  if(urlparts.scheme!="sftp"):
   return False;
  ssh = paramiko.SSHClient();
  ssh.load_system_host_keys();
  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
  try:
   ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
  except paramiko.ssh_exception.SSHException:
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  sftp = ssh.open_sftp();
  sftpfile = BytesIO();
  sftp.getfo(urlparts.path, sftpfile);
  sftp.close();
  ssh.close();
  sftpfile.seek(0, 0);
  return sftpfile;
else:
 def download_file_from_sftp_file(url):
  return False;

if(haveparamiko):
 def download_file_from_sftp_string(url):
  sftpfile = download_file_from_sftp_file(url);
  return sftpfile.read();
else:
 def download_file_from_ftp_string(url):
  return False;

if(haveparamiko):
 def upload_file_to_sftp_file(sftpfile, url):
  urlparts = urlparse.urlparse(url);
  file_name = os.path.basename(urlparts.path);
  file_dir = os.path.dirname(urlparts.path);
  sftp_port = urlparts.port;
  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
   return False;
  if(urlparts.port is None):
   sftp_port = 22;
  else:
   sftp_port = urlparts.port;
  if(urlparts.username is not None):
   sftp_username = urlparts.username;
  else:
   sftp_username = "anonymous";
  if(urlparts.password is not None):
   sftp_password = urlparts.password;
  elif(urlparts.password is None and urlparts.username=="anonymous"):
   sftp_password = "anonymous";
  else:
   sftp_password = "";
  if(urlparts.scheme!="sftp"):
   return False;
  ssh = paramiko.SSHClient();
  ssh.load_system_host_keys();
  ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
  try:
   ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
  except paramiko.ssh_exception.SSHException:
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  sftp = ssh.open_sftp();
  sftp.putfo(sftpfile, urlparts.path);
  sftp.close();
  ssh.close();
  sftpfile.seek(0, 0);
  return sftpfile;
else:
 def upload_file_to_sftp_file(sftpfile, url):
  return False;

if(haveparamiko):
 def upload_file_to_sftp_string(sftpstring, url):
  sftpfileo = BytesIO(sftpstring);
  sftpfile = upload_file_to_sftp_files(ftpfileo, url);
  sftpfileo.close();
  return sftpfile;
else:
 def upload_file_to_sftp_string(url):
  return False;


if(havepysftp):
 def download_file_from_pysftp_file(url):
  urlparts = urlparse.urlparse(url);
  file_name = os.path.basename(urlparts.path);
  file_dir = os.path.dirname(urlparts.path);
  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
   return False;
  sftp_port = urlparts.port;
  if(urlparts.port is None):
   sftp_port = 22;
  else:
   sftp_port = urlparts.port;
  if(urlparts.username is not None):
   sftp_username = urlparts.username;
  else:
   sftp_username = "anonymous";
  if(urlparts.password is not None):
   sftp_password = urlparts.password;
  elif(urlparts.password is None and urlparts.username=="anonymous"):
   sftp_password = "anonymous";
  else:
   sftp_password = "";
  if(urlparts.scheme!="sftp"):
   return False;
  try:
   pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
  except paramiko.ssh_exception.SSHException:
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  sftp = ssh.open_sftp();
  sftpfile = BytesIO();
  sftp.getfo(urlparts.path, sftpfile);
  sftp.close();
  ssh.close();
  sftpfile.seek(0, 0);
  return sftpfile;
else:
 def download_file_from_pysftp_file(url):
  return False;

if(havepysftp):
 def download_file_from_pysftp_string(url):
  sftpfile = download_file_from_pysftp_file(url);
  return sftpfile.read();
else:
 def download_file_from_ftp_string(url):
  return False;

if(havepysftp):
 def upload_file_to_pysftp_file(sftpfile, url):
  urlparts = urlparse.urlparse(url);
  file_name = os.path.basename(urlparts.path);
  file_dir = os.path.dirname(urlparts.path);
  sftp_port = urlparts.port;
  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
   return False;
  if(urlparts.port is None):
   sftp_port = 22;
  else:
   sftp_port = urlparts.port;
  if(urlparts.username is not None):
   sftp_username = urlparts.username;
  else:
   sftp_username = "anonymous";
  if(urlparts.password is not None):
   sftp_password = urlparts.password;
  elif(urlparts.password is None and urlparts.username=="anonymous"):
   sftp_password = "anonymous";
  else:
   sftp_password = "";
  if(urlparts.scheme!="sftp"):
   return False;
  try:
   pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
  except paramiko.ssh_exception.SSHException:
   return False;
  except socket.gaierror:
   log.info("Error With URL "+httpurl);
   return False;
  except socket.timeout:
   log.info("Error With URL "+httpurl);
   return False;
  sftp = ssh.open_sftp();
  sftp.putfo(sftpfile, urlparts.path);
  sftp.close();
  ssh.close();
  sftpfile.seek(0, 0);
  return sftpfile;
else:
 def upload_file_to_pysftp_file(sftpfile, url):
  return False;

if(havepysftp):
 def upload_file_to_pysftp_string(sftpstring, url):
  sftpfileo = BytesIO(sftpstring);
  sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
  sftpfileo.close();
  return sftpfile;
else:
 def upload_file_to_pysftp_string(url):
  return False;