pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 9/15/2023 Ver. 0.9.4 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 haverequests = False;
  24 try:
  25  import requests;
  26  haverequests = True;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  import mechanize;
  32  havemechanize = True;
  33 except ImportError:
  34  havemechanize = False;
  35 haveparamiko = False;
  36 try:
  37  import paramiko;
  38  haveparamiko = True;
  39 except ImportError:
  40  haveparamiko = False;
  41 haveurllib3 = False;
  42 try:
  43  import urllib3;
  44  haveurllib3 = True;
  45 except ImportError:
  46  haveurllib3 = False;
  47 havehttplib2 = False;
  48 try:
  49  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  50  havehttplib2 = True;
  51 except ImportError:
  52  havehttplib2 = False;
  53 havehttpx = False;
  54 try:
  55  import httpx;
  56  havehttpx = True;
  57 except ImportError:
  58  havehttpx = False;
  59 havebrotli = False;
  60 try:
  61  import brotli;
  62  havebrotli = True;
  63 except ImportError:
  64  havebrotli = False;
  65 if(sys.version[0]=="2"):
  66  try:
  67   from cStringIO import StringIO;
  68  except ImportError:
  69   from StringIO import StringIO;
  70  # From http://python-future.org/compatible_idioms.html
  71  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  72  from urllib import urlencode;
  73  from urllib import urlopen as urlopenalt;
  74  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
  75  import urlparse, cookielib;
  76  from httplib import HTTPConnection, HTTPSConnection;
  77 if(sys.version[0]>="3"):
  78  from io import StringIO, BytesIO;
  79  # From http://python-future.org/compatible_idioms.html
  80  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  81  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
  82  from urllib.error import HTTPError, URLError;
  83  import urllib.parse as urlparse;
  84  import http.cookiejar as cookielib;
  85  from http.client import HTTPConnection, HTTPSConnection;
  86
  87 __program_name__ = "PyWWW-Get";
  88 __program_alt_name__ = "PyWWWGet";
  89 __program_small_name__ = "wwwget";
  90 __project__ = __program_name__;
  91 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
  92 __version_info__ = (0, 9, 4, "RC 1", 1);
  93 __version_date_info__ = (2023, 9, 15, "RC 1", 1);
  94 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  95 __revision__ = __version_info__[3];
  96 __revision_id__ = "$Id$";
  97 if(__version_info__[4] is not None):
  98  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  99 if(__version_info__[4] is None):
 100  __version_date_plusrc__ = __version_date__;
 101 if(__version_info__[3] is not None):
 102  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 103 if(__version_info__[3] is None):
 104  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 105
 106 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 107 tmpfilesuffix = "-";
 108 pytempdir = tempfile.gettempdir();
 109
 110 PyBitness = platform.architecture();
 111 if(PyBitness=="32bit" or PyBitness=="32"):
 112  PyBitness = "32";
 113 elif(PyBitness=="64bit" or PyBitness=="64"):
 114  PyBitness = "64";
 115 else:
 116  PyBitness = "32";
 117
 118 compression_supported = "gzip, deflate";
 119 if(havebrotli):
 120  compression_supported = "gzip, deflate, br";
 121 else:
 122  compression_supported = "gzip, deflate";
 123
 124 geturls_cj = cookielib.CookieJar();
 125 windowsNT4_ua_string = "Windows NT 4.0";
 126 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 127 windows2k_ua_string = "Windows NT 5.0";
 128 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 129 windowsXP_ua_string = "Windows NT 5.1";
 130 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 131 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 132 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 133 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 134 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 135 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 136 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 137 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 138 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 139 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 140 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 141 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 142 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 143 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 144 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 145 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 146 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 147 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 148 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 149 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 150 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 151 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 152 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 153 if(platform.python_implementation()!=""):
 154  py_implementation = platform.python_implementation();
 155 if(platform.python_implementation()==""):
 156  py_implementation = "Python";
 157 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 158 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 159 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 160 geturls_ua = geturls_ua_firefox_windows7;
 161 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 162 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 163 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 164 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 165 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 166 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 167 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 168 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 169 geturls_headers_opera_windows7.update(windows7_ua_addon);
 170 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 171 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 172 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 173 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 174 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 175 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 176 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 177 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 178 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 179 geturls_headers = geturls_headers_firefox_windows7;
 180 geturls_download_sleep = 0;
 181
 182 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 183  if(outtype=="print" and dbgenable):
 184   print(dbgtxt);
 185   return True;
 186  elif(outtype=="log" and dbgenable):
 187   logging.info(dbgtxt);
 188   return True;
 189  elif(outtype=="warning" and dbgenable):
 190   logging.warning(dbgtxt);
 191   return True;
 192  elif(outtype=="error" and dbgenable):
 193   logging.error(dbgtxt);
 194   return True;
 195  elif(outtype=="critical" and dbgenable):
 196   logging.critical(dbgtxt);
 197   return True;
 198  elif(outtype=="exception" and dbgenable):
 199   logging.exception(dbgtxt);
 200   return True;
 201  elif(outtype=="logalt" and dbgenable):
 202   logging.log(dgblevel, dbgtxt);
 203   return True;
 204  elif(outtype=="debug" and dbgenable):
 205   logging.debug(dbgtxt);
 206   return True;
 207  elif(not dbgenable):
 208   return True;
 209  else:
 210   return False;
 211  return False;
 212
 213 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 214  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 215  if(not dbgout):
 216   return False;
 217  return dbgtxt;
 218
 219 def add_url_param(url, **params):
 220  n=3;
 221  parts = list(urlparse.urlsplit(url));
 222  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 223  d.update(params);
 224  parts[n]=urlencode(d);
 225  return urlparse.urlunsplit(parts);
 226
 227 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 228 def which_exec(execfile):
 229  for path in os.environ["PATH"].split(":"):
 230   if os.path.exists(path + "/" + execfile):
 231    return path + "/" + execfile;
 232
 233 def listize(varlist):
 234  il = 0;
 235  ix = len(varlist);
 236  ilx = 1;
 237  newlistreg = {};
 238  newlistrev = {};
 239  newlistfull = {};
 240  while(il < ix):
 241   newlistreg.update({ilx: varlist[il]});
 242   newlistrev.update({varlist[il]: ilx});
 243   ilx = ilx + 1;
 244   il = il + 1;
 245  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 246  return newlistfull;
 247
 248 def twolistize(varlist):
 249  il = 0;
 250  ix = len(varlist);
 251  ilx = 1;
 252  newlistnamereg = {};
 253  newlistnamerev = {};
 254  newlistdescreg = {};
 255  newlistdescrev = {};
 256  newlistfull = {};
 257  while(il < ix):
 258   newlistnamereg.update({ilx: varlist[il][0].strip()});
 259   newlistnamerev.update({varlist[il][0].strip(): ilx});
 260   newlistdescreg.update({ilx: varlist[il][1].strip()});
 261   newlistdescrev.update({varlist[il][1].strip(): ilx});
 262   ilx = ilx + 1;
 263   il = il + 1;
 264  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 265  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 266  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 267  return newlistfull;
 268
 269 def arglistize(proexec, *varlist):
 270  il = 0;
 271  ix = len(varlist);
 272  ilx = 1;
 273  newarglist = [proexec];
 274  while(il < ix):
 275   if varlist[il][0] is not None:
 276    newarglist.append(varlist[il][0]);
 277   if varlist[il][1] is not None:
 278    newarglist.append(varlist[il][1]);
 279   il = il + 1;
 280  return newarglist;
 281
 282 # hms_string by ArcGIS Python Recipes
 283 # https://arcpy.wordpress.com/2012/04/20/146/
 284 def hms_string(sec_elapsed):
 285  h = int(sec_elapsed / (60 * 60));
 286  m = int((sec_elapsed % (60 * 60)) / 60);
 287  s = sec_elapsed % 60.0;
 288  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 289
 290 # get_readable_size by Lipis
 291 # http://stackoverflow.com/posts/14998888/revisions
 292 def get_readable_size(bytes, precision=1, unit="IEC"):
 293  unit = unit.upper();
 294  if(unit!="IEC" and unit!="SI"):
 295   unit = "IEC";
 296  if(unit=="IEC"):
 297   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 298   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 299   unitsize = 1024.0;
 300  if(unit=="SI"):
 301   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 302   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 303   unitsize = 1000.0;
 304  return_val = {};
 305  orgbytes = bytes;
 306  for unit in units:
 307   if abs(bytes) < unitsize:
 308    strformat = "%3."+str(precision)+"f%s";
 309    pre_return_val = (strformat % (bytes, unit));
 310    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 311    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 312    alt_return_val = pre_return_val.split();
 313    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 314    return return_val;
 315   bytes /= unitsize;
 316  strformat = "%."+str(precision)+"f%s";
 317  pre_return_val = (strformat % (bytes, "YiB"));
 318  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 319  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 320  alt_return_val = pre_return_val.split();
 321  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 322  return return_val;
 323
 324 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 325  unit = unit.upper();
 326  usehashtypes = usehashtypes.lower();
 327  getfilesize = os.path.getsize(infile);
 328  return_val = get_readable_size(getfilesize, precision, unit);
 329  if(usehashes):
 330   hashtypelist = usehashtypes.split(",");
 331   openfile = open(infile, "rb");
 332   filecontents = openfile.read();
 333   openfile.close();
 334   listnumcount = 0;
 335   listnumend = len(hashtypelist);
 336   while(listnumcount < listnumend):
 337    hashtypelistlow = hashtypelist[listnumcount].strip();
 338    hashtypelistup = hashtypelistlow.upper();
 339    filehash = hashlib.new(hashtypelistup);
 340    filehash.update(filecontents);
 341    filegethash = filehash.hexdigest();
 342    return_val.update({hashtypelistup: filegethash});
 343    listnumcount += 1;
 344  return return_val;
 345
 346 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 347  unit = unit.upper();
 348  usehashtypes = usehashtypes.lower();
 349  getfilesize = len(instring);
 350  return_val = get_readable_size(getfilesize, precision, unit);
 351  if(usehashes):
 352   hashtypelist = usehashtypes.split(",");
 353   listnumcount = 0;
 354   listnumend = len(hashtypelist);
 355   while(listnumcount < listnumend):
 356    hashtypelistlow = hashtypelist[listnumcount].strip();
 357    hashtypelistup = hashtypelistlow.upper();
 358    filehash = hashlib.new(hashtypelistup);
 359    if(sys.version[0]=="2"):
 360     filehash.update(instring);
 361    if(sys.version[0]>="3"):
 362     filehash.update(instring.encode('utf-8'));
 363    filegethash = filehash.hexdigest();
 364    return_val.update({hashtypelistup: filegethash});
 365    listnumcount += 1;
 366  return return_val;
 367
 368 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 369  if isinstance(headers, dict):
 370   returnval = [];
 371   if(sys.version[0]=="2"):
 372    for headkey, headvalue in headers.iteritems():
 373     returnval.append((headkey, headvalue));
 374   if(sys.version[0]>="3"):
 375    for headkey, headvalue in headers.items():
 376     returnval.append((headkey, headvalue));
 377  elif isinstance(headers, list):
 378   returnval = headers;
 379  else:
 380   returnval = False;
 381  return returnval;
 382
 383 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 384  if isinstance(headers, dict):
 385   returnval = [];
 386   if(sys.version[0]=="2"):
 387    for headkey, headvalue in headers.iteritems():
 388     returnval.append(headkey+": "+headvalue);
 389   if(sys.version[0]>="3"):
 390    for headkey, headvalue in headers.items():
 391     returnval.append(headkey+": "+headvalue);
 392  elif isinstance(headers, list):
 393   returnval = headers;
 394  else:
 395   returnval = False;
 396  return returnval;
 397
 398 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 399  if isinstance(headers, list):
 400   returnval = {};
 401   mli = 0;
 402   mlil = len(headers);
 403   while(mli<mlil):
 404    returnval.update({headers[mli][0]: headers[mli][1]});
 405    mli = mli + 1;
 406  elif isinstance(headers, dict):
 407   returnval = headers;
 408  else:
 409   returnval = False;
 410  return returnval;
 411
 412 def get_httplib_support(checkvalue=None):
 413  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 414  returnval = [];
 415  returnval.append("ftp");
 416  returnval.append("httplib");
 417  if(havehttplib2):
 418   returnval.append("httplib2");
 419  returnval.append("urllib");
 420  if(haveurllib3):
 421   returnval.append("urllib3");
 422   returnval.append("request3");
 423  returnval.append("request");
 424  if(haverequests):
 425   returnval.append("requests");
 426  if(havehttpx):
 427   returnval.append("httpx");
 428   returnval.append("httpx2");
 429  if(havemechanize):
 430   returnval.append("mechanize");
 431  if(haveparamiko):
 432   returnval.append("sftp");
 433  if(not checkvalue is None):
 434   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 435    checkvalue = "urllib";
 436   if(checkvalue=="httplib1"):
 437    checkvalue = "httplib";
 438   if(checkvalue in returnval):
 439    returnval = True;
 440   else:
 441    returnval = False;
 442  return returnval;
 443
 444 def check_httplib_support(checkvalue="urllib"):
 445  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 446   checkvalue = "urllib";
 447  if(checkvalue=="httplib1"):
 448   checkvalue = "httplib";
 449  returnval = get_httplib_support(checkvalue);
 450  return returnval;
 451
 452 def get_httplib_support_list():
 453  returnval = get_httplib_support(None);
 454  return returnval;
 455
 456 def download_from_url(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 457  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 458  if(sleep<0):
 459   sleep = geturls_download_sleep;
 460  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 461   httplibuse = "urllib";
 462  if(httplibuse=="httplib1"):
 463   httplibuse = "httplib";
 464  if(not haverequests and httplibuse=="requests"):
 465   httplibuse = "urllib";
 466  if(not havehttpx and httplibuse=="httpx"):
 467   httplibuse = "urllib";
 468  if(not havehttpx and httplibuse=="httpx2"):
 469   httplibuse = "urllib";
 470  if(not havemechanize and httplibuse=="mechanize"):
 471   httplibuse = "urllib";
 472  if(not havehttplib2 and httplibuse=="httplib2"):
 473   httplibuse = "httplib";
 474  if(not haveparamiko and httplibuse=="sftp"):
 475   httplibuse = "ftp";
 476  if(httplibuse=="urllib"):
 477   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 478  elif(httplibuse=="request"):
 479   returnval = download_from_url_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 480  elif(httplibuse=="request3"):
 481   returnval = download_from_url_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 482  elif(httplibuse=="httplib"):
 483   returnval = download_from_url_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 484  elif(httplibuse=="httplib2"):
 485   returnval = download_from_url_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 486  elif(httplibuse=="urllib3"):
 487   returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 488  elif(httplibuse=="requests"):
 489   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 490  elif(httplibuse=="httpx"):
 491   returnval = download_from_url_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 492  elif(httplibuse=="httpx2"):
 493   returnval = download_from_url_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 494  elif(httplibuse=="mechanize"):
 495   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 496  elif(httplibuse=="ftp"):
 497   returnval = download_from_url_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 498  elif(httplibuse=="mechanize"):
 499   returnval = download_from_url_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 500  else:
 501   returnval = False;
 502  return returnval;
 503
 504 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
 505  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 506  if(sleep<0):
 507   sleep = geturls_download_sleep;
 508  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 509   httplibuse = "urllib";
 510  if(httplibuse=="httplib1"):
 511   httplibuse = "httplib";
 512  if(not haverequests and httplibuse=="requests"):
 513   httplibuse = "urllib";
 514  if(not havehttpx and httplibuse=="httpx"):
 515   httplibuse = "urllib";
 516  if(not havehttpx and httplibuse=="httpx2"):
 517   httplibuse = "urllib";
 518  if(not havemechanize and httplibuse=="mechanize"):
 519   httplibuse = "urllib";
 520  if(not havehttplib2 and httplibuse=="httplib2"):
 521   httplibuse = "httplib";
 522  if(not haveparamiko and httplibuse=="sftp"):
 523   httplibuse = "ftp";
 524  if(httplibuse=="urllib"):
 525   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 526  elif(httplibuse=="request"):
 527   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 528  elif(httplibuse=="request3"):
 529   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 530  elif(httplibuse=="httplib"):
 531   returnval = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 532  elif(httplibuse=="httplib2"):
 533   returnval = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 534  elif(httplibuse=="urllib3"):
 535   returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 536  elif(httplibuse=="requests"):
 537   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 538  elif(httplibuse=="httpx"):
 539   returnval = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 540  elif(httplibuse=="httpx2"):
 541   returnval = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 542  elif(httplibuse=="mechanize"):
 543   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 544  elif(httplibuse=="ftp"):
 545   returnval = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 546  elif(httplibuse=="sftp"):
 547   returnval = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 548  else:
 549   returnval = False;
 550  return returnval;
 551
 552 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 553  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 554  if(sleep<0):
 555   sleep = geturls_download_sleep;
 556  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 557   httplibuse = "urllib";
 558  if(httplibuse=="httplib1"):
 559   httplibuse = "httplib";
 560  if(not haverequests and httplibuse=="requests"):
 561   httplibuse = "urllib";
 562  if(not havehttpx and httplibuse=="httpx"):
 563   httplibuse = "urllib";
 564  if(not havehttpx and httplibuse=="httpx2"):
 565   httplibuse = "urllib";
 566  if(not havemechanize and httplibuse=="mechanize"):
 567   httplibuse = "urllib";
 568  if(not havehttplib2 and httplibuse=="httplib2"):
 569   httplibuse = "httplib";
 570  if(not haveparamiko and httplibuse=="sftp"):
 571   httplibuse = "ftp";
 572  if(httplibuse=="urllib"):
 573   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 574  elif(httplibuse=="request"):
 575   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 576  elif(httplibuse=="request3"):
 577   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 578  elif(httplibuse=="httplib"):
 579   returnval = download_from_url_to_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 580  elif(httplibuse=="httplib2"):
 581   returnval = download_from_url_to_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 582  elif(httplibuse=="urllib3"):
 583   returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 584  elif(httplibuse=="requests"):
 585   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 586  elif(httplibuse=="httpx"):
 587   returnval = download_from_url_to_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 588  elif(httplibuse=="httpx2"):
 589   returnval = download_from_url_to_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 590  elif(httplibuse=="mechanize"):
 591   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 592  elif(httplibuse=="ftp"):
 593   returnval = download_from_url_to_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 594  elif(httplibuse=="sftp"):
 595   returnval = download_from_url_to_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 596  else:
 597   returnval = False;
 598  return returnval;
 599
 600 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 601  global geturls_download_sleep, havebrotli;
 602  if(sleep<0):
 603   sleep = geturls_download_sleep;
 604  urlparts = urlparse.urlparse(httpurl);
 605  if(isinstance(httpheaders, list)):
 606    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 607  if(urlparts.username is not None or urlparts.password is not None):
 608   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 609   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 610  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 611  if(isinstance(httpheaders, dict)):
 612   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 613  geturls_opener.addheaders = httpheaders;
 614  time.sleep(sleep);
 615  if(postdata is not None and not isinstance(postdata, dict)):
 616   postdata = urlencode(postdata);
 617  try:
 618   if(httpmethod=="GET"):
 619    geturls_text = geturls_opener.open(httpurl);
 620   elif(httpmethod=="POST"):
 621    geturls_text = geturls_opener.open(httpurl, data=postdata);
 622   else:
 623    geturls_text = geturls_opener.open(httpurl);
 624  except HTTPError as geturls_text_error:
 625   geturls_text = geturls_text_error;
 626   log.info("Error With URL "+httpurl);
 627  except URLError:
 628   log.info("Error With URL "+httpurl);
 629   return False;
 630  except socket.timeout:
 631   log.info("Error With URL "+httpurl);
 632   return False;
 633  log.info("Downloading URL "+httpurl);
 634  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 635   if(sys.version[0]=="2"):
 636    strbuf = StringIO(geturls_text.read());
 637   if(sys.version[0]>="3"):
 638    strbuf = BytesIO(geturls_text.read());
 639   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 640   returnval_content = gzstrbuf.read()[:];
 641  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 642   returnval_content = geturls_text.read()[:];
 643  if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
 644   returnval_content = brotli.decompress(returnval_content);
 645  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 646  geturls_text.close();
 647  return returnval;
 648
 649 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 650  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 651  exec_time_start = time.time();
 652  myhash = hashlib.new("sha1");
 653  if(sys.version[0]=="2"):
 654   myhash.update(httpurl);
 655   myhash.update(str(buffersize));
 656   myhash.update(str(exec_time_start));
 657  if(sys.version[0]>="3"):
 658   myhash.update(httpurl.encode('utf-8'));
 659   myhash.update(str(buffersize).encode('utf-8'));
 660   myhash.update(str(exec_time_start).encode('utf-8'));
 661  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 662  if(sleep<0):
 663   sleep = geturls_download_sleep;
 664  urlparts = urlparse.urlparse(httpurl);
 665  if(isinstance(httpheaders, list)):
 666    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 667  if(urlparts.username is not None or urlparts.password is not None):
 668   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 669   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 670  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 671  if(isinstance(httpheaders, dict)):
 672   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 673  geturls_opener.addheaders = httpheaders;
 674  time.sleep(sleep);
 675  try:
 676   if(httpmethod=="GET"):
 677    geturls_text = geturls_opener.open(httpurl);
 678   elif(httpmethod=="POST"):
 679    geturls_text = geturls_opener.open(httpurl, data=postdata);
 680   else:
 681    geturls_text = geturls_opener.open(httpurl);
 682  except HTTPError as geturls_text_error:
 683   geturls_text = geturls_text_error;
 684   log.info("Error With URL "+httpurl);
 685  except URLError:
 686   log.info("Error With URL "+httpurl);
 687   return False;
 688  except socket.timeout:
 689   log.info("Error With URL "+httpurl);
 690   return False;
 691  except socket.timeout:
 692   log.info("Error With URL "+httpurl);
 693   return False;
 694  downloadsize = geturls_text.info().get('Content-Length');
 695  if(downloadsize is not None):
 696   downloadsize = int(downloadsize);
 697  if downloadsize is None: downloadsize = 0;
 698  fulldatasize = 0;
 699  prevdownsize = 0;
 700  log.info("Downloading URL "+httpurl);
 701  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 702   tmpfilename = f.name;
 703   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 704   while True:
 705    databytes = geturls_text.read(buffersize);
 706    if not databytes: break;
 707    datasize = len(databytes);
 708    fulldatasize = datasize + fulldatasize;
 709    percentage = "";
 710    if(downloadsize>0):
 711     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 712    downloaddiff = fulldatasize - prevdownsize;
 713    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 714    prevdownsize = fulldatasize;
 715    f.write(databytes);
 716   f.close();
 717  geturls_text.close();
 718  exec_time_end = time.time();
 719  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 720  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 721  return returnval;
 722
 723 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 724  global geturls_download_sleep;
 725  if(sleep<0):
 726   sleep = geturls_download_sleep;
 727  if(not outfile=="-"):
 728   outpath = outpath.rstrip(os.path.sep);
 729   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 730   if(not os.path.exists(outpath)):
 731    os.makedirs(outpath);
 732   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 733    return False;
 734   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 735    return False;
 736   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 737   if(not pretmpfilename):
 738    return False;
 739   tmpfilename = pretmpfilename['Filename'];
 740   downloadsize = os.path.getsize(tmpfilename);
 741   fulldatasize = 0;
 742   log.info("Moving file "+tmpfilename+" to "+filepath);
 743   exec_time_start = time.time();
 744   shutil.move(tmpfilename, filepath);
 745   exec_time_end = time.time();
 746   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 747   if(os.path.exists(tmpfilename)):
 748    os.remove(tmpfilename);
 749   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 750  if(outfile=="-" and sys.version[0]=="2"):
 751   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 752   if(not pretmpfilename):
 753    return False;
 754   tmpfilename = pretmpfilename['Filename'];
 755   downloadsize = os.path.getsize(tmpfilename);
 756   fulldatasize = 0;
 757   prevdownsize = 0;
 758   exec_time_start = time.time();
 759   with open(tmpfilename, 'rb') as ft:
 760    f = StringIO();
 761    while True:
 762     databytes = ft.read(buffersize[1]);
 763     if not databytes: break;
 764     datasize = len(databytes);
 765     fulldatasize = datasize + fulldatasize;
 766     percentage = "";
 767     if(downloadsize>0):
 768      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 769     downloaddiff = fulldatasize - prevdownsize;
 770     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 771     prevdownsize = fulldatasize;
 772     f.write(databytes);
 773    f.seek(0);
 774    fdata = f.getvalue();
 775    f.close();
 776    ft.close();
 777    os.remove(tmpfilename);
 778    exec_time_end = time.time();
 779    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 780   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 781  if(outfile=="-" and sys.version[0]>="3"):
 782   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 783   tmpfilename = pretmpfilename['Filename'];
 784   downloadsize = os.path.getsize(tmpfilename);
 785   fulldatasize = 0;
 786   prevdownsize = 0;
 787   exec_time_start = time.time();
 788   with open(tmpfilename, 'rb') as ft:
 789    f = BytesIO();
 790    while True:
 791     databytes = ft.read(buffersize[1]);
 792     if not databytes: break;
 793     datasize = len(databytes);
 794     fulldatasize = datasize + fulldatasize;
 795     percentage = "";
 796     if(downloadsize>0):
 797      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 798     downloaddiff = fulldatasize - prevdownsize;
 799     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 800     prevdownsize = fulldatasize;
 801     f.write(databytes);
 802    f.seek(0);
 803    fdata = f.getvalue();
 804    f.close();
 805    ft.close();
 806    os.remove(tmpfilename);
 807    exec_time_end = time.time();
 808    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 809   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 810  return returnval;
 811
 812 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 813  global geturls_download_sleep, havebrotli;
 814  if(sleep<0):
 815   sleep = geturls_download_sleep;
 816  urlparts = urlparse.urlparse(httpurl);
 817  if(isinstance(httpheaders, list)):
 818    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 819  if(urlparts.username is not None or urlparts.password is not None):
 820   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 821   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 822  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 823  geturls_opener.addheaders = httpheaders;
 824  time.sleep(sleep);
 825  if(urlparts[0]=="http"):
 826   httpconn = HTTPConnection(urlparts[1]);
 827  elif(urlparts[0]=="https"):
 828   httpconn = HTTPSConnection(urlparts[1]);
 829  else:
 830   return False;
 831  if(postdata is not None and not isinstance(postdata, dict)):
 832   postdata = urlencode(postdata);
 833  try:
 834   if(httpmethod=="GET"):
 835    httpconn.request("GET", urlparts[2], headers=httpheaders);
 836   elif(httpmethod=="POST"):
 837    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 838   else:
 839    httpconn.request("GET", urlparts[2], headers=httpheaders);
 840  except socket.timeout:
 841   log.info("Error With URL "+httpurl);
 842   return False;
 843  geturls_text = httpconn.getresponse();
 844  log.info("Downloading URL "+httpurl);
 845  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
 846   if(sys.version[0]=="2"):
 847    strbuf = StringIO(geturls_text.read());
 848   if(sys.version[0]>="3"):
 849    strbuf = BytesIO(geturls_text.read());
 850   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 851   returnval_content = gzstrbuf.read()[:];
 852  if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
 853   returnval_content = geturls_text.read()[:];
 854  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
 855   returnval_content = brotli.decompress(returnval_content);
 856  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 857  geturls_text.close();
 858  return returnval;
 859
 860 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 861  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 862  exec_time_start = time.time();
 863  myhash = hashlib.new("sha1");
 864  if(sys.version[0]=="2"):
 865   myhash.update(httpurl);
 866   myhash.update(str(buffersize));
 867   myhash.update(str(exec_time_start));
 868  if(sys.version[0]>="3"):
 869   myhash.update(httpurl.encode('utf-8'));
 870   myhash.update(str(buffersize).encode('utf-8'));
 871   myhash.update(str(exec_time_start).encode('utf-8'));
 872  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 873  if(sleep<0):
 874   sleep = geturls_download_sleep;
 875  urlparts = urlparse.urlparse(httpurl);
 876  if(isinstance(httpheaders, list)):
 877    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 878  if(urlparts.username is not None or urlparts.password is not None):
 879   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 880   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 881  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 882  geturls_opener.addheaders = httpheaders;
 883  time.sleep(sleep);
 884  if(urlparts[0]=="http"):
 885   httpconn = HTTPConnection(urlparts[1]);
 886  elif(urlparts[0]=="https"):
 887   httpconn = HTTPSConnection(urlparts[1]);
 888  else:
 889   return False;
 890  if(postdata is not None and not isinstance(postdata, dict)):
 891   postdata = urlencode(postdata);
 892  try:
 893   if(httpmethod=="GET"):
 894    httpconn.request("GET", urlparts[2], headers=httpheaders);
 895   elif(httpmethod=="POST"):
 896    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 897   else:
 898    httpconn.request("GET", urlparts[2], headers=httpheaders);
 899  except socket.timeout:
 900   log.info("Error With URL "+httpurl);
 901   return False;
 902  geturls_text = httpconn.getresponse();
 903  downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
 904  if(downloadsize is not None):
 905   downloadsize = int(downloadsize);
 906  if downloadsize is None: downloadsize = 0;
 907  fulldatasize = 0;
 908  prevdownsize = 0;
 909  log.info("Downloading URL "+httpurl);
 910  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 911   tmpfilename = f.name;
 912   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 913   while True:
 914    databytes = geturls_text.read(buffersize);
 915    if not databytes: break;
 916    datasize = len(databytes);
 917    fulldatasize = datasize + fulldatasize;
 918    percentage = "";
 919    if(downloadsize>0):
 920     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 921    downloaddiff = fulldatasize - prevdownsize;
 922    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 923    prevdownsize = fulldatasize;
 924    f.write(databytes);
 925   f.close();
 926  geturls_text.close();
 927  exec_time_end = time.time();
 928  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 929  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 930  return returnval;
 931
 932 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 933  global geturls_download_sleep;
 934  if(sleep<0):
 935   sleep = geturls_download_sleep;
 936  if(not outfile=="-"):
 937   outpath = outpath.rstrip(os.path.sep);
 938   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 939   if(not os.path.exists(outpath)):
 940    os.makedirs(outpath);
 941   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 942    return False;
 943   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 944    return False;
 945   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 946   if(not pretmpfilename):
 947    return False;
 948   tmpfilename = pretmpfilename['Filename'];
 949   downloadsize = os.path.getsize(tmpfilename);
 950   fulldatasize = 0;
 951   log.info("Moving file "+tmpfilename+" to "+filepath);
 952   exec_time_start = time.time();
 953   shutil.move(tmpfilename, filepath);
 954   exec_time_end = time.time();
 955   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 956   if(os.path.exists(tmpfilename)):
 957    os.remove(tmpfilename);
 958   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 959  if(outfile=="-" and sys.version[0]=="2"):
 960   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 961   if(not pretmpfilename):
 962    return False;
 963   tmpfilename = pretmpfilename['Filename'];
 964   downloadsize = os.path.getsize(tmpfilename);
 965   fulldatasize = 0;
 966   prevdownsize = 0;
 967   exec_time_start = time.time();
 968   with open(tmpfilename, 'rb') as ft:
 969    f = StringIO();
 970    while True:
 971     databytes = ft.read(buffersize[1]);
 972     if not databytes: break;
 973     datasize = len(databytes);
 974     fulldatasize = datasize + fulldatasize;
 975     percentage = "";
 976     if(downloadsize>0):
 977      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 978     downloaddiff = fulldatasize - prevdownsize;
 979     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 980     prevdownsize = fulldatasize;
 981     f.write(databytes);
 982    f.seek(0);
 983    fdata = f.getvalue();
 984    f.close();
 985    ft.close();
 986    os.remove(tmpfilename);
 987    exec_time_end = time.time();
 988    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 989   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 990  if(outfile=="-" and sys.version[0]>="3"):
 991   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 992   tmpfilename = pretmpfilename['Filename'];
 993   downloadsize = os.path.getsize(tmpfilename);
 994   fulldatasize = 0;
 995   prevdownsize = 0;
 996   exec_time_start = time.time();
 997   with open(tmpfilename, 'rb') as ft:
 998    f = BytesIO();
 999    while True:
1000     databytes = ft.read(buffersize[1]);
1001     if not databytes: break;
1002     datasize = len(databytes);
1003     fulldatasize = datasize + fulldatasize;
1004     percentage = "";
1005     if(downloadsize>0):
1006      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1007     downloaddiff = fulldatasize - prevdownsize;
1008     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1009     prevdownsize = fulldatasize;
1010     f.write(databytes);
1011    f.seek(0);
1012    fdata = f.getvalue();
1013    f.close();
1014    ft.close();
1015    os.remove(tmpfilename);
1016    exec_time_end = time.time();
1017    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1018   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1019  return returnval;
1020
1021 if(havehttplib2):
1022  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1023   global geturls_download_sleep, havebrotli;
1024   if(sleep<0):
1025    sleep = geturls_download_sleep;
1026   urlparts = urlparse.urlparse(httpurl);
1027   if(isinstance(httpheaders, list)):
1028    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1029   if(urlparts.username is not None or urlparts.password is not None):
1030    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1031    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1032   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1033   geturls_opener.addheaders = httpheaders;
1034   time.sleep(sleep);
1035   if(urlparts[0]=="http"):
1036    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1037   elif(urlparts[0]=="https"):
1038    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1039   else:
1040    return False;
1041   if(postdata is not None and not isinstance(postdata, dict)):
1042    postdata = urlencode(postdata);
1043   try:
1044    if(httpmethod=="GET"):
1045     httpconn.request("GET", urlparts[2], headers=httpheaders);
1046    elif(httpmethod=="POST"):
1047     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1048    else:
1049     httpconn.request("GET", urlparts[2], headers=httpheaders);
1050   except socket.timeout:
1051    log.info("Error With URL "+httpurl);
1052    return False;
1053   geturls_text = httpconn.getresponse();
1054   log.info("Downloading URL "+httpurl);
1055   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
1056    if(sys.version[0]=="2"):
1057     strbuf = StringIO(geturls_text.read());
1058    if(sys.version[0]>="3"):
1059     strbuf = BytesIO(geturls_text.read());
1060    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1061    returnval_content = gzstrbuf.read()[:];
1062   if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
1063    returnval_content = geturls_text.read()[:];
1064   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
1065    returnval_content = brotli.decompress(returnval_content);
1066   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1067   geturls_text.close();
1068   return returnval;
1069
1070 if(not havehttplib2):
1071  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1072   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1073   return returnval;
1074
1075 if(havehttplib2):
1076  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1077   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1078   exec_time_start = time.time();
1079   myhash = hashlib.new("sha1");
1080   if(sys.version[0]=="2"):
1081    myhash.update(httpurl);
1082    myhash.update(str(buffersize));
1083    myhash.update(str(exec_time_start));
1084   if(sys.version[0]>="3"):
1085    myhash.update(httpurl.encode('utf-8'));
1086    myhash.update(str(buffersize).encode('utf-8'));
1087    myhash.update(str(exec_time_start).encode('utf-8'));
1088   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1089   if(sleep<0):
1090    sleep = geturls_download_sleep;
1091   urlparts = urlparse.urlparse(httpurl);
1092   if(isinstance(httpheaders, list)):
1093    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1094   if(urlparts.username is not None or urlparts.password is not None):
1095    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1096    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1097   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1098   geturls_opener.addheaders = httpheaders;
1099   time.sleep(sleep);
1100   if(urlparts[0]=="http"):
1101    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1102   elif(urlparts[0]=="https"):
1103    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1104   else:
1105    return False;
1106   if(postdata is not None and not isinstance(postdata, dict)):
1107    postdata = urlencode(postdata);
1108   try:
1109    if(httpmethod=="GET"):
1110     httpconn.request("GET", urlparts[2], headers=httpheaders);
1111    elif(httpmethod=="POST"):
1112     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1113    else:
1114     httpconn.request("GET", urlparts[2], headers=httpheaders);
1115   except socket.timeout:
1116    log.info("Error With URL "+httpurl);
1117    return False;
1118   geturls_text = httpconn.getresponse();
1119   downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
1120   if(downloadsize is not None):
1121    downloadsize = int(downloadsize);
1122   if downloadsize is None: downloadsize = 0;
1123   fulldatasize = 0;
1124   prevdownsize = 0;
1125   log.info("Downloading URL "+httpurl);
1126   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1127    tmpfilename = f.name;
1128    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1129    while True:
1130     databytes = geturls_text.read(buffersize);
1131     if not databytes: break;
1132     datasize = len(databytes);
1133     fulldatasize = datasize + fulldatasize;
1134     percentage = "";
1135     if(downloadsize>0):
1136      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1137     downloaddiff = fulldatasize - prevdownsize;
1138     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1139     prevdownsize = fulldatasize;
1140     f.write(databytes);
1141    f.close();
1142   geturls_text.close();
1143   exec_time_end = time.time();
1144   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1145   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1146   return returnval;
1147
1148 if(not havehttplib2):
1149  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1150   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1151   return returnval;
1152
1153 if(havehttplib2):
1154  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1155   global geturls_download_sleep;
1156   if(sleep<0):
1157    sleep = geturls_download_sleep;
1158   if(not outfile=="-"):
1159    outpath = outpath.rstrip(os.path.sep);
1160    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1161    if(not os.path.exists(outpath)):
1162     os.makedirs(outpath);
1163    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1164     return False;
1165    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1166     return False;
1167    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1168    if(not pretmpfilename):
1169     return False;
1170    tmpfilename = pretmpfilename['Filename'];
1171    downloadsize = os.path.getsize(tmpfilename);
1172    fulldatasize = 0;
1173    log.info("Moving file "+tmpfilename+" to "+filepath);
1174    exec_time_start = time.time();
1175    shutil.move(tmpfilename, filepath);
1176    exec_time_end = time.time();
1177    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1178    if(os.path.exists(tmpfilename)):
1179     os.remove(tmpfilename);
1180    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1181   if(outfile=="-" and sys.version[0]=="2"):
1182    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1183    if(not pretmpfilename):
1184     return False;
1185    tmpfilename = pretmpfilename['Filename'];
1186    downloadsize = os.path.getsize(tmpfilename);
1187    fulldatasize = 0;
1188    prevdownsize = 0;
1189    exec_time_start = time.time();
1190    with open(tmpfilename, 'rb') as ft:
1191     f = StringIO();
1192     while True:
1193      databytes = ft.read(buffersize[1]);
1194      if not databytes: break;
1195      datasize = len(databytes);
1196      fulldatasize = datasize + fulldatasize;
1197      percentage = "";
1198      if(downloadsize>0):
1199       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1200      downloaddiff = fulldatasize - prevdownsize;
1201      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1202      prevdownsize = fulldatasize;
1203      f.write(databytes);
1204     f.seek(0);
1205     fdata = f.getvalue();
1206     f.close();
1207     ft.close();
1208     os.remove(tmpfilename);
1209     exec_time_end = time.time();
1210     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1211    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1212   if(outfile=="-" and sys.version[0]>="3"):
1213    pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1214    tmpfilename = pretmpfilename['Filename'];
1215    downloadsize = os.path.getsize(tmpfilename);
1216    fulldatasize = 0;
1217    prevdownsize = 0;
1218    exec_time_start = time.time();
1219    with open(tmpfilename, 'rb') as ft:
1220     f = BytesIO();
1221     while True:
1222      databytes = ft.read(buffersize[1]);
1223      if not databytes: break;
1224      datasize = len(databytes);
1225      fulldatasize = datasize + fulldatasize;
1226      percentage = "";
1227      if(downloadsize>0):
1228       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1229      downloaddiff = fulldatasize - prevdownsize;
1230      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1231      prevdownsize = fulldatasize;
1232      f.write(databytes);
1233     f.seek(0);
1234     fdata = f.getvalue();
1235     f.close();
1236     ft.close();
1237     os.remove(tmpfilename);
1238     exec_time_end = time.time();
1239     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1240    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1241   return returnval;
1242
1243 if(not havehttplib2):
1244  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1245   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1246   return returnval;
1247
1248 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1249  global geturls_download_sleep, havebrotli;
1250  if(sleep<0):
1251   sleep = geturls_download_sleep;
1252  urlparts = urlparse.urlparse(httpurl);
1253  if(isinstance(httpheaders, list)):
1254    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1255  if(urlparts.username is not None or urlparts.password is not None):
1256   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1257   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1258  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1259  if(isinstance(httpheaders, dict)):
1260   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1261  geturls_opener.addheaders = httpheaders;
1262  urllib.request.install_opener(geturls_opener);
1263  time.sleep(sleep);
1264  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1265  if(postdata is not None and not isinstance(postdata, dict)):
1266   postdata = urlencode(postdata);
1267  try:
1268   if(httpmethod=="GET"):
1269    geturls_request = Request(httpurl, headers=httpheaders);
1270    geturls_text = urlopen(geturls_request);
1271   elif(httpmethod=="POST"):
1272    geturls_request = Request(httpurl, headers=httpheaders);
1273    geturls_text = urlopen(geturls_request, data=postdata);
1274   else:
1275    geturls_request = Request(httpurl, headers=httpheaders);
1276    geturls_text = urlopen(geturls_request);
1277  except HTTPError as geturls_text_error:
1278   geturls_text = geturls_text_error;
1279   log.info("Error With URL "+httpurl);
1280  except URLError:
1281   log.info("Error With URL "+httpurl);
1282   return False;
1283  except socket.timeout:
1284   log.info("Error With URL "+httpurl);
1285   return False;
1286  log.info("Downloading URL "+httpurl);
1287  if(geturls_text.headers.get("Content-Encoding")=="gzip" or geturls_text.headers.get("Content-Encoding")=="deflate"):
1288   if(sys.version[0]=="2"):
1289    strbuf = StringIO(geturls_text.read());
1290   if(sys.version[0]>="3"):
1291    strbuf = BytesIO(geturls_text.read());
1292   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1293   returnval_content = gzstrbuf.read()[:];
1294  if(geturls_text.headers.get("Content-Encoding")!="gzip" and geturls_text.headers.get("Content-Encoding")!="deflate"):
1295   returnval_content = geturls_text.read()[:];
1296  if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1297   returnval_content = brotli.decompress(returnval_content);
1298  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1299  geturls_text.close();
1300  return returnval;
1301
1302 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1303  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1304  exec_time_start = time.time();
1305  myhash = hashlib.new("sha1");
1306  if(sys.version[0]=="2"):
1307   myhash.update(httpurl);
1308   myhash.update(str(buffersize));
1309   myhash.update(str(exec_time_start));
1310  if(sys.version[0]>="3"):
1311   myhash.update(httpurl.encode('utf-8'));
1312   myhash.update(str(buffersize).encode('utf-8'));
1313   myhash.update(str(exec_time_start).encode('utf-8'));
1314  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1315  if(sleep<0):
1316   sleep = geturls_download_sleep;
1317  urlparts = urlparse.urlparse(httpurl);
1318  if(isinstance(httpheaders, list)):
1319    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1320  if(urlparts.username is not None or urlparts.password is not None):
1321   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1322   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1323  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1324  if(isinstance(httpheaders, dict)):
1325   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1326  geturls_opener.addheaders = httpheaders;
1327  urllib.request.install_opener(geturls_opener);
1328  time.sleep(sleep);
1329  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1330  if(postdata is not None and not isinstance(postdata, dict)):
1331   postdata = urlencode(postdata);
1332  try:
1333   if(httpmethod=="GET"):
1334    geturls_request = Request(httpurl, headers=httpheaders);
1335    geturls_text = urlopen(geturls_request);
1336   elif(httpmethod=="POST"):
1337    geturls_request = Request(httpurl, headers=httpheaders);
1338    geturls_text = urlopen(geturls_request, data=postdata);
1339   else:
1340    geturls_request = Request(httpurl, headers=httpheaders);
1341    geturls_text = urlopen(geturls_request);
1342  except HTTPError as geturls_text_error:
1343   geturls_text = geturls_text_error;
1344   log.info("Error With URL "+httpurl);
1345  except URLError:
1346   log.info("Error With URL "+httpurl);
1347   return False;
1348  except socket.timeout:
1349   log.info("Error With URL "+httpurl);
1350   return False;
1351  downloadsize = geturls_text.headers.get('Content-Length');
1352  if(downloadsize is not None):
1353   downloadsize = int(downloadsize);
1354  if downloadsize is None: downloadsize = 0;
1355  fulldatasize = 0;
1356  prevdownsize = 0;
1357  log.info("Downloading URL "+httpurl);
1358  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1359   tmpfilename = f.name;
1360   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1361   while True:
1362    databytes = geturls_text.read(buffersize);
1363    if not databytes: break;
1364    datasize = len(databytes);
1365    fulldatasize = datasize + fulldatasize;
1366    percentage = "";
1367    if(downloadsize>0):
1368     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1369    downloaddiff = fulldatasize - prevdownsize;
1370    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1371    prevdownsize = fulldatasize;
1372    f.write(databytes);
1373   f.close();
1374  geturls_text.close();
1375  exec_time_end = time.time();
1376  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1377  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1378  return returnval;
1379
1380 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1381  global geturls_download_sleep;
1382  if(sleep<0):
1383   sleep = geturls_download_sleep;
1384  if(not outfile=="-"):
1385   outpath = outpath.rstrip(os.path.sep);
1386   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1387   if(not os.path.exists(outpath)):
1388    os.makedirs(outpath);
1389   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1390    return False;
1391   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1392    return False;
1393   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1394   if(not pretmpfilename):
1395    return False;
1396   tmpfilename = pretmpfilename['Filename'];
1397   downloadsize = os.path.getsize(tmpfilename);
1398   fulldatasize = 0;
1399   log.info("Moving file "+tmpfilename+" to "+filepath);
1400   exec_time_start = time.time();
1401   shutil.move(tmpfilename, filepath);
1402   exec_time_end = time.time();
1403   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1404   if(os.path.exists(tmpfilename)):
1405    os.remove(tmpfilename);
1406   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent':pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1407  if(outfile=="-" and sys.version[0]=="2"):
1408   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1409   if(not pretmpfilename):
1410    return False;
1411   tmpfilename = pretmpfilename['Filename'];
1412   downloadsize = os.path.getsize(tmpfilename);
1413   fulldatasize = 0;
1414   prevdownsize = 0;
1415   exec_time_start = time.time();
1416   with open(tmpfilename, 'rb') as ft:
1417    f = StringIO();
1418    while True:
1419     databytes = ft.read(buffersize[1]);
1420     if not databytes: break;
1421     datasize = len(databytes);
1422     fulldatasize = datasize + fulldatasize;
1423     percentage = "";
1424     if(downloadsize>0):
1425      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1426     downloaddiff = fulldatasize - prevdownsize;
1427     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1428     prevdownsize = fulldatasize;
1429     f.write(databytes);
1430    f.seek(0);
1431    fdata = f.getvalue();
1432    f.close();
1433    ft.close();
1434    os.remove(tmpfilename);
1435    exec_time_end = time.time();
1436    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1437   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1438  if(outfile=="-" and sys.version[0]>="3"):
1439   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1440   tmpfilename = pretmpfilename['Filename'];
1441   downloadsize = os.path.getsize(tmpfilename);
1442   fulldatasize = 0;
1443   prevdownsize = 0;
1444   exec_time_start = time.time();
1445   with open(tmpfilename, 'rb') as ft:
1446    f = BytesIO();
1447    while True:
1448     databytes = ft.read(buffersize[1]);
1449     if not databytes: break;
1450     datasize = len(databytes);
1451     fulldatasize = datasize + fulldatasize;
1452     percentage = "";
1453     if(downloadsize>0):
1454      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1455     downloaddiff = fulldatasize - prevdownsize;
1456     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1457     prevdownsize = fulldatasize;
1458     f.write(databytes);
1459    f.seek(0);
1460    fdata = f.getvalue();
1461    f.close();
1462    ft.close();
1463    os.remove(tmpfilename);
1464    exec_time_end = time.time();
1465    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1466   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1467  return returnval;
1468
1469 if(haverequests):
1470  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1471   global geturls_download_sleep, havebrotli;
1472   if(sleep<0):
1473    sleep = geturls_download_sleep;
1474   urlparts = urlparse.urlparse(httpurl);
1475   if(isinstance(httpheaders, list)):
1476    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1477   if(urlparts.username is not None or urlparts.password is not None):
1478    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1479    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1480   time.sleep(sleep);
1481   if(postdata is not None and not isinstance(postdata, dict)):
1482    postdata = urlencode(postdata);
1483   try:
1484    if(httpmethod=="GET"):
1485     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1486    elif(httpmethod=="POST"):
1487     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1488    else:
1489     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1490   except requests.exceptions.ConnectTimeout:
1491    log.info("Error With URL "+httpurl);
1492    return False;
1493   except socket.timeout:
1494    log.info("Error With URL "+httpurl);
1495    return False;
1496   log.info("Downloading URL "+httpurl);
1497   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1498    if(sys.version[0]=="2"):
1499     strbuf = StringIO(geturls_text.content);
1500    if(sys.version[0]>="3"):
1501     strbuf = BytesIO(geturls_text.content);
1502    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1503    returnval_content = gzstrbuf.content[:];
1504   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1505    returnval_content = geturls_text.content[:];
1506   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1507    returnval_content = brotli.decompress(returnval_content);
1508   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1509   geturls_text.close();
1510   return returnval;
1511
1512 if(not haverequests):
1513  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1514   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1515   return returnval;
1516
1517 if(haverequests):
1518  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1519   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1520   exec_time_start = time.time();
1521   myhash = hashlib.new("sha1");
1522   if(sys.version[0]=="2"):
1523    myhash.update(httpurl);
1524    myhash.update(str(buffersize));
1525    myhash.update(str(exec_time_start));
1526   if(sys.version[0]>="3"):
1527    myhash.update(httpurl.encode('utf-8'));
1528    myhash.update(str(buffersize).encode('utf-8'));
1529    myhash.update(str(exec_time_start).encode('utf-8'));
1530   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1531   if(sleep<0):
1532    sleep = geturls_download_sleep;
1533   urlparts = urlparse.urlparse(httpurl);
1534   if(isinstance(httpheaders, list)):
1535    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1536   if(urlparts.username is not None or urlparts.password is not None):
1537    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1538    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1539   time.sleep(sleep);
1540   if(postdata is not None and not isinstance(postdata, dict)):
1541    postdata = urlencode(postdata);
1542   try:
1543    if(httpmethod=="GET"):
1544     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1545    elif(httpmethod=="POST"):
1546     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1547    else:
1548     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1549   except requests.exceptions.ConnectTimeout:
1550    log.info("Error With URL "+httpurl);
1551    return False;
1552   except socket.timeout:
1553    log.info("Error With URL "+httpurl);
1554    return False;
1555   downloadsize = int(geturls_text.headers.get('Content-Length'));
1556   if(downloadsize is not None):
1557    downloadsize = int(downloadsize);
1558   if downloadsize is None: downloadsize = 0;
1559   fulldatasize = 0;
1560   prevdownsize = 0;
1561   log.info("Downloading URL "+httpurl);
1562   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1563    tmpfilename = f.name;
1564    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1565    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1566     datasize = len(databytes);
1567     fulldatasize = datasize + fulldatasize;
1568     percentage = "";
1569     if(downloadsize>0):
1570      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1571     downloaddiff = fulldatasize - prevdownsize;
1572     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1573     prevdownsize = fulldatasize;
1574     f.write(databytes);
1575    f.close();
1576   geturls_text.close();
1577   exec_time_end = time.time();
1578   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1579   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1580   return returnval;
1581
1582 if(not haverequests):
1583  def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1584   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1585   return returnval;
1586
1587 if(haverequests):
1588  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1589   global geturls_download_sleep;
1590   if(sleep<0):
1591    sleep = geturls_download_sleep;
1592   if(not outfile=="-"):
1593    outpath = outpath.rstrip(os.path.sep);
1594    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1595    if(not os.path.exists(outpath)):
1596     os.makedirs(outpath);
1597    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1598     return False;
1599    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1600     return False;
1601    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1602    if(not pretmpfilename):
1603     return False;
1604    tmpfilename = pretmpfilename['Filename'];
1605    downloadsize = os.path.getsize(tmpfilename);
1606    fulldatasize = 0;
1607    log.info("Moving file "+tmpfilename+" to "+filepath);
1608    exec_time_start = time.time();
1609    shutil.move(tmpfilename, filepath);
1610    exec_time_end = time.time();
1611    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1612    if(os.path.exists(tmpfilename)):
1613     os.remove(tmpfilename);
1614    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1615   if(outfile=="-" and sys.version[0]=="2"):
1616    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1617    if(not pretmpfilename):
1618     return False;
1619    tmpfilename = pretmpfilename['Filename'];
1620    downloadsize = os.path.getsize(tmpfilename);
1621    fulldatasize = 0;
1622    prevdownsize = 0;
1623    exec_time_start = time.time();
1624    with open(tmpfilename, 'rb') as ft:
1625     f = StringIO();
1626     while True:
1627      databytes = ft.read(buffersize[1]);
1628      if not databytes: break;
1629      datasize = len(databytes);
1630      fulldatasize = datasize + fulldatasize;
1631      percentage = "";
1632      if(downloadsize>0):
1633       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1634      downloaddiff = fulldatasize - prevdownsize;
1635      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1636      prevdownsize = fulldatasize;
1637      f.write(databytes);
1638     f.seek(0);
1639     fdata = f.getvalue();
1640     f.close();
1641     ft.close();
1642     os.remove(tmpfilename);
1643     exec_time_end = time.time();
1644     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1645    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1646   if(outfile=="-" and sys.version[0]>="3"):
1647    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1648    tmpfilename = pretmpfilename['Filename'];
1649    downloadsize = os.path.getsize(tmpfilename);
1650    fulldatasize = 0;
1651    prevdownsize = 0;
1652    exec_time_start = time.time();
1653    with open(tmpfilename, 'rb') as ft:
1654     f = BytesIO();
1655     while True:
1656      databytes = ft.read(buffersize[1]);
1657      if not databytes: break;
1658      datasize = len(databytes);
1659      fulldatasize = datasize + fulldatasize;
1660      percentage = "";
1661      if(downloadsize>0):
1662       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1663      downloaddiff = fulldatasize - prevdownsize;
1664      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1665      prevdownsize = fulldatasize;
1666      f.write(databytes);
1667     f.seek(0);
1668     fdata = f.getvalue();
1669     f.close();
1670     ft.close();
1671     os.remove(tmpfilename);
1672     exec_time_end = time.time();
1673     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1674    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1675   return returnval;
1676
1677 if(not haverequests):
1678  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1679   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1680   return returnval;
1681
1682 if(havehttpx):
1683  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1684   global geturls_download_sleep, havebrotli;
1685   if(sleep<0):
1686    sleep = geturls_download_sleep;
1687   urlparts = urlparse.urlparse(httpurl);
1688   if(isinstance(httpheaders, list)):
1689    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1690   if(urlparts.username is not None or urlparts.password is not None):
1691    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1692    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1693   time.sleep(sleep);
1694   if(postdata is not None and not isinstance(postdata, dict)):
1695    postdata = urlencode(postdata);
1696   try:
1697    if(httpmethod=="GET"):
1698     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1699     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1700    elif(httpmethod=="POST"):
1701     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1702     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1703    else:
1704     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1705     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1706   except httpx.ConnectTimeout:
1707    log.info("Error With URL "+httpurl);
1708    return False;
1709   except socket.timeout:
1710    log.info("Error With URL "+httpurl);
1711    return False;
1712   log.info("Downloading URL "+httpurl);
1713   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1714    if(sys.version[0]=="2"):
1715     strbuf = StringIO(geturls_text.content);
1716    if(sys.version[0]>="3"):
1717     strbuf = BytesIO(geturls_text.content);
1718    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1719    returnval_content = gzstrbuf.content[:];
1720   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1721    returnval_content = geturls_text.content[:];
1722   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1723    returnval_content = brotli.decompress(returnval_content);
1724   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1725   geturls_text.close();
1726   return returnval;
1727
1728 if(not havehttpx):
1729  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1730   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1731   return returnval;
1732
1733 if(havehttpx):
1734  def download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1735   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1736   exec_time_start = time.time();
1737   myhash = hashlib.new("sha1");
1738   if(sys.version[0]=="2"):
1739    myhash.update(httpurl);
1740    myhash.update(str(buffersize));
1741    myhash.update(str(exec_time_start));
1742   if(sys.version[0]>="3"):
1743    myhash.update(httpurl.encode('utf-8'));
1744    myhash.update(str(buffersize).encode('utf-8'));
1745    myhash.update(str(exec_time_start).encode('utf-8'));
1746   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1747   if(sleep<0):
1748    sleep = geturls_download_sleep;
1749   urlparts = urlparse.urlparse(httpurl);
1750   if(isinstance(httpheaders, list)):
1751    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1752   if(urlparts.username is not None or urlparts.password is not None):
1753    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1754    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1755   time.sleep(sleep);
1756   if(postdata is not None and not isinstance(postdata, dict)):
1757    postdata = urlencode(postdata);
1758   try:
1759    if(httpmethod=="GET"):
1760     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1761     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1762    elif(httpmethod=="POST"):
1763     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1764     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1765    else:
1766     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1767     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1768   except httpx.ConnectTimeout:
1769    log.info("Error With URL "+httpurl);
1770    return False;
1771   except socket.timeout:
1772    log.info("Error With URL "+httpurl);
1773    return False;
1774   downloadsize = int(geturls_text.headers.get('Content-Length'));
1775   if(downloadsize is not None):
1776    downloadsize = int(downloadsize);
1777   if downloadsize is None: downloadsize = 0;
1778   fulldatasize = 0;
1779   prevdownsize = 0;
1780   log.info("Downloading URL "+httpurl);
1781   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1782    tmpfilename = f.name;
1783    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1784    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1785     datasize = len(databytes);
1786     fulldatasize = datasize + fulldatasize;
1787     percentage = "";
1788     if(downloadsize>0):
1789      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1790     downloaddiff = fulldatasize - prevdownsize;
1791     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1792     prevdownsize = fulldatasize;
1793     f.write(databytes);
1794    f.close();
1795   geturls_text.close();
1796   exec_time_end = time.time();
1797   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1798   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1799   return returnval;
1800
1801 if(not havehttpx):
1802  def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1803   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1804   return returnval;
1805
1806 if(havehttpx):
1807  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1808   global geturls_download_sleep;
1809   if(sleep<0):
1810    sleep = geturls_download_sleep;
1811   if(not outfile=="-"):
1812    outpath = outpath.rstrip(os.path.sep);
1813    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1814    if(not os.path.exists(outpath)):
1815     os.makedirs(outpath);
1816    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1817     return False;
1818    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1819     return False;
1820    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1821    if(not pretmpfilename):
1822     return False;
1823    tmpfilename = pretmpfilename['Filename'];
1824    downloadsize = os.path.getsize(tmpfilename);
1825    fulldatasize = 0;
1826    log.info("Moving file "+tmpfilename+" to "+filepath);
1827    exec_time_start = time.time();
1828    shutil.move(tmpfilename, filepath);
1829    exec_time_end = time.time();
1830    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1831    if(os.path.exists(tmpfilename)):
1832     os.remove(tmpfilename);
1833    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1834   if(outfile=="-" and sys.version[0]=="2"):
1835    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1836    if(not pretmpfilename):
1837     return False;
1838    tmpfilename = pretmpfilename['Filename'];
1839    downloadsize = os.path.getsize(tmpfilename);
1840    fulldatasize = 0;
1841    prevdownsize = 0;
1842    exec_time_start = time.time();
1843    with open(tmpfilename, 'rb') as ft:
1844     f = StringIO();
1845     while True:
1846      databytes = ft.read(buffersize[1]);
1847      if not databytes: break;
1848      datasize = len(databytes);
1849      fulldatasize = datasize + fulldatasize;
1850      percentage = "";
1851      if(downloadsize>0):
1852       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1853      downloaddiff = fulldatasize - prevdownsize;
1854      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1855      prevdownsize = fulldatasize;
1856      f.write(databytes);
1857     f.seek(0);
1858     fdata = f.getvalue();
1859     f.close();
1860     ft.close();
1861     os.remove(tmpfilename);
1862     exec_time_end = time.time();
1863     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1864    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1865   if(outfile=="-" and sys.version[0]>="3"):
1866    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1867    tmpfilename = pretmpfilename['Filename'];
1868    downloadsize = os.path.getsize(tmpfilename);
1869    fulldatasize = 0;
1870    prevdownsize = 0;
1871    exec_time_start = time.time();
1872    with open(tmpfilename, 'rb') as ft:
1873     f = BytesIO();
1874     while True:
1875      databytes = ft.read(buffersize[1]);
1876      if not databytes: break;
1877      datasize = len(databytes);
1878      fulldatasize = datasize + fulldatasize;
1879      percentage = "";
1880      if(downloadsize>0):
1881       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1882      downloaddiff = fulldatasize - prevdownsize;
1883      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1884      prevdownsize = fulldatasize;
1885      f.write(databytes);
1886     f.seek(0);
1887     fdata = f.getvalue();
1888     f.close();
1889     ft.close();
1890     os.remove(tmpfilename);
1891     exec_time_end = time.time();
1892     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1893    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1894   return returnval;
1895
1896 if(not havehttpx):
1897  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1898   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1899   return returnval;
1900
1901 if(havehttpx):
1902  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1903   global geturls_download_sleep, havebrotli;
1904   if(sleep<0):
1905    sleep = geturls_download_sleep;
1906   urlparts = urlparse.urlparse(httpurl);
1907   if(isinstance(httpheaders, list)):
1908    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1909   if(urlparts.username is not None or urlparts.password is not None):
1910    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1911    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1912   time.sleep(sleep);
1913   if(postdata is not None and not isinstance(postdata, dict)):
1914    postdata = urlencode(postdata);
1915   try:
1916    if(httpmethod=="GET"):
1917     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1918     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1919    elif(httpmethod=="POST"):
1920     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1921     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1922    else:
1923     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1924     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1925   except httpx.ConnectTimeout:
1926    log.info("Error With URL "+httpurl);
1927    return False;
1928   except socket.timeout:
1929    log.info("Error With URL "+httpurl);
1930    return False;
1931   log.info("Downloading URL "+httpurl);
1932   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1933    if(sys.version[0]=="2"):
1934     strbuf = StringIO(geturls_text.content);
1935    if(sys.version[0]>="3"):
1936     strbuf = BytesIO(geturls_text.content);
1937    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1938    returnval_content = gzstrbuf.content[:];
1939   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1940    returnval_content = geturls_text.content[:];
1941   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1942    returnval_content = brotli.decompress(returnval_content);
1943   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1944   geturls_text.close();
1945   return returnval;
1946
1947 if(not havehttpx):
1948  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1949   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1950   return returnval;
1951
1952 if(havehttpx):
1953  def download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1954   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1955   exec_time_start = time.time();
1956   myhash = hashlib.new("sha1");
1957   if(sys.version[0]=="2"):
1958    myhash.update(httpurl);
1959    myhash.update(str(buffersize));
1960    myhash.update(str(exec_time_start));
1961   if(sys.version[0]>="3"):
1962    myhash.update(httpurl.encode('utf-8'));
1963    myhash.update(str(buffersize).encode('utf-8'));
1964    myhash.update(str(exec_time_start).encode('utf-8'));
1965   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1966   if(sleep<0):
1967    sleep = geturls_download_sleep;
1968   urlparts = urlparse.urlparse(httpurl);
1969   if(isinstance(httpheaders, list)):
1970    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1971   if(urlparts.username is not None or urlparts.password is not None):
1972    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1973    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1974   time.sleep(sleep);
1975   if(postdata is not None and not isinstance(postdata, dict)):
1976    postdata = urlencode(postdata);
1977   try:
1978    if(httpmethod=="GET"):
1979     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1980     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1981    elif(httpmethod=="POST"):
1982     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1983     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1984    else:
1985     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1986     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1987   except httpx.ConnectTimeout:
1988    log.info("Error With URL "+httpurl);
1989    return False;
1990   except socket.timeout:
1991    log.info("Error With URL "+httpurl);
1992    return False;
1993   downloadsize = int(geturls_text.headers.get('Content-Length'));
1994   if(downloadsize is not None):
1995    downloadsize = int(downloadsize);
1996   if downloadsize is None: downloadsize = 0;
1997   fulldatasize = 0;
1998   prevdownsize = 0;
1999   log.info("Downloading URL "+httpurl);
2000   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2001    tmpfilename = f.name;
2002    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
2003    for databytes in geturls_text.iter_content(chunk_size=buffersize):
2004     datasize = len(databytes);
2005     fulldatasize = datasize + fulldatasize;
2006     percentage = "";
2007     if(downloadsize>0):
2008      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2009     downloaddiff = fulldatasize - prevdownsize;
2010     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2011     prevdownsize = fulldatasize;
2012     f.write(databytes);
2013    f.close();
2014   geturls_text.close();
2015   exec_time_end = time.time();
2016   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2017   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2018   return returnval;
2019
2020 if(not havehttpx):
2021  def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2022   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2023   return returnval;
2024
2025 if(havehttpx):
2026  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2027   global geturls_download_sleep;
2028   if(sleep<0):
2029    sleep = geturls_download_sleep;
2030   if(not outfile=="-"):
2031    outpath = outpath.rstrip(os.path.sep);
2032    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2033    if(not os.path.exists(outpath)):
2034     os.makedirs(outpath);
2035    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2036     return False;
2037    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2038     return False;
2039    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2040    if(not pretmpfilename):
2041     return False;
2042    tmpfilename = pretmpfilename['Filename'];
2043    downloadsize = os.path.getsize(tmpfilename);
2044    fulldatasize = 0;
2045    log.info("Moving file "+tmpfilename+" to "+filepath);
2046    exec_time_start = time.time();
2047    shutil.move(tmpfilename, filepath);
2048    exec_time_end = time.time();
2049    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2050    if(os.path.exists(tmpfilename)):
2051     os.remove(tmpfilename);
2052    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2053   if(outfile=="-" and sys.version[0]=="2"):
2054    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2055    if(not pretmpfilename):
2056     return False;
2057    tmpfilename = pretmpfilename['Filename'];
2058    downloadsize = os.path.getsize(tmpfilename);
2059    fulldatasize = 0;
2060    prevdownsize = 0;
2061    exec_time_start = time.time();
2062    with open(tmpfilename, 'rb') as ft:
2063     f = StringIO();
2064     while True:
2065      databytes = ft.read(buffersize[1]);
2066      if not databytes: break;
2067      datasize = len(databytes);
2068      fulldatasize = datasize + fulldatasize;
2069      percentage = "";
2070      if(downloadsize>0):
2071       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2072      downloaddiff = fulldatasize - prevdownsize;
2073      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2074      prevdownsize = fulldatasize;
2075      f.write(databytes);
2076     f.seek(0);
2077     fdata = f.getvalue();
2078     f.close();
2079     ft.close();
2080     os.remove(tmpfilename);
2081     exec_time_end = time.time();
2082     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2083    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2084   if(outfile=="-" and sys.version[0]>="3"):
2085    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2086    tmpfilename = pretmpfilename['Filename'];
2087    downloadsize = os.path.getsize(tmpfilename);
2088    fulldatasize = 0;
2089    prevdownsize = 0;
2090    exec_time_start = time.time();
2091    with open(tmpfilename, 'rb') as ft:
2092     f = BytesIO();
2093     while True:
2094      databytes = ft.read(buffersize[1]);
2095      if not databytes: break;
2096      datasize = len(databytes);
2097      fulldatasize = datasize + fulldatasize;
2098      percentage = "";
2099      if(downloadsize>0):
2100       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2101      downloaddiff = fulldatasize - prevdownsize;
2102      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2103      prevdownsize = fulldatasize;
2104      f.write(databytes);
2105     f.seek(0);
2106     fdata = f.getvalue();
2107     f.close();
2108     ft.close();
2109     os.remove(tmpfilename);
2110     exec_time_end = time.time();
2111     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2112    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2113   return returnval;
2114
2115 if(not havehttpx):
2116  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2117   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2118   return returnval;
2119
2120 if(haveurllib3):
2121  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2122   global geturls_download_sleep, havebrotli;
2123   if(sleep<0):
2124    sleep = geturls_download_sleep;
2125   urlparts = urlparse.urlparse(httpurl);
2126   if(isinstance(httpheaders, list)):
2127    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2128   if(urlparts.username is not None or urlparts.password is not None):
2129    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2130    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2131   time.sleep(sleep);
2132   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2133   if(postdata is not None and not isinstance(postdata, dict)):
2134    postdata = urlencode(postdata);
2135   try:
2136    if(httpmethod=="GET"):
2137     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2138    elif(httpmethod=="POST"):
2139     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2140    else:
2141     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2142   except urllib3.exceptions.ConnectTimeoutError:
2143    log.info("Error With URL "+httpurl);
2144    return False;
2145   except urllib3.exceptions.MaxRetryError:
2146    log.info("Error With URL "+httpurl);
2147    return False;
2148   except socket.timeout:
2149    log.info("Error With URL "+httpurl);
2150    return False;
2151   log.info("Downloading URL "+httpurl);
2152   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2153    if(sys.version[0]=="2"):
2154     strbuf = StringIO(geturls_text.read());
2155    if(sys.version[0]>="3"):
2156     strbuf = BytesIO(geturls_text.read());
2157    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2158    returnval_content = gzstrbuf.read()[:];
2159   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2160    returnval_content = geturls_text.read()[:];
2161   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2162    returnval_content = brotli.decompress(returnval_content);
2163   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2164   geturls_text.close();
2165   return returnval;
2166
2167 if(not haveurllib3):
2168  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2169   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2170   return returnval;
2171
2172 if(haveurllib3):
2173  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2174   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2175   exec_time_start = time.time();
2176   myhash = hashlib.new("sha1");
2177   if(sys.version[0]=="2"):
2178    myhash.update(httpurl);
2179    myhash.update(str(buffersize));
2180    myhash.update(str(exec_time_start));
2181   if(sys.version[0]>="3"):
2182    myhash.update(httpurl.encode('utf-8'));
2183    myhash.update(str(buffersize).encode('utf-8'));
2184    myhash.update(str(exec_time_start).encode('utf-8'));
2185   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2186   if(sleep<0):
2187    sleep = geturls_download_sleep;
2188   urlparts = urlparse.urlparse(httpurl);
2189   if(isinstance(httpheaders, list)):
2190    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2191   if(urlparts.username is not None or urlparts.password is not None):
2192    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2193    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2194   time.sleep(sleep);
2195   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2196   if(postdata is not None and not isinstance(postdata, dict)):
2197    postdata = urlencode(postdata);
2198   try:
2199    if(httpmethod=="GET"):
2200     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2201    elif(httpmethod=="POST"):
2202     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2203    else:
2204     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2205   except urllib3.exceptions.ConnectTimeoutError:
2206    log.info("Error With URL "+httpurl);
2207    return False;
2208   except urllib3.exceptions.MaxRetryError:
2209    log.info("Error With URL "+httpurl);
2210    return False;
2211   except socket.timeout:
2212    log.info("Error With URL "+httpurl);
2213    return False;
2214   downloadsize = int(geturls_text.headers.get('Content-Length'));
2215   if(downloadsize is not None):
2216    downloadsize = int(downloadsize);
2217   if downloadsize is None: downloadsize = 0;
2218   fulldatasize = 0;
2219   prevdownsize = 0;
2220   log.info("Downloading URL "+httpurl);
2221   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2222    tmpfilename = f.name;
2223    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2224    while True:
2225     databytes = geturls_text.read(buffersize);
2226     if not databytes: break;
2227     datasize = len(databytes);
2228     fulldatasize = datasize + fulldatasize;
2229     percentage = "";
2230     if(downloadsize>0):
2231      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2232     downloaddiff = fulldatasize - prevdownsize;
2233     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2234     prevdownsize = fulldatasize;
2235     f.write(databytes);
2236    f.close();
2237   geturls_text.close();
2238   exec_time_end = time.time();
2239   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2240   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2241   return returnval;
2242
2243 if(not haveurllib3):
2244  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2245   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2246   return returnval;
2247
2248 if(haveurllib3):
2249  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2250   global geturls_download_sleep;
2251   if(sleep<0):
2252    sleep = geturls_download_sleep;
2253   if(not outfile=="-"):
2254    outpath = outpath.rstrip(os.path.sep);
2255    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2256    if(not os.path.exists(outpath)):
2257     os.makedirs(outpath);
2258    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2259     return False;
2260    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2261     return False;
2262    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2263    if(not pretmpfilename):
2264     return False;
2265    tmpfilename = pretmpfilename['Filename'];
2266    downloadsize = os.path.getsize(tmpfilename);
2267    fulldatasize = 0;
2268    log.info("Moving file "+tmpfilename+" to "+filepath);
2269    exec_time_start = time.time();
2270    shutil.move(tmpfilename, filepath);
2271    exec_time_end = time.time();
2272    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2273    if(os.path.exists(tmpfilename)):
2274     os.remove(tmpfilename);
2275    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2276   if(outfile=="-" and sys.version[0]=="2"):
2277    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2278    if(not pretmpfilename):
2279     return False;
2280    tmpfilename = pretmpfilename['Filename'];
2281    downloadsize = os.path.getsize(tmpfilename);
2282    fulldatasize = 0;
2283    prevdownsize = 0;
2284    exec_time_start = time.time();
2285    with open(tmpfilename, 'rb') as ft:
2286     f = StringIO();
2287     while True:
2288      databytes = ft.read(buffersize[1]);
2289      if not databytes: break;
2290      datasize = len(databytes);
2291      fulldatasize = datasize + fulldatasize;
2292      percentage = "";
2293      if(downloadsize>0):
2294       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2295      downloaddiff = fulldatasize - prevdownsize;
2296      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2297      prevdownsize = fulldatasize;
2298      f.write(databytes);
2299     f.seek(0);
2300     fdata = f.getvalue();
2301     f.close();
2302     ft.close();
2303     os.remove(tmpfilename);
2304     exec_time_end = time.time();
2305     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2306    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2307   if(outfile=="-" and sys.version[0]>="3"):
2308    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2309    tmpfilename = pretmpfilename['Filename'];
2310    downloadsize = os.path.getsize(tmpfilename);
2311    fulldatasize = 0;
2312    prevdownsize = 0;
2313    exec_time_start = time.time();
2314    with open(tmpfilename, 'rb') as ft:
2315     f = BytesIO();
2316     while True:
2317      databytes = ft.read(buffersize[1]);
2318      if not databytes: break;
2319      datasize = len(databytes);
2320      fulldatasize = datasize + fulldatasize;
2321      percentage = "";
2322      if(downloadsize>0):
2323       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2324      downloaddiff = fulldatasize - prevdownsize;
2325      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2326      prevdownsize = fulldatasize;
2327      f.write(databytes);
2328     f.seek(0);
2329     fdata = f.getvalue();
2330     f.close();
2331     ft.close();
2332     os.remove(tmpfilename);
2333     exec_time_end = time.time();
2334     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2335    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2336   return returnval;
2337
2338 if(not haveurllib3):
2339  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2340   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2341   return returnval;
2342
2343 if(haveurllib3):
2344  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2345   global geturls_download_sleep, havebrotli;
2346   if(sleep<0):
2347    sleep = geturls_download_sleep;
2348   urlparts = urlparse.urlparse(httpurl);
2349   if(isinstance(httpheaders, list)):
2350    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2351   if(urlparts.username is not None or urlparts.password is not None):
2352    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2353    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2354   time.sleep(sleep);
2355   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2356   if(postdata is not None and not isinstance(postdata, dict)):
2357    postdata = urlencode(postdata);
2358   try:
2359    if(httpmethod=="GET"):
2360     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2361    elif(httpmethod=="POST"):
2362     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2363    else:
2364     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2365   except urllib3.exceptions.ConnectTimeoutError:
2366    log.info("Error With URL "+httpurl);
2367    return False;
2368   except urllib3.exceptions.MaxRetryError:
2369    log.info("Error With URL "+httpurl);
2370    return False;
2371   except socket.timeout:
2372    log.info("Error With URL "+httpurl);
2373    return False;
2374   log.info("Downloading URL "+httpurl);
2375   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2376    if(sys.version[0]=="2"):
2377     strbuf = StringIO(geturls_text.read());
2378    if(sys.version[0]>="3"):
2379     strbuf = BytesIO(geturls_text.read());
2380    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2381    returnval_content = gzstrbuf.read()[:];
2382   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2383    returnval_content = geturls_text.read()[:];
2384   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2385    returnval_content = brotli.decompress(returnval_content);
2386   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2387   geturls_text.close();
2388   return returnval;
2389
2390 if(not haveurllib3):
2391  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2392   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2393   return returnval;
2394
2395 if(haveurllib3):
2396  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2397   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2398   exec_time_start = time.time();
2399   myhash = hashlib.new("sha1");
2400   if(sys.version[0]=="2"):
2401    myhash.update(httpurl);
2402    myhash.update(str(buffersize));
2403    myhash.update(str(exec_time_start));
2404   if(sys.version[0]>="3"):
2405    myhash.update(httpurl.encode('utf-8'));
2406    myhash.update(str(buffersize).encode('utf-8'));
2407    myhash.update(str(exec_time_start).encode('utf-8'));
2408   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2409   if(sleep<0):
2410    sleep = geturls_download_sleep;
2411   urlparts = urlparse.urlparse(httpurl);
2412   if(isinstance(httpheaders, list)):
2413    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2414   if(urlparts.username is not None or urlparts.password is not None):
2415    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2416    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2417   time.sleep(sleep);
2418   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2419   if(postdata is not None and not isinstance(postdata, dict)):
2420    postdata = urlencode(postdata);
2421   try:
2422    if(httpmethod=="GET"):
2423     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2424    elif(httpmethod=="POST"):
2425     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2426    else:
2427     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2428   except urllib3.exceptions.ConnectTimeoutError:
2429    log.info("Error With URL "+httpurl);
2430    return False;
2431   except urllib3.exceptions.MaxRetryError:
2432    log.info("Error With URL "+httpurl);
2433    return False;
2434   except socket.timeout:
2435    log.info("Error With URL "+httpurl);
2436    return False;
2437   downloadsize = int(geturls_text.headers.get('Content-Length'));
2438   if(downloadsize is not None):
2439    downloadsize = int(downloadsize);
2440   if downloadsize is None: downloadsize = 0;
2441   fulldatasize = 0;
2442   prevdownsize = 0;
2443   log.info("Downloading URL "+httpurl);
2444   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2445    tmpfilename = f.name;
2446    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2447    while True:
2448     databytes = geturls_text.read(buffersize);
2449     if not databytes: break;
2450     datasize = len(databytes);
2451     fulldatasize = datasize + fulldatasize;
2452     percentage = "";
2453     if(downloadsize>0):
2454      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2455     downloaddiff = fulldatasize - prevdownsize;
2456     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2457     prevdownsize = fulldatasize;
2458     f.write(databytes);
2459    f.close();
2460   geturls_text.close();
2461   exec_time_end = time.time();
2462   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2463   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2464   return returnval;
2465
2466 if(not haveurllib3):
2467  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2468   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2469   return returnval;
2470
2471 if(haveurllib3):
2472  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2473   global geturls_download_sleep;
2474   if(sleep<0):
2475    sleep = geturls_download_sleep;
2476   if(not outfile=="-"):
2477    outpath = outpath.rstrip(os.path.sep);
2478    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2479    if(not os.path.exists(outpath)):
2480     os.makedirs(outpath);
2481    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2482     return False;
2483    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2484     return False;
2485    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2486    if(not pretmpfilename):
2487     return False;
2488    tmpfilename = pretmpfilename['Filename'];
2489    downloadsize = os.path.getsize(tmpfilename);
2490    fulldatasize = 0;
2491    log.info("Moving file "+tmpfilename+" to "+filepath);
2492    exec_time_start = time.time();
2493    shutil.move(tmpfilename, filepath);
2494    exec_time_end = time.time();
2495    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2496    if(os.path.exists(tmpfilename)):
2497     os.remove(tmpfilename);
2498    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2499   if(outfile=="-" and sys.version[0]=="2"):
2500    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2501    if(not pretmpfilename):
2502     return False;
2503    tmpfilename = pretmpfilename['Filename'];
2504    downloadsize = os.path.getsize(tmpfilename);
2505    fulldatasize = 0;
2506    prevdownsize = 0;
2507    exec_time_start = time.time();
2508    with open(tmpfilename, 'rb') as ft:
2509     f = StringIO();
2510     while True:
2511      databytes = ft.read(buffersize[1]);
2512      if not databytes: break;
2513      datasize = len(databytes);
2514      fulldatasize = datasize + fulldatasize;
2515      percentage = "";
2516      if(downloadsize>0):
2517       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2518      downloaddiff = fulldatasize - prevdownsize;
2519      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2520      prevdownsize = fulldatasize;
2521      f.write(databytes);
2522     f.seek(0);
2523     fdata = f.getvalue();
2524     f.close();
2525     ft.close();
2526     os.remove(tmpfilename);
2527     exec_time_end = time.time();
2528     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2529    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2530   if(outfile=="-" and sys.version[0]>="3"):
2531    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2532    tmpfilename = pretmpfilename['Filename'];
2533    downloadsize = os.path.getsize(tmpfilename);
2534    fulldatasize = 0;
2535    prevdownsize = 0;
2536    exec_time_start = time.time();
2537    with open(tmpfilename, 'rb') as ft:
2538     f = BytesIO();
2539     while True:
2540      databytes = ft.read(buffersize[1]);
2541      if not databytes: break;
2542      datasize = len(databytes);
2543      fulldatasize = datasize + fulldatasize;
2544      percentage = "";
2545      if(downloadsize>0):
2546       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2547      downloaddiff = fulldatasize - prevdownsize;
2548      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2549      prevdownsize = fulldatasize;
2550      f.write(databytes);
2551     f.seek(0);
2552     fdata = f.getvalue();
2553     f.close();
2554     ft.close();
2555     os.remove(tmpfilename);
2556     exec_time_end = time.time();
2557     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2558    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2559   return returnval;
2560
2561 if(not haveurllib3):
2562  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2563   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2564   return returnval;
2565
2566 if(havemechanize):
2567  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2568   global geturls_download_sleep, havebrotli;
2569   if(sleep<0):
2570    sleep = geturls_download_sleep;
2571   urlparts = urlparse.urlparse(httpurl);
2572   if(isinstance(httpheaders, list)):
2573    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2574   if(urlparts.username is not None or urlparts.password is not None):
2575    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2576    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2577   geturls_opener = mechanize.Browser();
2578   if(isinstance(httpheaders, dict)):
2579    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2580   time.sleep(sleep);
2581   geturls_opener.addheaders = httpheaders;
2582   geturls_opener.set_cookiejar(httpcookie);
2583   geturls_opener.set_handle_robots(False);
2584   if(postdata is not None and not isinstance(postdata, dict)):
2585    postdata = urlencode(postdata);
2586   try:
2587    if(httpmethod=="GET"):
2588     geturls_text = geturls_opener.open(httpurl);
2589    elif(httpmethod=="POST"):
2590     geturls_text = geturls_opener.open(httpurl, data=postdata);
2591    else:
2592     geturls_text = geturls_opener.open(httpurl);
2593   except mechanize.HTTPError as geturls_text_error:
2594    geturls_text = geturls_text_error;
2595    log.info("Error With URL "+httpurl);
2596   except URLError:
2597    log.info("Error With URL "+httpurl);
2598    return False;
2599   except socket.timeout:
2600    log.info("Error With URL "+httpurl);
2601    return False;
2602   log.info("Downloading URL "+httpurl);
2603   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2604    if(sys.version[0]=="2"):
2605     strbuf = StringIO(geturls_text.read());
2606    if(sys.version[0]>="3"):
2607     strbuf = BytesIO(geturls_text.read());
2608    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2609    returnval_content = gzstrbuf.read()[:];
2610   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2611    returnval_content = geturls_text.read()[:];
2612   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2613    returnval_content = brotli.decompress(returnval_content);
2614   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2615   geturls_text.close();
2616   return returnval;
2617
2618 if(not havemechanize):
2619  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2620   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2621   return returnval;
2622
2623 if(havemechanize):
2624  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2625   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2626   exec_time_start = time.time();
2627   myhash = hashlib.new("sha1");
2628   if(sys.version[0]=="2"):
2629    myhash.update(httpurl);
2630    myhash.update(str(buffersize));
2631    myhash.update(str(exec_time_start));
2632   if(sys.version[0]>="3"):
2633    myhash.update(httpurl.encode('utf-8'));
2634    myhash.update(str(buffersize).encode('utf-8'));
2635    myhash.update(str(exec_time_start).encode('utf-8'));
2636   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2637   if(sleep<0):
2638    sleep = geturls_download_sleep;
2639   urlparts = urlparse.urlparse(httpurl);
2640   if(isinstance(httpheaders, list)):
2641    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2642   if(urlparts.username is not None or urlparts.password is not None):
2643    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2644    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2645   geturls_opener = mechanize.Browser();
2646   if(isinstance(httpheaders, dict)):
2647    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2648   time.sleep(sleep);
2649   geturls_opener.addheaders = httpheaders;
2650   geturls_opener.set_cookiejar(httpcookie);
2651   geturls_opener.set_handle_robots(False);
2652   if(postdata is not None and not isinstance(postdata, dict)):
2653    postdata = urlencode(postdata);
2654   try:
2655    if(httpmethod=="GET"):
2656     geturls_text = geturls_opener.open(httpurl);
2657    elif(httpmethod=="POST"):
2658     geturls_text = geturls_opener.open(httpurl, data=postdata);
2659    else:
2660     geturls_text = geturls_opener.open(httpurl);
2661   except mechanize.HTTPError as geturls_text_error:
2662    geturls_text = geturls_text_error;
2663    log.info("Error With URL "+httpurl);
2664   except URLError:
2665    log.info("Error With URL "+httpurl);
2666    return False;
2667   except socket.timeout:
2668    log.info("Error With URL "+httpurl);
2669    return False;
2670   downloadsize = int(geturls_text.info().get('Content-Length'));
2671   if(downloadsize is not None):
2672    downloadsize = int(downloadsize);
2673   if downloadsize is None: downloadsize = 0;
2674   fulldatasize = 0;
2675   prevdownsize = 0;
2676   log.info("Downloading URL "+httpurl);
2677   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2678    tmpfilename = f.name;
2679    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2680    while True:
2681     databytes = geturls_text.read(buffersize);
2682     if not databytes: break;
2683     datasize = len(databytes);
2684     fulldatasize = datasize + fulldatasize;
2685     percentage = "";
2686     if(downloadsize>0):
2687      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2688     downloaddiff = fulldatasize - prevdownsize;
2689     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2690     prevdownsize = fulldatasize;
2691     f.write(databytes);
2692    f.close();
2693   geturls_text.close();
2694   exec_time_end = time.time();
2695   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2696   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2697   return returnval;
2698
2699 if(not havemechanize):
2700  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2701   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2702   return returnval;
2703
2704 if(havemechanize):
2705  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2706   global geturls_download_sleep;
2707   if(sleep<0):
2708    sleep = geturls_download_sleep;
2709   if(not outfile=="-"):
2710    outpath = outpath.rstrip(os.path.sep);
2711    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2712    if(not os.path.exists(outpath)):
2713     os.makedirs(outpath);
2714    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2715     return False;
2716    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2717     return False;
2718    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2719    if(not pretmpfilename):
2720     return False;
2721    tmpfilename = pretmpfilename['Filename'];
2722    downloadsize = os.path.getsize(tmpfilename);
2723    fulldatasize = 0;
2724    log.info("Moving file "+tmpfilename+" to "+filepath);
2725    exec_time_start = time.time();
2726    shutil.move(tmpfilename, filepath);
2727    exec_time_end = time.time();
2728    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2729    if(os.path.exists(tmpfilename)):
2730     os.remove(tmpfilename);
2731    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2732   if(outfile=="-" and sys.version[0]=="2"):
2733    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2734    if(not pretmpfilename):
2735     return False;
2736    tmpfilename = pretmpfilename['Filename'];
2737    downloadsize = os.path.getsize(tmpfilename);
2738    fulldatasize = 0;
2739    prevdownsize = 0;
2740    exec_time_start = time.time();
2741    with open(tmpfilename, 'rb') as ft:
2742     f = StringIO();
2743     while True:
2744      databytes = ft.read(buffersize[1]);
2745      if not databytes: break;
2746      datasize = len(databytes);
2747      fulldatasize = datasize + fulldatasize;
2748      percentage = "";
2749      if(downloadsize>0):
2750       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2751      downloaddiff = fulldatasize - prevdownsize;
2752      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2753      prevdownsize = fulldatasize;
2754      f.write(databytes);
2755     f.seek(0);
2756     fdata = f.getvalue();
2757     f.close();
2758     ft.close();
2759     os.remove(tmpfilename);
2760     exec_time_end = time.time();
2761     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2762    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2763   if(outfile=="-" and sys.version[0]>="3"):
2764    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2765    tmpfilename = pretmpfilename['Filename'];
2766    downloadsize = os.path.getsize(tmpfilename);
2767    fulldatasize = 0;
2768    prevdownsize = 0;
2769    exec_time_start = time.time();
2770    with open(tmpfilename, 'rb') as ft:
2771     f = BytesIO();
2772     while True:
2773      databytes = ft.read(buffersize[1]);
2774      if not databytes: break;
2775      datasize = len(databytes);
2776      fulldatasize = datasize + fulldatasize;
2777      percentage = "";
2778      if(downloadsize>0):
2779       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2780      downloaddiff = fulldatasize - prevdownsize;
2781      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2782      prevdownsize = fulldatasize;
2783      f.write(databytes);
2784     f.seek(0);
2785     fdata = f.getvalue();
2786     f.close();
2787     ft.close();
2788     os.remove(tmpfilename);
2789     exec_time_end = time.time();
2790     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2791    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2792   return returnval;
2793
2794 if(not havemechanize):
2795  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2796   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2797   return returnval;
2798
2799 def download_file_from_ftp_file(url):
2800  urlparts = urlparse.urlparse(url);
2801  file_name = os.path.basename(urlparts.path);
2802  file_dir = os.path.dirname(urlparts.path);
2803  if(urlparts.username is not None):
2804   ftp_username = urlparts.username;
2805  else:
2806   ftp_username = "anonymous";
2807  if(urlparts.password is not None):
2808   ftp_password = urlparts.password;
2809  elif(urlparts.password is None and urlparts.username=="anonymous"):
2810   ftp_password = "anonymous";
2811  else:
2812   ftp_password = "";
2813  if(urlparts.scheme=="ftp"):
2814   ftp = FTP();
2815  elif(urlparts.scheme=="ftps"):
2816   ftp = FTP_TLS();
2817  else:
2818   return False;
2819  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2820   return False;
2821  ftp_port = urlparts.port;
2822  if(urlparts.port is None):
2823   ftp_port = 21;
2824  ftp.connect(urlparts.hostname, ftp_port);
2825  ftp.login(urlparts.username, urlparts.password);
2826  if(urlparts.scheme=="ftps"):
2827   ftp.prot_p();
2828  ftpfile = BytesIO();
2829  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2830  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2831  ftp.close();
2832  ftpfile.seek(0, 0);
2833  return ftpfile;
2834
2835 def download_file_from_ftp_string(url):
2836  ftpfile = download_file_from_ftp_file(url);
2837  return ftpfile.read();
2838
2839 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2840  global geturls_download_sleep, havebrotli;
2841  if(sleep<0):
2842   sleep = geturls_download_sleep;
2843  urlparts = urlparse.urlparse(httpurl);
2844  if(isinstance(httpheaders, list)):
2845    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2846  if(isinstance(httpheaders, dict)):
2847   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2848  time.sleep(sleep);
2849  geturls_text = download_file_from_ftp_file(httpurl);
2850  log.info("Downloading URL "+httpurl);
2851  returnval_content = geturls_text.read()[:];
2852  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': httpmethod, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
2853  geturls_text.close();
2854  return returnval;
2855
2856 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2857  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2858  exec_time_start = time.time();
2859  myhash = hashlib.new("sha1");
2860  if(sys.version[0]=="2"):
2861   myhash.update(httpurl);
2862   myhash.update(str(buffersize));
2863   myhash.update(str(exec_time_start));
2864  if(sys.version[0]>="3"):
2865   myhash.update(httpurl.encode('utf-8'));
2866   myhash.update(str(buffersize).encode('utf-8'));
2867   myhash.update(str(exec_time_start).encode('utf-8'));
2868  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2869  if(sleep<0):
2870   sleep = geturls_download_sleep;
2871  urlparts = urlparse.urlparse(httpurl);
2872  if(isinstance(httpheaders, list)):
2873    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2874  if(isinstance(httpheaders, dict)):
2875   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2876  time.sleep(sleep);
2877  geturls_text = download_file_from_ftp_file(httpurl);
2878  geturls_text.seek(0, 2);
2879  downloadsize = geturls_text.tell();
2880  geturls_text.seek(0, 0);
2881  if(downloadsize is not None):
2882   downloadsize = int(downloadsize);
2883  if downloadsize is None: downloadsize = 0;
2884  fulldatasize = 0;
2885  prevdownsize = 0;
2886  log.info("Downloading URL "+httpurl);
2887  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2888   tmpfilename = f.name;
2889   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
2890   while True:
2891    databytes = geturls_text.read(buffersize);
2892    if not databytes: break;
2893    datasize = len(databytes);
2894    fulldatasize = datasize + fulldatasize;
2895    percentage = "";
2896    if(downloadsize>0):
2897     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2898    downloaddiff = fulldatasize - prevdownsize;
2899    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2900    prevdownsize = fulldatasize;
2901    f.write(databytes);
2902   f.close();
2903  geturls_text.close();
2904  exec_time_end = time.time();
2905  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2906  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2907  return returnval;
2908
2909 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2910  global geturls_download_sleep;
2911  if(sleep<0):
2912   sleep = geturls_download_sleep;
2913  if(not outfile=="-"):
2914   outpath = outpath.rstrip(os.path.sep);
2915   filepath = os.path.realpath(outpath+os.path.sep+outfile);
2916   if(not os.path.exists(outpath)):
2917    os.makedirs(outpath);
2918   if(os.path.exists(outpath) and os.path.isfile(outpath)):
2919    return False;
2920   if(os.path.exists(filepath) and os.path.isdir(filepath)):
2921    return False;
2922   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2923   if(not pretmpfilename):
2924    return False;
2925   tmpfilename = pretmpfilename['Filename'];
2926   downloadsize = os.path.getsize(tmpfilename);
2927   fulldatasize = 0;
2928   log.info("Moving file "+tmpfilename+" to "+filepath);
2929   exec_time_start = time.time();
2930   shutil.move(tmpfilename, filepath);
2931   exec_time_end = time.time();
2932   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2933   if(os.path.exists(tmpfilename)):
2934    os.remove(tmpfilename);
2935   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2936  if(outfile=="-" and sys.version[0]=="2"):
2937   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2938   if(not pretmpfilename):
2939    return False;
2940   tmpfilename = pretmpfilename['Filename'];
2941   downloadsize = os.path.getsize(tmpfilename);
2942   fulldatasize = 0;
2943   prevdownsize = 0;
2944   exec_time_start = time.time();
2945   with open(tmpfilename, 'rb') as ft:
2946    f = StringIO();
2947    while True:
2948     databytes = ft.read(buffersize[1]);
2949     if not databytes: break;
2950     datasize = len(databytes);
2951     fulldatasize = datasize + fulldatasize;
2952     percentage = "";
2953     if(downloadsize>0):
2954      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2955     downloaddiff = fulldatasize - prevdownsize;
2956     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2957     prevdownsize = fulldatasize;
2958     f.write(databytes);
2959    f.seek(0);
2960    fdata = f.getvalue();
2961    f.close();
2962    ft.close();
2963    os.remove(tmpfilename);
2964    exec_time_end = time.time();
2965    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2966   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2967  if(outfile=="-" and sys.version[0]>="3"):
2968   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2969   tmpfilename = pretmpfilename['Filename'];
2970   downloadsize = os.path.getsize(tmpfilename);
2971   fulldatasize = 0;
2972   prevdownsize = 0;
2973   exec_time_start = time.time();
2974   with open(tmpfilename, 'rb') as ft:
2975    f = BytesIO();
2976    while True:
2977     databytes = ft.read(buffersize[1]);
2978     if not databytes: break;
2979     datasize = len(databytes);
2980     fulldatasize = datasize + fulldatasize;
2981     percentage = "";
2982     if(downloadsize>0):
2983      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2984     downloaddiff = fulldatasize - prevdownsize;
2985     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2986     prevdownsize = fulldatasize;
2987     f.write(databytes);
2988    f.seek(0);
2989    fdata = f.getvalue();
2990    f.close();
2991    ft.close();
2992    os.remove(tmpfilename);
2993    exec_time_end = time.time();
2994    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2995   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2996  return returnval;
2997
2998 def upload_file_to_ftp_file(ftpfile, url):
2999  urlparts = urlparse.urlparse(url);
3000  file_name = os.path.basename(urlparts.path);
3001  file_dir = os.path.dirname(urlparts.path);
3002  if(urlparts.username is not None):
3003   ftp_username = urlparts.username;
3004  else:
3005   ftp_username = "anonymous";
3006  if(urlparts.password is not None):
3007   ftp_password = urlparts.password;
3008  elif(urlparts.password is None and urlparts.username=="anonymous"):
3009   ftp_password = "anonymous";
3010  else:
3011   ftp_password = "";
3012  if(urlparts.scheme=="ftp"):
3013   ftp = FTP();
3014  elif(urlparts.scheme=="ftps"):
3015   ftp = FTP_TLS();
3016  else:
3017   return False;
3018  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3019   return False;
3020  ftp_port = urlparts.port;
3021  if(urlparts.port is None):
3022   ftp_port = 21;
3023  ftp.connect(urlparts.hostname, ftp_port);
3024  ftp.login(urlparts.username, urlparts.password);
3025  if(urlparts.scheme=="ftps"):
3026   ftp.prot_p();
3027  ftp.storbinary("STOR "+urlparts.path, ftpfile);
3028  ftp.close();
3029  ftpfile.seek(0, 0);
3030  return ftpfile;
3031
3032 def upload_file_to_ftp_string(ftpstring, url):
3033  ftpfileo = BytesIO(ftpstring);
3034  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
3035  ftpfileo.close();
3036  return ftpfile;
3037
3038 if(haveparamiko):
3039  def download_file_from_sftp_file(url):
3040   urlparts = urlparse.urlparse(url);
3041   file_name = os.path.basename(urlparts.path);
3042   file_dir = os.path.dirname(urlparts.path);
3043   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3044    return False;
3045   sftp_port = urlparts.port;
3046   if(urlparts.port is None):
3047    sftp_port = 22;
3048   else:
3049    sftp_port = urlparts.port;
3050   if(urlparts.username is not None):
3051    sftp_username = urlparts.username;
3052   else:
3053    sftp_username = "anonymous";
3054   if(urlparts.password is not None):
3055    sftp_password = urlparts.password;
3056   elif(urlparts.password is None and urlparts.username=="anonymous"):
3057    sftp_password = "anonymous";
3058   else:
3059    sftp_password = "";
3060   if(urlparts.scheme!="sftp"):
3061    return False;
3062   ssh = paramiko.SSHClient();
3063   ssh.load_system_host_keys();
3064   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
3065   try:
3066    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
3067   except paramiko.ssh_exception.SSHException:
3068    return False;
3069   sftp = ssh.open_sftp();
3070   sftpfile = BytesIO();
3071   sftp.getfo(urlparts.path, sftpfile);
3072   sftp.close();
3073   ssh.close();
3074   sftpfile.seek(0, 0);
3075   return sftpfile;
3076 else:
3077  def download_file_from_sftp_file(url):
3078   return False;
3079
3080 if(haveparamiko):
3081  def download_file_from_sftp_string(url):
3082   sftpfile = download_file_from_sftp_file(url);
3083   return sftpfile.read();
3084 else:
3085  def download_file_from_ftp_string(url):
3086   return False;
3087
3088 if(haveparamiko):
3089  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3090   global geturls_download_sleep, havebrotli;
3091   if(sleep<0):
3092    sleep = geturls_download_sleep;
3093   urlparts = urlparse.urlparse(httpurl);
3094   if(isinstance(httpheaders, list)):
3095     httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3096   if(isinstance(httpheaders, dict)):
3097    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
3098   time.sleep(sleep);
3099   geturls_text = download_file_from_sftp_file(httpurl);
3100   log.info("Downloading URL "+httpurl);
3101   returnval_content = geturls_text.read()[:];
3102   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': httpmethod, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
3103   geturls_text.close();
3104   return returnval;
3105
3106 if(not haveparamiko):
3107  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3108   return False;
3109
3110 if(haveparamiko):
3111  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3112   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
3113   exec_time_start = time.time();
3114   myhash = hashlib.new("sha1");
3115   if(sys.version[0]=="2"):
3116    myhash.update(httpurl);
3117    myhash.update(str(buffersize));
3118    myhash.update(str(exec_time_start));
3119   if(sys.version[0]>="3"):
3120    myhash.update(httpurl.encode('utf-8'));
3121    myhash.update(str(buffersize).encode('utf-8'));
3122    myhash.update(str(exec_time_start).encode('utf-8'));
3123   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3124   if(sleep<0):
3125    sleep = geturls_download_sleep;
3126   urlparts = urlparse.urlparse(httpurl);
3127   if(isinstance(httpheaders, list)):
3128     httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3129   if(isinstance(httpheaders, dict)):
3130    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
3131   time.sleep(sleep);
3132   geturls_text = download_file_from_sftp_file(httpurl);
3133   geturls_text.seek(0, 2);
3134   downloadsize = geturls_text.tell();
3135   geturls_text.seek(0, 0);
3136   if(downloadsize is not None):
3137    downloadsize = int(downloadsize);
3138   if downloadsize is None: downloadsize = 0;
3139   fulldatasize = 0;
3140   prevdownsize = 0;
3141   log.info("Downloading URL "+httpurl);
3142   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3143    tmpfilename = f.name;
3144    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
3145    while True:
3146     databytes = geturls_text.read(buffersize);
3147     if not databytes: break;
3148     datasize = len(databytes);
3149     fulldatasize = datasize + fulldatasize;
3150     percentage = "";
3151     if(downloadsize>0):
3152      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3153     downloaddiff = fulldatasize - prevdownsize;
3154     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3155     prevdownsize = fulldatasize;
3156     f.write(databytes);
3157    f.close();
3158   geturls_text.close();
3159   exec_time_end = time.time();
3160   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3161   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3162   return returnval;
3163
3164 if(not haveparamiko):
3165  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3166   return False;
3167
3168 if(haveparamiko):
3169  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3170   global geturls_download_sleep;
3171   if(sleep<0):
3172    sleep = geturls_download_sleep;
3173   if(not outfile=="-"):
3174    outpath = outpath.rstrip(os.path.sep);
3175    filepath = os.path.realpath(outpath+os.path.sep+outfile);
3176    if(not os.path.exists(outpath)):
3177     os.makedirs(outpath);
3178    if(os.path.exists(outpath) and os.path.isfile(outpath)):
3179     return False;
3180    if(os.path.exists(filepath) and os.path.isdir(filepath)):
3181     return False;
3182    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3183    if(not pretmpfilename):
3184     return False;
3185    tmpfilename = pretmpfilename['Filename'];
3186    downloadsize = os.path.getsize(tmpfilename);
3187    fulldatasize = 0;
3188    log.info("Moving file "+tmpfilename+" to "+filepath);
3189    exec_time_start = time.time();
3190    shutil.move(tmpfilename, filepath);
3191    exec_time_end = time.time();
3192    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3193    if(os.path.exists(tmpfilename)):
3194     os.remove(tmpfilename);
3195    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3196   if(outfile=="-" and sys.version[0]=="2"):
3197    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3198    if(not pretmpfilename):
3199     return False;
3200    tmpfilename = pretmpfilename['Filename'];
3201    downloadsize = os.path.getsize(tmpfilename);
3202    fulldatasize = 0;
3203    prevdownsize = 0;
3204    exec_time_start = time.time();
3205    with open(tmpfilename, 'rb') as ft:
3206     f = StringIO();
3207     while True:
3208      databytes = ft.read(buffersize[1]);
3209      if not databytes: break;
3210      datasize = len(databytes);
3211      fulldatasize = datasize + fulldatasize;
3212      percentage = "";
3213      if(downloadsize>0):
3214       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3215      downloaddiff = fulldatasize - prevdownsize;
3216      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3217      prevdownsize = fulldatasize;
3218      f.write(databytes);
3219     f.seek(0);
3220     fdata = f.getvalue();
3221     f.close();
3222     ft.close();
3223     os.remove(tmpfilename);
3224     exec_time_end = time.time();
3225     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3226    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3227   if(outfile=="-" and sys.version[0]>="3"):
3228    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3229    tmpfilename = pretmpfilename['Filename'];
3230    downloadsize = os.path.getsize(tmpfilename);
3231    fulldatasize = 0;
3232    prevdownsize = 0;
3233    exec_time_start = time.time();
3234    with open(tmpfilename, 'rb') as ft:
3235     f = BytesIO();
3236     while True:
3237      databytes = ft.read(buffersize[1]);
3238      if not databytes: break;
3239      datasize = len(databytes);
3240      fulldatasize = datasize + fulldatasize;
3241      percentage = "";
3242      if(downloadsize>0):
3243       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3244      downloaddiff = fulldatasize - prevdownsize;
3245      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3246      prevdownsize = fulldatasize;
3247      f.write(databytes);
3248     f.seek(0);
3249     fdata = f.getvalue();
3250     f.close();
3251     ft.close();
3252     os.remove(tmpfilename);
3253     exec_time_end = time.time();
3254     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3255    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3256   return returnval;
3257
3258 if(not haveparamiko):
3259  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3260   return False;
3261
3262 if(haveparamiko):
3263  def upload_file_to_sftp_file(sftpfile, url):
3264   urlparts = urlparse.urlparse(url);
3265   file_name = os.path.basename(urlparts.path);
3266   file_dir = os.path.dirname(urlparts.path);
3267   sftp_port = urlparts.port;
3268   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3269    return False;
3270   if(urlparts.port is None):
3271    sftp_port = 22;
3272   else:
3273    sftp_port = urlparts.port;
3274   if(urlparts.username is not None):
3275    sftp_username = urlparts.username;
3276   else:
3277    sftp_username = "anonymous";
3278   if(urlparts.password is not None):
3279    sftp_password = urlparts.password;
3280   elif(urlparts.password is None and urlparts.username=="anonymous"):
3281    sftp_password = "anonymous";
3282   else:
3283    sftp_password = "";
3284   if(urlparts.scheme!="sftp"):
3285    return False;
3286   ssh = paramiko.SSHClient();
3287   ssh.load_system_host_keys();
3288   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
3289   try:
3290    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
3291   except paramiko.ssh_exception.SSHException:
3292    return False;
3293   sftp = ssh.open_sftp();
3294   sftp.putfo(sftpfile, urlparts.path);
3295   sftp.close();
3296   ssh.close();
3297   sftpfile.seek(0, 0);
3298   return sftpfile;
3299 else:
3300  def upload_file_to_sftp_file(sftpfile, url):
3301   return False;
3302
3303 if(haveparamiko):
3304  def upload_file_to_sftp_string(sftpstring, url):
3305   sftpfileo = BytesIO(sftpstring);
3306   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
3307   sftpfileo.close();
3308   return sftpfile;
3309 else:
3310  def upload_file_to_sftp_string(url):
3311   return False;