pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 9/15/2023 Ver. 0.9.4 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 haverequests = False;
  24 try:
  25  import requests;
  26  haverequests = True;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  import mechanize;
  32  havemechanize = True;
  33 except ImportError:
  34  havemechanize = False;
  35 haveparamiko = False;
  36 try:
  37  import paramiko;
  38  haveparamiko = True;
  39 except ImportError:
  40  haveparamiko = False;
  41 haveurllib3 = False;
  42 try:
  43  import urllib3;
  44  haveurllib3 = True;
  45 except ImportError:
  46  haveurllib3 = False;
  47 havehttplib2 = False;
  48 try:
  49  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  50  havehttplib2 = True;
  51 except ImportError:
  52  havehttplib2 = False;
  53 havehttpx = False;
  54 try:
  55  import httpx;
  56  havehttpx = True;
  57 except ImportError:
  58  havehttpx = False;
  59 havebrotli = False;
  60 try:
  61  import brotli;
  62  havebrotli = True;
  63 except ImportError:
  64  havebrotli = False;
  65 if(sys.version[0]=="2"):
  66  try:
  67   from cStringIO import StringIO;
  68  except ImportError:
  69   from StringIO import StringIO;
  70  # From http://python-future.org/compatible_idioms.html
  71  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  72  from urllib import urlencode;
  73  from urllib import urlopen as urlopenalt;
  74  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
  75  import urlparse, cookielib;
  76  from httplib import HTTPConnection, HTTPSConnection;
  77 if(sys.version[0]>="3"):
  78  from io import StringIO, BytesIO;
  79  # From http://python-future.org/compatible_idioms.html
  80  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  81  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
  82  from urllib.error import HTTPError, URLError;
  83  import urllib.parse as urlparse;
  84  import http.cookiejar as cookielib;
  85  from http.client import HTTPConnection, HTTPSConnection;
  86
  87 __program_name__ = "PyWWW-Get";
  88 __program_alt_name__ = "PyWWWGet";
  89 __program_small_name__ = "wwwget";
  90 __project__ = __program_name__;
  91 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
  92 __version_info__ = (0, 9, 4, "RC 1", 1);
  93 __version_date_info__ = (2023, 9, 15, "RC 1", 1);
  94 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  95 __revision__ = __version_info__[3];
  96 __revision_id__ = "$Id$";
  97 if(__version_info__[4] is not None):
  98  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  99 if(__version_info__[4] is None):
 100  __version_date_plusrc__ = __version_date__;
 101 if(__version_info__[3] is not None):
 102  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 103 if(__version_info__[3] is None):
 104  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 105
 106 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 107 tmpfilesuffix = "-";
 108 pytempdir = tempfile.gettempdir();
 109
 110 compression_supported = "gzip, deflate";
 111 if(havebrotli):
 112  compression_supported = "gzip, deflate, br";
 113 else:
 114  compression_supported = "gzip, deflate";
 115
 116 geturls_cj = cookielib.CookieJar();
 117 windowsNT4_ua_string = "Windows NT 4.0";
 118 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 119 windows2k_ua_string = "Windows NT 5.0";
 120 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 121 windowsXP_ua_string = "Windows NT 5.1";
 122 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 123 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 124 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 125 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 126 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 127 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 128 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 129 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 130 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 131 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 132 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 133 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 134 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 135 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 136 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 137 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 138 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 139 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 140 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 141 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 142 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 143 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 144 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 145 if(platform.python_implementation()!=""):
 146  py_implementation = platform.python_implementation();
 147 if(platform.python_implementation()==""):
 148  py_implementation = "Python";
 149 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 150 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 151 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 152 geturls_ua = geturls_ua_firefox_windows7;
 153 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 154 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 155 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 156 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 157 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 158 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 159 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 160 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 161 geturls_headers_opera_windows7.update(windows7_ua_addon);
 162 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 163 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 164 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 165 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 166 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 167 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__)};
 168 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__)};
 169 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 170 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 171 geturls_headers = geturls_headers_firefox_windows7;
 172 geturls_download_sleep = 0;
 173
 174 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 175  if(outtype=="print" and dbgenable):
 176   print(dbgtxt);
 177   return True;
 178  elif(outtype=="log" and dbgenable):
 179   logging.info(dbgtxt);
 180   return True;
 181  elif(outtype=="warning" and dbgenable):
 182   logging.warning(dbgtxt);
 183   return True;
 184  elif(outtype=="error" and dbgenable):
 185   logging.error(dbgtxt);
 186   return True;
 187  elif(outtype=="critical" and dbgenable):
 188   logging.critical(dbgtxt);
 189   return True;
 190  elif(outtype=="exception" and dbgenable):
 191   logging.exception(dbgtxt);
 192   return True;
 193  elif(outtype=="logalt" and dbgenable):
 194   logging.log(dgblevel, dbgtxt);
 195   return True;
 196  elif(outtype=="debug" and dbgenable):
 197   logging.debug(dbgtxt);
 198   return True;
 199  elif(not dbgenable):
 200   return True;
 201  else:
 202   return False;
 203  return False;
 204
 205 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 206  verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 207  return dbgtxt;
 208
 209 def add_url_param(url, **params):
 210  n=3;
 211  parts = list(urlparse.urlsplit(url));
 212  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 213  d.update(params);
 214  parts[n]=urlencode(d);
 215  return urlparse.urlunsplit(parts);
 216
 217 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 218 def which_exec(execfile):
 219  for path in os.environ["PATH"].split(":"):
 220   if os.path.exists(path + "/" + execfile):
 221    return path + "/" + execfile;
 222
 223 def listize(varlist):
 224  il = 0;
 225  ix = len(varlist);
 226  ilx = 1;
 227  newlistreg = {};
 228  newlistrev = {};
 229  newlistfull = {};
 230  while(il < ix):
 231   newlistreg.update({ilx: varlist[il]});
 232   newlistrev.update({varlist[il]: ilx});
 233   ilx = ilx + 1;
 234   il = il + 1;
 235  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 236  return newlistfull;
 237
 238 def twolistize(varlist):
 239  il = 0;
 240  ix = len(varlist);
 241  ilx = 1;
 242  newlistnamereg = {};
 243  newlistnamerev = {};
 244  newlistdescreg = {};
 245  newlistdescrev = {};
 246  newlistfull = {};
 247  while(il < ix):
 248   newlistnamereg.update({ilx: varlist[il][0].strip()});
 249   newlistnamerev.update({varlist[il][0].strip(): ilx});
 250   newlistdescreg.update({ilx: varlist[il][1].strip()});
 251   newlistdescrev.update({varlist[il][1].strip(): ilx});
 252   ilx = ilx + 1;
 253   il = il + 1;
 254  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 255  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 256  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 257  return newlistfull;
 258
 259 def arglistize(proexec, *varlist):
 260  il = 0;
 261  ix = len(varlist);
 262  ilx = 1;
 263  newarglist = [proexec];
 264  while(il < ix):
 265   if varlist[il][0] is not None:
 266    newarglist.append(varlist[il][0]);
 267   if varlist[il][1] is not None:
 268    newarglist.append(varlist[il][1]);
 269   il = il + 1;
 270  return newarglist;
 271
 272 # hms_string by ArcGIS Python Recipes
 273 # https://arcpy.wordpress.com/2012/04/20/146/
 274 def hms_string(sec_elapsed):
 275  h = int(sec_elapsed / (60 * 60));
 276  m = int((sec_elapsed % (60 * 60)) / 60);
 277  s = sec_elapsed % 60.0;
 278  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 279
 280 # get_readable_size by Lipis
 281 # http://stackoverflow.com/posts/14998888/revisions
 282 def get_readable_size(bytes, precision=1, unit="IEC"):
 283  unit = unit.upper();
 284  if(unit!="IEC" and unit!="SI"):
 285   unit = "IEC";
 286  if(unit=="IEC"):
 287   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 288   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 289   unitsize = 1024.0;
 290  if(unit=="SI"):
 291   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 292   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 293   unitsize = 1000.0;
 294  return_val = {};
 295  orgbytes = bytes;
 296  for unit in units:
 297   if abs(bytes) < unitsize:
 298    strformat = "%3."+str(precision)+"f%s";
 299    pre_return_val = (strformat % (bytes, unit));
 300    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 301    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 302    alt_return_val = pre_return_val.split();
 303    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 304    return return_val;
 305   bytes /= unitsize;
 306  strformat = "%."+str(precision)+"f%s";
 307  pre_return_val = (strformat % (bytes, "YiB"));
 308  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 309  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 310  alt_return_val = pre_return_val.split();
 311  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 312  return return_val;
 313
 314 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 315  unit = unit.upper();
 316  usehashtypes = usehashtypes.lower();
 317  getfilesize = os.path.getsize(infile);
 318  return_val = get_readable_size(getfilesize, precision, unit);
 319  if(usehashes):
 320   hashtypelist = usehashtypes.split(",");
 321   openfile = open(infile, "rb");
 322   filecontents = openfile.read();
 323   openfile.close();
 324   listnumcount = 0;
 325   listnumend = len(hashtypelist);
 326   while(listnumcount < listnumend):
 327    hashtypelistlow = hashtypelist[listnumcount].strip();
 328    hashtypelistup = hashtypelistlow.upper();
 329    filehash = hashlib.new(hashtypelistup);
 330    filehash.update(filecontents);
 331    filegethash = filehash.hexdigest();
 332    return_val.update({hashtypelistup: filegethash});
 333    listnumcount += 1;
 334  return return_val;
 335
 336 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 337  unit = unit.upper();
 338  usehashtypes = usehashtypes.lower();
 339  getfilesize = len(instring);
 340  return_val = get_readable_size(getfilesize, precision, unit);
 341  if(usehashes):
 342   hashtypelist = usehashtypes.split(",");
 343   listnumcount = 0;
 344   listnumend = len(hashtypelist);
 345   while(listnumcount < listnumend):
 346    hashtypelistlow = hashtypelist[listnumcount].strip();
 347    hashtypelistup = hashtypelistlow.upper();
 348    filehash = hashlib.new(hashtypelistup);
 349    if(sys.version[0]=="2"):
 350     filehash.update(instring);
 351    if(sys.version[0]>="3"):
 352     filehash.update(instring.encode('utf-8'));
 353    filegethash = filehash.hexdigest();
 354    return_val.update({hashtypelistup: filegethash});
 355    listnumcount += 1;
 356  return return_val;
 357
 358 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 359  if isinstance(headers, dict):
 360   returnval = [];
 361   if(sys.version[0]=="2"):
 362    for headkey, headvalue in headers.iteritems():
 363     returnval.append((headkey, headvalue));
 364   if(sys.version[0]>="3"):
 365    for headkey, headvalue in headers.items():
 366     returnval.append((headkey, headvalue));
 367  elif isinstance(headers, list):
 368   returnval = headers;
 369  else:
 370   returnval = False;
 371  return returnval;
 372
 373 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 374  if isinstance(headers, dict):
 375   returnval = [];
 376   if(sys.version[0]=="2"):
 377    for headkey, headvalue in headers.iteritems():
 378     returnval.append(headkey+": "+headvalue);
 379   if(sys.version[0]>="3"):
 380    for headkey, headvalue in headers.items():
 381     returnval.append(headkey+": "+headvalue);
 382  elif isinstance(headers, list):
 383   returnval = headers;
 384  else:
 385   returnval = False;
 386  return returnval;
 387
 388 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 389  if isinstance(headers, list):
 390   returnval = {};
 391   mli = 0;
 392   mlil = len(headers);
 393   while(mli<mlil):
 394    returnval.update({headers[mli][0]: headers[mli][1]});
 395    mli = mli + 1;
 396  elif isinstance(headers, dict):
 397   returnval = headers;
 398  else:
 399   returnval = False;
 400  return returnval;
 401
 402 def get_httplib_support(checkvalue=None):
 403  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 404  returnval = [];
 405  returnval.append("httplib");
 406  if(havehttplib2):
 407   returnval.append("httplib2");
 408  returnval.append("urllib");
 409  if(haveurllib3):
 410   returnval.append("urllib3");
 411   returnval.append("request3");
 412  returnval.append("request");
 413  if(haverequests):
 414   returnval.append("requests");
 415  if(havehttpx):
 416   returnval.append("httpx");
 417  if(havemechanize):
 418   returnval.append("mechanize");
 419  if(not checkvalue is None):
 420   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 421    checkvalue = "urllib";
 422   if(checkvalue=="httplib1"):
 423    checkvalue = "httplib";
 424   if(checkvalue in returnval):
 425    returnval = True;
 426   else:
 427    returnval = False;
 428  return returnval;
 429
 430 def check_httplib_support(checkvalue="urllib"):
 431  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 432   checkvalue = "urllib";
 433  if(checkvalue=="httplib1"):
 434   checkvalue = "httplib";
 435  returnval = get_httplib_support(checkvalue);
 436  return returnval;
 437
 438 def get_httplib_support_list():
 439  returnval = get_httplib_support(None);
 440  return returnval;
 441
 442 def download_from_url(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 443  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 444  if(sleep<0):
 445   sleep = geturls_download_sleep;
 446  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 447   httplibuse = "urllib";
 448  if(httplibuse=="httplib1"):
 449   httplibuse = "httplib";
 450  if(not haverequests and httplibuse=="requests"):
 451   httplibuse = "urllib";
 452  if(not havehttpx and httplibuse=="httpx"):
 453   httplibuse = "urllib";
 454  if(not havemechanize and httplibuse=="mechanize"):
 455   httplibuse = "urllib";
 456  if(not havehttplib2 and httplibuse=="httplib2"):
 457   httplibuse = "httplib";
 458  if(httplibuse=="urllib"):
 459   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 460  elif(httplibuse=="request"):
 461   returnval = download_from_url_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 462  elif(httplibuse=="request3"):
 463   returnval = download_from_url_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 464  elif(httplibuse=="httplib"):
 465   returnval = download_from_url_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 466  elif(httplibuse=="httplib2"):
 467   returnval = download_from_url_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 468  elif(httplibuse=="urllib3"):
 469   returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 470  elif(httplibuse=="requests"):
 471   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 472  elif(httplibuse=="httpx"):
 473   returnval = download_from_url_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 474  elif(httplibuse=="mechanize"):
 475   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 476  else:
 477   returnval = False;
 478  return returnval;
 479
 480 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
 481  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 482  if(sleep<0):
 483   sleep = geturls_download_sleep;
 484  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 485   httplibuse = "urllib";
 486  if(httplibuse=="httplib1"):
 487   httplibuse = "httplib";
 488  if(not haverequests and httplibuse=="requests"):
 489   httplibuse = "urllib";
 490  if(not havehttpx and httplibuse=="httpx"):
 491   httplibuse = "urllib";
 492  if(not havemechanize and httplibuse=="mechanize"):
 493   httplibuse = "urllib";
 494  if(not havehttplib2 and httplibuse=="httplib2"):
 495   httplibuse = "httplib";
 496  if(httplibuse=="urllib"):
 497   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 498  elif(httplibuse=="request"):
 499   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 500  elif(httplibuse=="request3"):
 501   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 502  elif(httplibuse=="httplib"):
 503   returnval = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 504  elif(httplibuse=="httplib2"):
 505   returnval = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 506  elif(httplibuse=="urllib3"):
 507   returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 508  elif(httplibuse=="requests"):
 509   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 510  elif(httplibuse=="httpx"):
 511   returnval = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 512  elif(httplibuse=="mechanize"):
 513   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 514  else:
 515   returnval = False;
 516  return returnval;
 517
 518 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 519  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx;
 520  if(sleep<0):
 521   sleep = geturls_download_sleep;
 522  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 523   httplibuse = "urllib";
 524  if(httplibuse=="httplib1"):
 525   httplibuse = "httplib";
 526  if(not haverequests and httplibuse=="requests"):
 527   httplibuse = "urllib";
 528  if(not havehttpx and httplibuse=="httpx"):
 529   httplibuse = "urllib";
 530  if(not havemechanize and httplibuse=="mechanize"):
 531   httplibuse = "urllib";
 532  if(not havehttplib2 and httplibuse=="httplib2"):
 533   httplibuse = "httplib";
 534  if(httplibuse=="urllib"):
 535   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 536  elif(httplibuse=="request"):
 537   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 538  elif(httplibuse=="request3"):
 539   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 540  elif(httplibuse=="httplib"):
 541   returnval = download_from_url_to_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 542  elif(httplibuse=="httplib2"):
 543   returnval = download_from_url_to_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 544  elif(httplibuse=="urllib3"):
 545   returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 546  elif(httplibuse=="requests"):
 547   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 548  elif(httplibuse=="httpx"):
 549   returnval = download_from_url_to_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 550  elif(httplibuse=="mechanize"):
 551   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 552  else:
 553   returnval = False;
 554  return returnval;
 555
 556 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 557  global geturls_download_sleep, havebrotli;
 558  if(sleep<0):
 559   sleep = geturls_download_sleep;
 560  urlparts = urlparse.urlparse(httpurl);
 561  if(isinstance(httpheaders, list)):
 562    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 563  if(urlparts.username is not None or urlparts.password is not None):
 564   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 565   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 566  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 567  if(isinstance(httpheaders, dict)):
 568   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 569  geturls_opener.addheaders = httpheaders;
 570  time.sleep(sleep);
 571  if(postdata is not None and not isinstance(postdata, dict)):
 572   postdata = urlencode(postdata);
 573  try:
 574   if(httpmethod=="GET"):
 575    geturls_text = geturls_opener.open(httpurl);
 576   elif(httpmethod=="POST"):
 577    geturls_text = geturls_opener.open(httpurl, data=postdata);
 578   else:
 579    geturls_text = geturls_opener.open(httpurl);
 580  except HTTPError as geturls_text_error:
 581   geturls_text = geturls_text_error;
 582   log.info("Error With URL "+httpurl);
 583  except URLError:
 584   log.info("Error With URL "+httpurl);
 585   return False;
 586  except socket.timeout:
 587   log.info("Error With URL "+httpurl);
 588   return False;
 589  log.info("Downloading URL "+httpurl);
 590  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 591   if(sys.version[0]=="2"):
 592    strbuf = StringIO(geturls_text.read());
 593   if(sys.version[0]>="3"):
 594    strbuf = BytesIO(geturls_text.read());
 595   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 596   returnval_content = gzstrbuf.read()[:];
 597  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 598   returnval_content = geturls_text.read()[:];
 599  if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
 600   returnval_content = brotli.decompress(returnval_content);
 601  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 602  geturls_text.close();
 603  return returnval;
 604
 605 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 606  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 607  exec_time_start = time.time();
 608  myhash = hashlib.new("sha1");
 609  if(sys.version[0]=="2"):
 610   myhash.update(httpurl);
 611   myhash.update(str(buffersize));
 612   myhash.update(str(exec_time_start));
 613  if(sys.version[0]>="3"):
 614   myhash.update(httpurl.encode('utf-8'));
 615   myhash.update(str(buffersize).encode('utf-8'));
 616   myhash.update(str(exec_time_start).encode('utf-8'));
 617  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 618  if(sleep<0):
 619   sleep = geturls_download_sleep;
 620  urlparts = urlparse.urlparse(httpurl);
 621  if(isinstance(httpheaders, list)):
 622    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 623  if(urlparts.username is not None or urlparts.password is not None):
 624   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 625   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 626  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 627  if(isinstance(httpheaders, dict)):
 628   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 629  geturls_opener.addheaders = httpheaders;
 630  time.sleep(sleep);
 631  try:
 632   if(httpmethod=="GET"):
 633    geturls_text = geturls_opener.open(httpurl);
 634   elif(httpmethod=="POST"):
 635    geturls_text = geturls_opener.open(httpurl, data=postdata);
 636   else:
 637    geturls_text = geturls_opener.open(httpurl);
 638  except HTTPError as geturls_text_error:
 639   geturls_text = geturls_text_error;
 640   log.info("Error With URL "+httpurl);
 641  except URLError:
 642   log.info("Error With URL "+httpurl);
 643   return False;
 644  except socket.timeout:
 645   log.info("Error With URL "+httpurl);
 646   return False;
 647  except socket.timeout:
 648   log.info("Error With URL "+httpurl);
 649   return False;
 650  downloadsize = geturls_text.info().get('Content-Length');
 651  if(downloadsize is not None):
 652   downloadsize = int(downloadsize);
 653  if downloadsize is None: downloadsize = 0;
 654  fulldatasize = 0;
 655  prevdownsize = 0;
 656  log.info("Downloading URL "+httpurl);
 657  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 658   tmpfilename = f.name;
 659   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 660   while True:
 661    databytes = geturls_text.read(buffersize);
 662    if not databytes: break;
 663    datasize = len(databytes);
 664    fulldatasize = datasize + fulldatasize;
 665    percentage = "";
 666    if(downloadsize>0):
 667     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 668    downloaddiff = fulldatasize - prevdownsize;
 669    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 670    prevdownsize = fulldatasize;
 671    f.write(databytes);
 672   f.close();
 673  geturls_text.close();
 674  exec_time_end = time.time();
 675  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 676  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 677  return returnval;
 678
 679 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 680  global geturls_download_sleep;
 681  if(sleep<0):
 682   sleep = geturls_download_sleep;
 683  if(not outfile=="-"):
 684   outpath = outpath.rstrip(os.path.sep);
 685   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 686   if(not os.path.exists(outpath)):
 687    os.makedirs(outpath);
 688   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 689    return False;
 690   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 691    return False;
 692   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 693   if(not pretmpfilename):
 694    return False;
 695   tmpfilename = pretmpfilename['Filename'];
 696   downloadsize = os.path.getsize(tmpfilename);
 697   fulldatasize = 0;
 698   log.info("Moving file "+tmpfilename+" to "+filepath);
 699   exec_time_start = time.time();
 700   shutil.move(tmpfilename, filepath);
 701   exec_time_end = time.time();
 702   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 703   if(os.path.exists(tmpfilename)):
 704    os.remove(tmpfilename);
 705   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 706  if(outfile=="-" and sys.version[0]=="2"):
 707   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 708   if(not pretmpfilename):
 709    return False;
 710   tmpfilename = pretmpfilename['Filename'];
 711   downloadsize = os.path.getsize(tmpfilename);
 712   fulldatasize = 0;
 713   prevdownsize = 0;
 714   exec_time_start = time.time();
 715   with open(tmpfilename, 'rb') as ft:
 716    f = StringIO();
 717    while True:
 718     databytes = ft.read(buffersize[1]);
 719     if not databytes: break;
 720     datasize = len(databytes);
 721     fulldatasize = datasize + fulldatasize;
 722     percentage = "";
 723     if(downloadsize>0):
 724      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 725     downloaddiff = fulldatasize - prevdownsize;
 726     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 727     prevdownsize = fulldatasize;
 728     f.write(databytes);
 729    f.seek(0);
 730    fdata = f.getvalue();
 731    f.close();
 732    ft.close();
 733    os.remove(tmpfilename);
 734    exec_time_end = time.time();
 735    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 736   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 737  if(outfile=="-" and sys.version[0]>="3"):
 738   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 739   tmpfilename = pretmpfilename['Filename'];
 740   downloadsize = os.path.getsize(tmpfilename);
 741   fulldatasize = 0;
 742   prevdownsize = 0;
 743   exec_time_start = time.time();
 744   with open(tmpfilename, 'rb') as ft:
 745    f = BytesIO();
 746    while True:
 747     databytes = ft.read(buffersize[1]);
 748     if not databytes: break;
 749     datasize = len(databytes);
 750     fulldatasize = datasize + fulldatasize;
 751     percentage = "";
 752     if(downloadsize>0):
 753      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 754     downloaddiff = fulldatasize - prevdownsize;
 755     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 756     prevdownsize = fulldatasize;
 757     f.write(databytes);
 758    f.seek(0);
 759    fdata = f.getvalue();
 760    f.close();
 761    ft.close();
 762    os.remove(tmpfilename);
 763    exec_time_end = time.time();
 764    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 765   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 766  return returnval;
 767
 768 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 769  global geturls_download_sleep, havebrotli;
 770  if(sleep<0):
 771   sleep = geturls_download_sleep;
 772  urlparts = urlparse.urlparse(httpurl);
 773  if(isinstance(httpheaders, list)):
 774    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 775  if(urlparts.username is not None or urlparts.password is not None):
 776   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 777   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 778  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 779  geturls_opener.addheaders = httpheaders;
 780  time.sleep(sleep);
 781  if(urlparts[0]=="http"):
 782   httpconn = HTTPConnection(urlparts[1]);
 783  elif(urlparts[0]=="https"):
 784   httpconn = HTTPSConnection(urlparts[1]);
 785  else:
 786   return False;
 787  if(postdata is not None and not isinstance(postdata, dict)):
 788   postdata = urlencode(postdata);
 789  try:
 790   if(httpmethod=="GET"):
 791    httpconn.request("GET", urlparts[2], headers=httpheaders);
 792   elif(httpmethod=="POST"):
 793    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 794   else:
 795    httpconn.request("GET", urlparts[2], headers=httpheaders);
 796  except socket.timeout:
 797   log.info("Error With URL "+httpurl);
 798   return False;
 799  geturls_text = httpconn.getresponse();
 800  log.info("Downloading URL "+httpurl);
 801  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
 802   if(sys.version[0]=="2"):
 803    strbuf = StringIO(geturls_text.read());
 804   if(sys.version[0]>="3"):
 805    strbuf = BytesIO(geturls_text.read());
 806   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 807   returnval_content = gzstrbuf.read()[:];
 808  if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
 809   returnval_content = geturls_text.read()[:];
 810  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
 811   returnval_content = brotli.decompress(returnval_content);
 812  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 813  geturls_text.close();
 814  return returnval;
 815
 816 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 817  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 818  exec_time_start = time.time();
 819  myhash = hashlib.new("sha1");
 820  if(sys.version[0]=="2"):
 821   myhash.update(httpurl);
 822   myhash.update(str(buffersize));
 823   myhash.update(str(exec_time_start));
 824  if(sys.version[0]>="3"):
 825   myhash.update(httpurl.encode('utf-8'));
 826   myhash.update(str(buffersize).encode('utf-8'));
 827   myhash.update(str(exec_time_start).encode('utf-8'));
 828  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 829  if(sleep<0):
 830   sleep = geturls_download_sleep;
 831  urlparts = urlparse.urlparse(httpurl);
 832  if(isinstance(httpheaders, list)):
 833    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 834  if(urlparts.username is not None or urlparts.password is not None):
 835   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 836   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 837  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 838  geturls_opener.addheaders = httpheaders;
 839  time.sleep(sleep);
 840  if(urlparts[0]=="http"):
 841   httpconn = HTTPConnection(urlparts[1]);
 842  elif(urlparts[0]=="https"):
 843   httpconn = HTTPSConnection(urlparts[1]);
 844  else:
 845   return False;
 846  if(postdata is not None and not isinstance(postdata, dict)):
 847   postdata = urlencode(postdata);
 848  try:
 849   if(httpmethod=="GET"):
 850    httpconn.request("GET", urlparts[2], headers=httpheaders);
 851   elif(httpmethod=="POST"):
 852    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 853   else:
 854    httpconn.request("GET", urlparts[2], headers=httpheaders);
 855  except socket.timeout:
 856   log.info("Error With URL "+httpurl);
 857   return False;
 858  geturls_text = httpconn.getresponse();
 859  downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
 860  if(downloadsize is not None):
 861   downloadsize = int(downloadsize);
 862  if downloadsize is None: downloadsize = 0;
 863  fulldatasize = 0;
 864  prevdownsize = 0;
 865  log.info("Downloading URL "+httpurl);
 866  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 867   tmpfilename = f.name;
 868   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 869   while True:
 870    databytes = geturls_text.read(buffersize);
 871    if not databytes: break;
 872    datasize = len(databytes);
 873    fulldatasize = datasize + fulldatasize;
 874    percentage = "";
 875    if(downloadsize>0):
 876     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 877    downloaddiff = fulldatasize - prevdownsize;
 878    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 879    prevdownsize = fulldatasize;
 880    f.write(databytes);
 881   f.close();
 882  geturls_text.close();
 883  exec_time_end = time.time();
 884  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 885  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 886  return returnval;
 887
 888 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 889  global geturls_download_sleep;
 890  if(sleep<0):
 891   sleep = geturls_download_sleep;
 892  if(not outfile=="-"):
 893   outpath = outpath.rstrip(os.path.sep);
 894   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 895   if(not os.path.exists(outpath)):
 896    os.makedirs(outpath);
 897   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 898    return False;
 899   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 900    return False;
 901   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 902   if(not pretmpfilename):
 903    return False;
 904   tmpfilename = pretmpfilename['Filename'];
 905   downloadsize = os.path.getsize(tmpfilename);
 906   fulldatasize = 0;
 907   log.info("Moving file "+tmpfilename+" to "+filepath);
 908   exec_time_start = time.time();
 909   shutil.move(tmpfilename, filepath);
 910   exec_time_end = time.time();
 911   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 912   if(os.path.exists(tmpfilename)):
 913    os.remove(tmpfilename);
 914   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 915  if(outfile=="-" and sys.version[0]=="2"):
 916   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 917   if(not pretmpfilename):
 918    return False;
 919   tmpfilename = pretmpfilename['Filename'];
 920   downloadsize = os.path.getsize(tmpfilename);
 921   fulldatasize = 0;
 922   prevdownsize = 0;
 923   exec_time_start = time.time();
 924   with open(tmpfilename, 'rb') as ft:
 925    f = StringIO();
 926    while True:
 927     databytes = ft.read(buffersize[1]);
 928     if not databytes: break;
 929     datasize = len(databytes);
 930     fulldatasize = datasize + fulldatasize;
 931     percentage = "";
 932     if(downloadsize>0):
 933      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 934     downloaddiff = fulldatasize - prevdownsize;
 935     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 936     prevdownsize = fulldatasize;
 937     f.write(databytes);
 938    f.seek(0);
 939    fdata = f.getvalue();
 940    f.close();
 941    ft.close();
 942    os.remove(tmpfilename);
 943    exec_time_end = time.time();
 944    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 945   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 946  if(outfile=="-" and sys.version[0]>="3"):
 947   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 948   tmpfilename = pretmpfilename['Filename'];
 949   downloadsize = os.path.getsize(tmpfilename);
 950   fulldatasize = 0;
 951   prevdownsize = 0;
 952   exec_time_start = time.time();
 953   with open(tmpfilename, 'rb') as ft:
 954    f = BytesIO();
 955    while True:
 956     databytes = ft.read(buffersize[1]);
 957     if not databytes: break;
 958     datasize = len(databytes);
 959     fulldatasize = datasize + fulldatasize;
 960     percentage = "";
 961     if(downloadsize>0):
 962      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 963     downloaddiff = fulldatasize - prevdownsize;
 964     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 965     prevdownsize = fulldatasize;
 966     f.write(databytes);
 967    f.seek(0);
 968    fdata = f.getvalue();
 969    f.close();
 970    ft.close();
 971    os.remove(tmpfilename);
 972    exec_time_end = time.time();
 973    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 974   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 975  return returnval;
 976
 977 if(havehttplib2):
 978  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 979   global geturls_download_sleep, havebrotli;
 980   if(sleep<0):
 981    sleep = geturls_download_sleep;
 982   urlparts = urlparse.urlparse(httpurl);
 983   if(isinstance(httpheaders, list)):
 984    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 985   if(urlparts.username is not None or urlparts.password is not None):
 986    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 987    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 988   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 989   geturls_opener.addheaders = httpheaders;
 990   time.sleep(sleep);
 991   if(urlparts[0]=="http"):
 992    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
 993   elif(urlparts[0]=="https"):
 994    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
 995   else:
 996    return False;
 997   if(postdata is not None and not isinstance(postdata, dict)):
 998    postdata = urlencode(postdata);
 999   try:
1000    if(httpmethod=="GET"):
1001     httpconn.request("GET", urlparts[2], headers=httpheaders);
1002    elif(httpmethod=="POST"):
1003     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1004    else:
1005     httpconn.request("GET", urlparts[2], headers=httpheaders);
1006   except socket.timeout:
1007    log.info("Error With URL "+httpurl);
1008    return False;
1009   geturls_text = httpconn.getresponse();
1010   log.info("Downloading URL "+httpurl);
1011   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
1012    if(sys.version[0]=="2"):
1013     strbuf = StringIO(geturls_text.read());
1014    if(sys.version[0]>="3"):
1015     strbuf = BytesIO(geturls_text.read());
1016    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1017    returnval_content = gzstrbuf.read()[:];
1018   if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
1019    returnval_content = geturls_text.read()[:];
1020   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
1021    returnval_content = brotli.decompress(returnval_content);
1022   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1023   geturls_text.close();
1024   return returnval;
1025
1026 if(not havehttplib2):
1027  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1028   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1029   return returnval;
1030
1031 if(havehttplib2):
1032  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1033   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1034   exec_time_start = time.time();
1035   myhash = hashlib.new("sha1");
1036   if(sys.version[0]=="2"):
1037    myhash.update(httpurl);
1038    myhash.update(str(buffersize));
1039    myhash.update(str(exec_time_start));
1040   if(sys.version[0]>="3"):
1041    myhash.update(httpurl.encode('utf-8'));
1042    myhash.update(str(buffersize).encode('utf-8'));
1043    myhash.update(str(exec_time_start).encode('utf-8'));
1044   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1045   if(sleep<0):
1046    sleep = geturls_download_sleep;
1047   urlparts = urlparse.urlparse(httpurl);
1048   if(isinstance(httpheaders, list)):
1049    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1050   if(urlparts.username is not None or urlparts.password is not None):
1051    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1052    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1053   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1054   geturls_opener.addheaders = httpheaders;
1055   time.sleep(sleep);
1056   if(urlparts[0]=="http"):
1057    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1058   elif(urlparts[0]=="https"):
1059    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1060   else:
1061    return False;
1062   if(postdata is not None and not isinstance(postdata, dict)):
1063    postdata = urlencode(postdata);
1064   try:
1065    if(httpmethod=="GET"):
1066     httpconn.request("GET", urlparts[2], headers=httpheaders);
1067    elif(httpmethod=="POST"):
1068     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1069    else:
1070     httpconn.request("GET", urlparts[2], headers=httpheaders);
1071   except socket.timeout:
1072    log.info("Error With URL "+httpurl);
1073    return False;
1074   geturls_text = httpconn.getresponse();
1075   downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
1076   if(downloadsize is not None):
1077    downloadsize = int(downloadsize);
1078   if downloadsize is None: downloadsize = 0;
1079   fulldatasize = 0;
1080   prevdownsize = 0;
1081   log.info("Downloading URL "+httpurl);
1082   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1083    tmpfilename = f.name;
1084    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1085    while True:
1086     databytes = geturls_text.read(buffersize);
1087     if not databytes: break;
1088     datasize = len(databytes);
1089     fulldatasize = datasize + fulldatasize;
1090     percentage = "";
1091     if(downloadsize>0):
1092      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1093     downloaddiff = fulldatasize - prevdownsize;
1094     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1095     prevdownsize = fulldatasize;
1096     f.write(databytes);
1097    f.close();
1098   geturls_text.close();
1099   exec_time_end = time.time();
1100   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1101   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1102   return returnval;
1103
1104 if(not havehttplib2):
1105  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1106   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1107   return returnval;
1108
1109 if(havehttplib2):
1110  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1111   global geturls_download_sleep;
1112   if(sleep<0):
1113    sleep = geturls_download_sleep;
1114   if(not outfile=="-"):
1115    outpath = outpath.rstrip(os.path.sep);
1116    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1117    if(not os.path.exists(outpath)):
1118     os.makedirs(outpath);
1119    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1120     return False;
1121    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1122     return False;
1123    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1124    if(not pretmpfilename):
1125     return False;
1126    tmpfilename = pretmpfilename['Filename'];
1127    downloadsize = os.path.getsize(tmpfilename);
1128    fulldatasize = 0;
1129    log.info("Moving file "+tmpfilename+" to "+filepath);
1130    exec_time_start = time.time();
1131    shutil.move(tmpfilename, filepath);
1132    exec_time_end = time.time();
1133    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1134    if(os.path.exists(tmpfilename)):
1135     os.remove(tmpfilename);
1136    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1137   if(outfile=="-" and sys.version[0]=="2"):
1138    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1139    if(not pretmpfilename):
1140     return False;
1141    tmpfilename = pretmpfilename['Filename'];
1142    downloadsize = os.path.getsize(tmpfilename);
1143    fulldatasize = 0;
1144    prevdownsize = 0;
1145    exec_time_start = time.time();
1146    with open(tmpfilename, 'rb') as ft:
1147     f = StringIO();
1148     while True:
1149      databytes = ft.read(buffersize[1]);
1150      if not databytes: break;
1151      datasize = len(databytes);
1152      fulldatasize = datasize + fulldatasize;
1153      percentage = "";
1154      if(downloadsize>0):
1155       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1156      downloaddiff = fulldatasize - prevdownsize;
1157      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1158      prevdownsize = fulldatasize;
1159      f.write(databytes);
1160     f.seek(0);
1161     fdata = f.getvalue();
1162     f.close();
1163     ft.close();
1164     os.remove(tmpfilename);
1165     exec_time_end = time.time();
1166     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1167    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1168   if(outfile=="-" and sys.version[0]>="3"):
1169    pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1170    tmpfilename = pretmpfilename['Filename'];
1171    downloadsize = os.path.getsize(tmpfilename);
1172    fulldatasize = 0;
1173    prevdownsize = 0;
1174    exec_time_start = time.time();
1175    with open(tmpfilename, 'rb') as ft:
1176     f = BytesIO();
1177     while True:
1178      databytes = ft.read(buffersize[1]);
1179      if not databytes: break;
1180      datasize = len(databytes);
1181      fulldatasize = datasize + fulldatasize;
1182      percentage = "";
1183      if(downloadsize>0):
1184       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1185      downloaddiff = fulldatasize - prevdownsize;
1186      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1187      prevdownsize = fulldatasize;
1188      f.write(databytes);
1189     f.seek(0);
1190     fdata = f.getvalue();
1191     f.close();
1192     ft.close();
1193     os.remove(tmpfilename);
1194     exec_time_end = time.time();
1195     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1196    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1197   return returnval;
1198
1199 if(not havehttplib2):
1200  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1201   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1202   return returnval;
1203
1204 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1205  global geturls_download_sleep, havebrotli;
1206  if(sleep<0):
1207   sleep = geturls_download_sleep;
1208  urlparts = urlparse.urlparse(httpurl);
1209  if(isinstance(httpheaders, list)):
1210    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1211  if(urlparts.username is not None or urlparts.password is not None):
1212   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1213   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1214  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1215  if(isinstance(httpheaders, dict)):
1216   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1217  geturls_opener.addheaders = httpheaders;
1218  urllib.request.install_opener(geturls_opener);
1219  time.sleep(sleep);
1220  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1221  if(postdata is not None and not isinstance(postdata, dict)):
1222   postdata = urlencode(postdata);
1223  try:
1224   if(httpmethod=="GET"):
1225    geturls_request = Request(httpurl, headers=httpheaders);
1226    geturls_text = urlopen(geturls_request);
1227   elif(httpmethod=="POST"):
1228    geturls_request = Request(httpurl, headers=httpheaders);
1229    geturls_text = urlopen(geturls_request, data=postdata);
1230   else:
1231    geturls_request = Request(httpurl, headers=httpheaders);
1232    geturls_text = urlopen(geturls_request);
1233  except HTTPError as geturls_text_error:
1234   geturls_text = geturls_text_error;
1235   log.info("Error With URL "+httpurl);
1236  except URLError:
1237   log.info("Error With URL "+httpurl);
1238   return False;
1239  except socket.timeout:
1240   log.info("Error With URL "+httpurl);
1241   return False;
1242  log.info("Downloading URL "+httpurl);
1243  if(geturls_text.headers.get("Content-Encoding")=="gzip" or geturls_text.headers.get("Content-Encoding")=="deflate"):
1244   if(sys.version[0]=="2"):
1245    strbuf = StringIO(geturls_text.read());
1246   if(sys.version[0]>="3"):
1247    strbuf = BytesIO(geturls_text.read());
1248   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1249   returnval_content = gzstrbuf.read()[:];
1250  if(geturls_text.headers.get("Content-Encoding")!="gzip" and geturls_text.headers.get("Content-Encoding")!="deflate"):
1251   returnval_content = geturls_text.read()[:];
1252  if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1253   returnval_content = brotli.decompress(returnval_content);
1254  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1255  geturls_text.close();
1256  return returnval;
1257
1258 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1259  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1260  exec_time_start = time.time();
1261  myhash = hashlib.new("sha1");
1262  if(sys.version[0]=="2"):
1263   myhash.update(httpurl);
1264   myhash.update(str(buffersize));
1265   myhash.update(str(exec_time_start));
1266  if(sys.version[0]>="3"):
1267   myhash.update(httpurl.encode('utf-8'));
1268   myhash.update(str(buffersize).encode('utf-8'));
1269   myhash.update(str(exec_time_start).encode('utf-8'));
1270  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1271  if(sleep<0):
1272   sleep = geturls_download_sleep;
1273  urlparts = urlparse.urlparse(httpurl);
1274  if(isinstance(httpheaders, list)):
1275    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1276  if(urlparts.username is not None or urlparts.password is not None):
1277   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1278   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1279  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1280  if(isinstance(httpheaders, dict)):
1281   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1282  geturls_opener.addheaders = httpheaders;
1283  urllib.request.install_opener(geturls_opener);
1284  time.sleep(sleep);
1285  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1286  if(postdata is not None and not isinstance(postdata, dict)):
1287   postdata = urlencode(postdata);
1288  try:
1289   if(httpmethod=="GET"):
1290    geturls_request = Request(httpurl, headers=httpheaders);
1291    geturls_text = urlopen(geturls_request);
1292   elif(httpmethod=="POST"):
1293    geturls_request = Request(httpurl, headers=httpheaders);
1294    geturls_text = urlopen(geturls_request, data=postdata);
1295   else:
1296    geturls_request = Request(httpurl, headers=httpheaders);
1297    geturls_text = urlopen(geturls_request);
1298  except HTTPError as geturls_text_error:
1299   geturls_text = geturls_text_error;
1300   log.info("Error With URL "+httpurl);
1301  except URLError:
1302   log.info("Error With URL "+httpurl);
1303   return False;
1304  except socket.timeout:
1305   log.info("Error With URL "+httpurl);
1306   return False;
1307  downloadsize = geturls_text.headers.get('Content-Length');
1308  if(downloadsize is not None):
1309   downloadsize = int(downloadsize);
1310  if downloadsize is None: downloadsize = 0;
1311  fulldatasize = 0;
1312  prevdownsize = 0;
1313  log.info("Downloading URL "+httpurl);
1314  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1315   tmpfilename = f.name;
1316   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1317   while True:
1318    databytes = geturls_text.read(buffersize);
1319    if not databytes: break;
1320    datasize = len(databytes);
1321    fulldatasize = datasize + fulldatasize;
1322    percentage = "";
1323    if(downloadsize>0):
1324     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1325    downloaddiff = fulldatasize - prevdownsize;
1326    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1327    prevdownsize = fulldatasize;
1328    f.write(databytes);
1329   f.close();
1330  geturls_text.close();
1331  exec_time_end = time.time();
1332  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1333  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1334  return returnval;
1335
1336 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1337  global geturls_download_sleep;
1338  if(sleep<0):
1339   sleep = geturls_download_sleep;
1340  if(not outfile=="-"):
1341   outpath = outpath.rstrip(os.path.sep);
1342   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1343   if(not os.path.exists(outpath)):
1344    os.makedirs(outpath);
1345   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1346    return False;
1347   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1348    return False;
1349   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1350   if(not pretmpfilename):
1351    return False;
1352   tmpfilename = pretmpfilename['Filename'];
1353   downloadsize = os.path.getsize(tmpfilename);
1354   fulldatasize = 0;
1355   log.info("Moving file "+tmpfilename+" to "+filepath);
1356   exec_time_start = time.time();
1357   shutil.move(tmpfilename, filepath);
1358   exec_time_end = time.time();
1359   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1360   if(os.path.exists(tmpfilename)):
1361    os.remove(tmpfilename);
1362   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent':pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1363  if(outfile=="-" and sys.version[0]=="2"):
1364   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1365   if(not pretmpfilename):
1366    return False;
1367   tmpfilename = pretmpfilename['Filename'];
1368   downloadsize = os.path.getsize(tmpfilename);
1369   fulldatasize = 0;
1370   prevdownsize = 0;
1371   exec_time_start = time.time();
1372   with open(tmpfilename, 'rb') as ft:
1373    f = StringIO();
1374    while True:
1375     databytes = ft.read(buffersize[1]);
1376     if not databytes: break;
1377     datasize = len(databytes);
1378     fulldatasize = datasize + fulldatasize;
1379     percentage = "";
1380     if(downloadsize>0):
1381      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1382     downloaddiff = fulldatasize - prevdownsize;
1383     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1384     prevdownsize = fulldatasize;
1385     f.write(databytes);
1386    f.seek(0);
1387    fdata = f.getvalue();
1388    f.close();
1389    ft.close();
1390    os.remove(tmpfilename);
1391    exec_time_end = time.time();
1392    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1393   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1394  if(outfile=="-" and sys.version[0]>="3"):
1395   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1396   tmpfilename = pretmpfilename['Filename'];
1397   downloadsize = os.path.getsize(tmpfilename);
1398   fulldatasize = 0;
1399   prevdownsize = 0;
1400   exec_time_start = time.time();
1401   with open(tmpfilename, 'rb') as ft:
1402    f = BytesIO();
1403    while True:
1404     databytes = ft.read(buffersize[1]);
1405     if not databytes: break;
1406     datasize = len(databytes);
1407     fulldatasize = datasize + fulldatasize;
1408     percentage = "";
1409     if(downloadsize>0):
1410      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1411     downloaddiff = fulldatasize - prevdownsize;
1412     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1413     prevdownsize = fulldatasize;
1414     f.write(databytes);
1415    f.seek(0);
1416    fdata = f.getvalue();
1417    f.close();
1418    ft.close();
1419    os.remove(tmpfilename);
1420    exec_time_end = time.time();
1421    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1422   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1423  return returnval;
1424
1425 if(haverequests):
1426  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1427   global geturls_download_sleep, havebrotli;
1428   if(sleep<0):
1429    sleep = geturls_download_sleep;
1430   urlparts = urlparse.urlparse(httpurl);
1431   if(isinstance(httpheaders, list)):
1432    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1433   if(urlparts.username is not None or urlparts.password is not None):
1434    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1435    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1436   time.sleep(sleep);
1437   if(postdata is not None and not isinstance(postdata, dict)):
1438    postdata = urlencode(postdata);
1439   try:
1440    if(httpmethod=="GET"):
1441     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1442    elif(httpmethod=="POST"):
1443     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1444    else:
1445     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1446   except requests.exceptions.ConnectTimeout:
1447    log.info("Error With URL "+httpurl);
1448    return False;
1449   except socket.timeout:
1450    log.info("Error With URL "+httpurl);
1451    return False;
1452   log.info("Downloading URL "+httpurl);
1453   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1454    if(sys.version[0]=="2"):
1455     strbuf = StringIO(geturls_text.content);
1456    if(sys.version[0]>="3"):
1457     strbuf = BytesIO(geturls_text.content);
1458    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1459    returnval_content = gzstrbuf.content[:];
1460   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1461    returnval_content = geturls_text.content[:];
1462   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1463    returnval_content = brotli.decompress(returnval_content);
1464   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1465   geturls_text.close();
1466   return returnval;
1467
1468 if(not haverequests):
1469  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1470   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1471   return returnval;
1472
1473 if(haverequests):
1474  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1475   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1476   exec_time_start = time.time();
1477   myhash = hashlib.new("sha1");
1478   if(sys.version[0]=="2"):
1479    myhash.update(httpurl);
1480    myhash.update(str(buffersize));
1481    myhash.update(str(exec_time_start));
1482   if(sys.version[0]>="3"):
1483    myhash.update(httpurl.encode('utf-8'));
1484    myhash.update(str(buffersize).encode('utf-8'));
1485    myhash.update(str(exec_time_start).encode('utf-8'));
1486   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1487   if(sleep<0):
1488    sleep = geturls_download_sleep;
1489   urlparts = urlparse.urlparse(httpurl);
1490   if(isinstance(httpheaders, list)):
1491    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1492   if(urlparts.username is not None or urlparts.password is not None):
1493    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1494    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1495   time.sleep(sleep);
1496   if(postdata is not None and not isinstance(postdata, dict)):
1497    postdata = urlencode(postdata);
1498   try:
1499    if(httpmethod=="GET"):
1500     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1501    elif(httpmethod=="POST"):
1502     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1503    else:
1504     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1505   except requests.exceptions.ConnectTimeout:
1506    log.info("Error With URL "+httpurl);
1507    return False;
1508   except socket.timeout:
1509    log.info("Error With URL "+httpurl);
1510    return False;
1511   downloadsize = int(geturls_text.headers.get('Content-Length'));
1512   if(downloadsize is not None):
1513    downloadsize = int(downloadsize);
1514   if downloadsize is None: downloadsize = 0;
1515   fulldatasize = 0;
1516   prevdownsize = 0;
1517   log.info("Downloading URL "+httpurl);
1518   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1519    tmpfilename = f.name;
1520    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1521    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1522     datasize = len(databytes);
1523     fulldatasize = datasize + fulldatasize;
1524     percentage = "";
1525     if(downloadsize>0):
1526      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1527     downloaddiff = fulldatasize - prevdownsize;
1528     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1529     prevdownsize = fulldatasize;
1530     f.write(databytes);
1531    f.close();
1532   geturls_text.close();
1533   exec_time_end = time.time();
1534   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1535   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1536   return returnval;
1537
1538 if(not haverequests):
1539  def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1540   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1541   return returnval;
1542
1543 if(haverequests):
1544  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1545   global geturls_download_sleep;
1546   if(sleep<0):
1547    sleep = geturls_download_sleep;
1548   if(not outfile=="-"):
1549    outpath = outpath.rstrip(os.path.sep);
1550    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1551    if(not os.path.exists(outpath)):
1552     os.makedirs(outpath);
1553    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1554     return False;
1555    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1556     return False;
1557    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1558    if(not pretmpfilename):
1559     return False;
1560    tmpfilename = pretmpfilename['Filename'];
1561    downloadsize = os.path.getsize(tmpfilename);
1562    fulldatasize = 0;
1563    log.info("Moving file "+tmpfilename+" to "+filepath);
1564    exec_time_start = time.time();
1565    shutil.move(tmpfilename, filepath);
1566    exec_time_end = time.time();
1567    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1568    if(os.path.exists(tmpfilename)):
1569     os.remove(tmpfilename);
1570    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1571   if(outfile=="-" and sys.version[0]=="2"):
1572    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1573    if(not pretmpfilename):
1574     return False;
1575    tmpfilename = pretmpfilename['Filename'];
1576    downloadsize = os.path.getsize(tmpfilename);
1577    fulldatasize = 0;
1578    prevdownsize = 0;
1579    exec_time_start = time.time();
1580    with open(tmpfilename, 'rb') as ft:
1581     f = StringIO();
1582     while True:
1583      databytes = ft.read(buffersize[1]);
1584      if not databytes: break;
1585      datasize = len(databytes);
1586      fulldatasize = datasize + fulldatasize;
1587      percentage = "";
1588      if(downloadsize>0):
1589       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1590      downloaddiff = fulldatasize - prevdownsize;
1591      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1592      prevdownsize = fulldatasize;
1593      f.write(databytes);
1594     f.seek(0);
1595     fdata = f.getvalue();
1596     f.close();
1597     ft.close();
1598     os.remove(tmpfilename);
1599     exec_time_end = time.time();
1600     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1601    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1602   if(outfile=="-" and sys.version[0]>="3"):
1603    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1604    tmpfilename = pretmpfilename['Filename'];
1605    downloadsize = os.path.getsize(tmpfilename);
1606    fulldatasize = 0;
1607    prevdownsize = 0;
1608    exec_time_start = time.time();
1609    with open(tmpfilename, 'rb') as ft:
1610     f = BytesIO();
1611     while True:
1612      databytes = ft.read(buffersize[1]);
1613      if not databytes: break;
1614      datasize = len(databytes);
1615      fulldatasize = datasize + fulldatasize;
1616      percentage = "";
1617      if(downloadsize>0):
1618       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1619      downloaddiff = fulldatasize - prevdownsize;
1620      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1621      prevdownsize = fulldatasize;
1622      f.write(databytes);
1623     f.seek(0);
1624     fdata = f.getvalue();
1625     f.close();
1626     ft.close();
1627     os.remove(tmpfilename);
1628     exec_time_end = time.time();
1629     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1630    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1631   return returnval;
1632
1633 if(not haverequests):
1634  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1635   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1636   return returnval;
1637
1638 if(havehttpx):
1639  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1640   global geturls_download_sleep, havebrotli;
1641   if(sleep<0):
1642    sleep = geturls_download_sleep;
1643   urlparts = urlparse.urlparse(httpurl);
1644   if(isinstance(httpheaders, list)):
1645    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1646   if(urlparts.username is not None or urlparts.password is not None):
1647    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1648    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1649   time.sleep(sleep);
1650   if(postdata is not None and not isinstance(postdata, dict)):
1651    postdata = urlencode(postdata);
1652   try:
1653    if(httpmethod=="GET"):
1654     geturls_text = httpx.get(httpurl, headers=httpheaders, cookies=httpcookie);
1655    elif(httpmethod=="POST"):
1656     geturls_text = httpx.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1657    else:
1658     geturls_text = httpx.get(httpurl, headers=httpheaders, cookies=httpcookie);
1659   except requests.exceptions.ConnectTimeout:
1660    log.info("Error With URL "+httpurl);
1661    return False;
1662   except socket.timeout:
1663    log.info("Error With URL "+httpurl);
1664    return False;
1665   log.info("Downloading URL "+httpurl);
1666   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1667    if(sys.version[0]=="2"):
1668     strbuf = StringIO(geturls_text.content);
1669    if(sys.version[0]>="3"):
1670     strbuf = BytesIO(geturls_text.content);
1671    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1672    returnval_content = gzstrbuf.content[:];
1673   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1674    returnval_content = geturls_text.content[:];
1675   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1676    returnval_content = brotli.decompress(returnval_content);
1677   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1678   geturls_text.close();
1679   return returnval;
1680
1681 if(not havehttpx):
1682  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1683   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1684   return returnval;
1685
1686 if(havehttpx):
1687  def download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1688   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1689   exec_time_start = time.time();
1690   myhash = hashlib.new("sha1");
1691   if(sys.version[0]=="2"):
1692    myhash.update(httpurl);
1693    myhash.update(str(buffersize));
1694    myhash.update(str(exec_time_start));
1695   if(sys.version[0]>="3"):
1696    myhash.update(httpurl.encode('utf-8'));
1697    myhash.update(str(buffersize).encode('utf-8'));
1698    myhash.update(str(exec_time_start).encode('utf-8'));
1699   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1700   if(sleep<0):
1701    sleep = geturls_download_sleep;
1702   urlparts = urlparse.urlparse(httpurl);
1703   if(isinstance(httpheaders, list)):
1704    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1705   if(urlparts.username is not None or urlparts.password is not None):
1706    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1707    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1708   time.sleep(sleep);
1709   if(postdata is not None and not isinstance(postdata, dict)):
1710    postdata = urlencode(postdata);
1711   try:
1712    if(httpmethod=="GET"):
1713     geturls_text = httpx.get(httpurl, headers=httpheaders, cookies=httpcookie);
1714    elif(httpmethod=="POST"):
1715     geturls_text = httpx.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1716    else:
1717     geturls_text = httpx.get(httpurl, headers=httpheaders, cookies=httpcookie);
1718   except requests.exceptions.ConnectTimeout:
1719    log.info("Error With URL "+httpurl);
1720    return False;
1721   except socket.timeout:
1722    log.info("Error With URL "+httpurl);
1723    return False;
1724   downloadsize = int(geturls_text.headers.get('Content-Length'));
1725   if(downloadsize is not None):
1726    downloadsize = int(downloadsize);
1727   if downloadsize is None: downloadsize = 0;
1728   fulldatasize = 0;
1729   prevdownsize = 0;
1730   log.info("Downloading URL "+httpurl);
1731   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1732    tmpfilename = f.name;
1733    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1734    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1735     datasize = len(databytes);
1736     fulldatasize = datasize + fulldatasize;
1737     percentage = "";
1738     if(downloadsize>0):
1739      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1740     downloaddiff = fulldatasize - prevdownsize;
1741     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1742     prevdownsize = fulldatasize;
1743     f.write(databytes);
1744    f.close();
1745   geturls_text.close();
1746   exec_time_end = time.time();
1747   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1748   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1749   return returnval;
1750
1751 if(not havehttpx):
1752  def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1753   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1754   return returnval;
1755
1756 if(havehttpx):
1757  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1758   global geturls_download_sleep;
1759   if(sleep<0):
1760    sleep = geturls_download_sleep;
1761   if(not outfile=="-"):
1762    outpath = outpath.rstrip(os.path.sep);
1763    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1764    if(not os.path.exists(outpath)):
1765     os.makedirs(outpath);
1766    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1767     return False;
1768    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1769     return False;
1770    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1771    if(not pretmpfilename):
1772     return False;
1773    tmpfilename = pretmpfilename['Filename'];
1774    downloadsize = os.path.getsize(tmpfilename);
1775    fulldatasize = 0;
1776    log.info("Moving file "+tmpfilename+" to "+filepath);
1777    exec_time_start = time.time();
1778    shutil.move(tmpfilename, filepath);
1779    exec_time_end = time.time();
1780    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1781    if(os.path.exists(tmpfilename)):
1782     os.remove(tmpfilename);
1783    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1784   if(outfile=="-" and sys.version[0]=="2"):
1785    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1786    if(not pretmpfilename):
1787     return False;
1788    tmpfilename = pretmpfilename['Filename'];
1789    downloadsize = os.path.getsize(tmpfilename);
1790    fulldatasize = 0;
1791    prevdownsize = 0;
1792    exec_time_start = time.time();
1793    with open(tmpfilename, 'rb') as ft:
1794     f = StringIO();
1795     while True:
1796      databytes = ft.read(buffersize[1]);
1797      if not databytes: break;
1798      datasize = len(databytes);
1799      fulldatasize = datasize + fulldatasize;
1800      percentage = "";
1801      if(downloadsize>0):
1802       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1803      downloaddiff = fulldatasize - prevdownsize;
1804      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1805      prevdownsize = fulldatasize;
1806      f.write(databytes);
1807     f.seek(0);
1808     fdata = f.getvalue();
1809     f.close();
1810     ft.close();
1811     os.remove(tmpfilename);
1812     exec_time_end = time.time();
1813     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1814    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1815   if(outfile=="-" and sys.version[0]>="3"):
1816    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1817    tmpfilename = pretmpfilename['Filename'];
1818    downloadsize = os.path.getsize(tmpfilename);
1819    fulldatasize = 0;
1820    prevdownsize = 0;
1821    exec_time_start = time.time();
1822    with open(tmpfilename, 'rb') as ft:
1823     f = BytesIO();
1824     while True:
1825      databytes = ft.read(buffersize[1]);
1826      if not databytes: break;
1827      datasize = len(databytes);
1828      fulldatasize = datasize + fulldatasize;
1829      percentage = "";
1830      if(downloadsize>0):
1831       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1832      downloaddiff = fulldatasize - prevdownsize;
1833      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1834      prevdownsize = fulldatasize;
1835      f.write(databytes);
1836     f.seek(0);
1837     fdata = f.getvalue();
1838     f.close();
1839     ft.close();
1840     os.remove(tmpfilename);
1841     exec_time_end = time.time();
1842     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1843    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1844   return returnval;
1845
1846 if(not havehttpx):
1847  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1848   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1849   return returnval;
1850
1851 if(haveurllib3):
1852  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1853   global geturls_download_sleep, havebrotli;
1854   if(sleep<0):
1855    sleep = geturls_download_sleep;
1856   urlparts = urlparse.urlparse(httpurl);
1857   if(isinstance(httpheaders, list)):
1858    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1859   if(urlparts.username is not None or urlparts.password is not None):
1860    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1861    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1862   time.sleep(sleep);
1863   urllib_pool = urllib3.PoolManager(headers=httpheaders);
1864   if(postdata is not None and not isinstance(postdata, dict)):
1865    postdata = urlencode(postdata);
1866   try:
1867    if(httpmethod=="GET"):
1868     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1869    elif(httpmethod=="POST"):
1870     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
1871    else:
1872     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1873   except urllib3.exceptions.ConnectTimeoutError:
1874    log.info("Error With URL "+httpurl);
1875    return False;
1876   except urllib3.exceptions.MaxRetryError:
1877    log.info("Error With URL "+httpurl);
1878    return False;
1879   except socket.timeout:
1880    log.info("Error With URL "+httpurl);
1881    return False;
1882   log.info("Downloading URL "+httpurl);
1883   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
1884    if(sys.version[0]=="2"):
1885     strbuf = StringIO(geturls_text.read());
1886    if(sys.version[0]>="3"):
1887     strbuf = BytesIO(geturls_text.read());
1888    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1889    returnval_content = gzstrbuf.read()[:];
1890   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
1891    returnval_content = geturls_text.read()[:];
1892   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
1893    returnval_content = brotli.decompress(returnval_content);
1894   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
1895   geturls_text.close();
1896   return returnval;
1897
1898 if(not haveurllib3):
1899  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1900   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1901   return returnval;
1902
1903 if(haveurllib3):
1904  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1905   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1906   exec_time_start = time.time();
1907   myhash = hashlib.new("sha1");
1908   if(sys.version[0]=="2"):
1909    myhash.update(httpurl);
1910    myhash.update(str(buffersize));
1911    myhash.update(str(exec_time_start));
1912   if(sys.version[0]>="3"):
1913    myhash.update(httpurl.encode('utf-8'));
1914    myhash.update(str(buffersize).encode('utf-8'));
1915    myhash.update(str(exec_time_start).encode('utf-8'));
1916   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1917   if(sleep<0):
1918    sleep = geturls_download_sleep;
1919   urlparts = urlparse.urlparse(httpurl);
1920   if(isinstance(httpheaders, list)):
1921    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1922   if(urlparts.username is not None or urlparts.password is not None):
1923    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1924    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1925   time.sleep(sleep);
1926   urllib_pool = urllib3.PoolManager(headers=httpheaders);
1927   if(postdata is not None and not isinstance(postdata, dict)):
1928    postdata = urlencode(postdata);
1929   try:
1930    if(httpmethod=="GET"):
1931     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1932    elif(httpmethod=="POST"):
1933     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
1934    else:
1935     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1936   except urllib3.exceptions.ConnectTimeoutError:
1937    log.info("Error With URL "+httpurl);
1938    return False;
1939   except urllib3.exceptions.MaxRetryError:
1940    log.info("Error With URL "+httpurl);
1941    return False;
1942   except socket.timeout:
1943    log.info("Error With URL "+httpurl);
1944    return False;
1945   downloadsize = int(geturls_text.headers.get('Content-Length'));
1946   if(downloadsize is not None):
1947    downloadsize = int(downloadsize);
1948   if downloadsize is None: downloadsize = 0;
1949   fulldatasize = 0;
1950   prevdownsize = 0;
1951   log.info("Downloading URL "+httpurl);
1952   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1953    tmpfilename = f.name;
1954    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
1955    while True:
1956     databytes = geturls_text.read(buffersize);
1957     if not databytes: break;
1958     datasize = len(databytes);
1959     fulldatasize = datasize + fulldatasize;
1960     percentage = "";
1961     if(downloadsize>0):
1962      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1963     downloaddiff = fulldatasize - prevdownsize;
1964     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1965     prevdownsize = fulldatasize;
1966     f.write(databytes);
1967    f.close();
1968   geturls_text.close();
1969   exec_time_end = time.time();
1970   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1971   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1972   return returnval;
1973
1974 if(not haveurllib3):
1975  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1976   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1977   return returnval;
1978
1979 if(haveurllib3):
1980  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1981   global geturls_download_sleep;
1982   if(sleep<0):
1983    sleep = geturls_download_sleep;
1984   if(not outfile=="-"):
1985    outpath = outpath.rstrip(os.path.sep);
1986    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1987    if(not os.path.exists(outpath)):
1988     os.makedirs(outpath);
1989    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1990     return False;
1991    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1992     return False;
1993    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1994    if(not pretmpfilename):
1995     return False;
1996    tmpfilename = pretmpfilename['Filename'];
1997    downloadsize = os.path.getsize(tmpfilename);
1998    fulldatasize = 0;
1999    log.info("Moving file "+tmpfilename+" to "+filepath);
2000    exec_time_start = time.time();
2001    shutil.move(tmpfilename, filepath);
2002    exec_time_end = time.time();
2003    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2004    if(os.path.exists(tmpfilename)):
2005     os.remove(tmpfilename);
2006    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2007   if(outfile=="-" and sys.version[0]=="2"):
2008    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2009    if(not pretmpfilename):
2010     return False;
2011    tmpfilename = pretmpfilename['Filename'];
2012    downloadsize = os.path.getsize(tmpfilename);
2013    fulldatasize = 0;
2014    prevdownsize = 0;
2015    exec_time_start = time.time();
2016    with open(tmpfilename, 'rb') as ft:
2017     f = StringIO();
2018     while True:
2019      databytes = ft.read(buffersize[1]);
2020      if not databytes: break;
2021      datasize = len(databytes);
2022      fulldatasize = datasize + fulldatasize;
2023      percentage = "";
2024      if(downloadsize>0):
2025       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2026      downloaddiff = fulldatasize - prevdownsize;
2027      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2028      prevdownsize = fulldatasize;
2029      f.write(databytes);
2030     f.seek(0);
2031     fdata = f.getvalue();
2032     f.close();
2033     ft.close();
2034     os.remove(tmpfilename);
2035     exec_time_end = time.time();
2036     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2037    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2038   if(outfile=="-" and sys.version[0]>="3"):
2039    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2040    tmpfilename = pretmpfilename['Filename'];
2041    downloadsize = os.path.getsize(tmpfilename);
2042    fulldatasize = 0;
2043    prevdownsize = 0;
2044    exec_time_start = time.time();
2045    with open(tmpfilename, 'rb') as ft:
2046     f = BytesIO();
2047     while True:
2048      databytes = ft.read(buffersize[1]);
2049      if not databytes: break;
2050      datasize = len(databytes);
2051      fulldatasize = datasize + fulldatasize;
2052      percentage = "";
2053      if(downloadsize>0):
2054       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2055      downloaddiff = fulldatasize - prevdownsize;
2056      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2057      prevdownsize = fulldatasize;
2058      f.write(databytes);
2059     f.seek(0);
2060     fdata = f.getvalue();
2061     f.close();
2062     ft.close();
2063     os.remove(tmpfilename);
2064     exec_time_end = time.time();
2065     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2066    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2067   return returnval;
2068
2069 if(not haveurllib3):
2070  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2071   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2072   return returnval;
2073
2074 if(haveurllib3):
2075  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2076   global geturls_download_sleep, havebrotli;
2077   if(sleep<0):
2078    sleep = geturls_download_sleep;
2079   urlparts = urlparse.urlparse(httpurl);
2080   if(isinstance(httpheaders, list)):
2081    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2082   if(urlparts.username is not None or urlparts.password is not None):
2083    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2084    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2085   time.sleep(sleep);
2086   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2087   if(postdata is not None and not isinstance(postdata, dict)):
2088    postdata = urlencode(postdata);
2089   try:
2090    if(httpmethod=="GET"):
2091     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2092    elif(httpmethod=="POST"):
2093     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2094    else:
2095     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2096   except urllib3.exceptions.ConnectTimeoutError:
2097    log.info("Error With URL "+httpurl);
2098    return False;
2099   except urllib3.exceptions.MaxRetryError:
2100    log.info("Error With URL "+httpurl);
2101    return False;
2102   except socket.timeout:
2103    log.info("Error With URL "+httpurl);
2104    return False;
2105   log.info("Downloading URL "+httpurl);
2106   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2107    if(sys.version[0]=="2"):
2108     strbuf = StringIO(geturls_text.read());
2109    if(sys.version[0]>="3"):
2110     strbuf = BytesIO(geturls_text.read());
2111    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2112    returnval_content = gzstrbuf.read()[:];
2113   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2114    returnval_content = geturls_text.read()[:];
2115   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2116    returnval_content = brotli.decompress(returnval_content);
2117   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2118   geturls_text.close();
2119   return returnval;
2120
2121 if(not haveurllib3):
2122  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2123   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2124   return returnval;
2125
2126 if(haveurllib3):
2127  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2128   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2129   exec_time_start = time.time();
2130   myhash = hashlib.new("sha1");
2131   if(sys.version[0]=="2"):
2132    myhash.update(httpurl);
2133    myhash.update(str(buffersize));
2134    myhash.update(str(exec_time_start));
2135   if(sys.version[0]>="3"):
2136    myhash.update(httpurl.encode('utf-8'));
2137    myhash.update(str(buffersize).encode('utf-8'));
2138    myhash.update(str(exec_time_start).encode('utf-8'));
2139   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2140   if(sleep<0):
2141    sleep = geturls_download_sleep;
2142   urlparts = urlparse.urlparse(httpurl);
2143   if(isinstance(httpheaders, list)):
2144    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2145   if(urlparts.username is not None or urlparts.password is not None):
2146    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2147    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2148   time.sleep(sleep);
2149   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2150   if(postdata is not None and not isinstance(postdata, dict)):
2151    postdata = urlencode(postdata);
2152   try:
2153    if(httpmethod=="GET"):
2154     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2155    elif(httpmethod=="POST"):
2156     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2157    else:
2158     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2159   except urllib3.exceptions.ConnectTimeoutError:
2160    log.info("Error With URL "+httpurl);
2161    return False;
2162   except urllib3.exceptions.MaxRetryError:
2163    log.info("Error With URL "+httpurl);
2164    return False;
2165   except socket.timeout:
2166    log.info("Error With URL "+httpurl);
2167    return False;
2168   downloadsize = int(geturls_text.headers.get('Content-Length'));
2169   if(downloadsize is not None):
2170    downloadsize = int(downloadsize);
2171   if downloadsize is None: downloadsize = 0;
2172   fulldatasize = 0;
2173   prevdownsize = 0;
2174   log.info("Downloading URL "+httpurl);
2175   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2176    tmpfilename = f.name;
2177    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2178    while True:
2179     databytes = geturls_text.read(buffersize);
2180     if not databytes: break;
2181     datasize = len(databytes);
2182     fulldatasize = datasize + fulldatasize;
2183     percentage = "";
2184     if(downloadsize>0):
2185      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2186     downloaddiff = fulldatasize - prevdownsize;
2187     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2188     prevdownsize = fulldatasize;
2189     f.write(databytes);
2190    f.close();
2191   geturls_text.close();
2192   exec_time_end = time.time();
2193   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2194   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2195   return returnval;
2196
2197 if(not haveurllib3):
2198  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2199   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2200   return returnval;
2201
2202 if(haveurllib3):
2203  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2204   global geturls_download_sleep;
2205   if(sleep<0):
2206    sleep = geturls_download_sleep;
2207   if(not outfile=="-"):
2208    outpath = outpath.rstrip(os.path.sep);
2209    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2210    if(not os.path.exists(outpath)):
2211     os.makedirs(outpath);
2212    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2213     return False;
2214    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2215     return False;
2216    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2217    if(not pretmpfilename):
2218     return False;
2219    tmpfilename = pretmpfilename['Filename'];
2220    downloadsize = os.path.getsize(tmpfilename);
2221    fulldatasize = 0;
2222    log.info("Moving file "+tmpfilename+" to "+filepath);
2223    exec_time_start = time.time();
2224    shutil.move(tmpfilename, filepath);
2225    exec_time_end = time.time();
2226    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2227    if(os.path.exists(tmpfilename)):
2228     os.remove(tmpfilename);
2229    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2230   if(outfile=="-" and sys.version[0]=="2"):
2231    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2232    if(not pretmpfilename):
2233     return False;
2234    tmpfilename = pretmpfilename['Filename'];
2235    downloadsize = os.path.getsize(tmpfilename);
2236    fulldatasize = 0;
2237    prevdownsize = 0;
2238    exec_time_start = time.time();
2239    with open(tmpfilename, 'rb') as ft:
2240     f = StringIO();
2241     while True:
2242      databytes = ft.read(buffersize[1]);
2243      if not databytes: break;
2244      datasize = len(databytes);
2245      fulldatasize = datasize + fulldatasize;
2246      percentage = "";
2247      if(downloadsize>0):
2248       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2249      downloaddiff = fulldatasize - prevdownsize;
2250      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2251      prevdownsize = fulldatasize;
2252      f.write(databytes);
2253     f.seek(0);
2254     fdata = f.getvalue();
2255     f.close();
2256     ft.close();
2257     os.remove(tmpfilename);
2258     exec_time_end = time.time();
2259     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2260    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2261   if(outfile=="-" and sys.version[0]>="3"):
2262    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2263    tmpfilename = pretmpfilename['Filename'];
2264    downloadsize = os.path.getsize(tmpfilename);
2265    fulldatasize = 0;
2266    prevdownsize = 0;
2267    exec_time_start = time.time();
2268    with open(tmpfilename, 'rb') as ft:
2269     f = BytesIO();
2270     while True:
2271      databytes = ft.read(buffersize[1]);
2272      if not databytes: break;
2273      datasize = len(databytes);
2274      fulldatasize = datasize + fulldatasize;
2275      percentage = "";
2276      if(downloadsize>0):
2277       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2278      downloaddiff = fulldatasize - prevdownsize;
2279      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2280      prevdownsize = fulldatasize;
2281      f.write(databytes);
2282     f.seek(0);
2283     fdata = f.getvalue();
2284     f.close();
2285     ft.close();
2286     os.remove(tmpfilename);
2287     exec_time_end = time.time();
2288     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2289    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2290   return returnval;
2291
2292 if(not haveurllib3):
2293  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2294   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2295   return returnval;
2296
2297 if(havemechanize):
2298  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2299   global geturls_download_sleep, havebrotli;
2300   if(sleep<0):
2301    sleep = geturls_download_sleep;
2302   urlparts = urlparse.urlparse(httpurl);
2303   if(isinstance(httpheaders, list)):
2304    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2305   if(urlparts.username is not None or urlparts.password is not None):
2306    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2307    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2308   geturls_opener = mechanize.Browser();
2309   if(isinstance(httpheaders, dict)):
2310    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2311   time.sleep(sleep);
2312   geturls_opener.addheaders = httpheaders;
2313   geturls_opener.set_cookiejar(httpcookie);
2314   geturls_opener.set_handle_robots(False);
2315   if(postdata is not None and not isinstance(postdata, dict)):
2316    postdata = urlencode(postdata);
2317   try:
2318    if(httpmethod=="GET"):
2319     geturls_text = geturls_opener.open(httpurl);
2320    elif(httpmethod=="POST"):
2321     geturls_text = geturls_opener.open(httpurl, data=postdata);
2322    else:
2323     geturls_text = geturls_opener.open(httpurl);
2324   except mechanize.HTTPError as geturls_text_error:
2325    geturls_text = geturls_text_error;
2326    log.info("Error With URL "+httpurl);
2327   except URLError:
2328    log.info("Error With URL "+httpurl);
2329    return False;
2330   except socket.timeout:
2331    log.info("Error With URL "+httpurl);
2332    return False;
2333   log.info("Downloading URL "+httpurl);
2334   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2335    if(sys.version[0]=="2"):
2336     strbuf = StringIO(geturls_text.read());
2337    if(sys.version[0]>="3"):
2338     strbuf = BytesIO(geturls_text.read());
2339    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2340    returnval_content = gzstrbuf.read()[:];
2341   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2342    returnval_content = geturls_text.read()[:];
2343   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2344    returnval_content = brotli.decompress(returnval_content);
2345   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2346   geturls_text.close();
2347   return returnval;
2348
2349 if(not havemechanize):
2350  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2351   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2352   return returnval;
2353
2354 if(havemechanize):
2355  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2356   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2357   exec_time_start = time.time();
2358   myhash = hashlib.new("sha1");
2359   if(sys.version[0]=="2"):
2360    myhash.update(httpurl);
2361    myhash.update(str(buffersize));
2362    myhash.update(str(exec_time_start));
2363   if(sys.version[0]>="3"):
2364    myhash.update(httpurl.encode('utf-8'));
2365    myhash.update(str(buffersize).encode('utf-8'));
2366    myhash.update(str(exec_time_start).encode('utf-8'));
2367   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2368   if(sleep<0):
2369    sleep = geturls_download_sleep;
2370   urlparts = urlparse.urlparse(httpurl);
2371   if(isinstance(httpheaders, list)):
2372    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2373   if(urlparts.username is not None or urlparts.password is not None):
2374    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2375    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2376   geturls_opener = mechanize.Browser();
2377   if(isinstance(httpheaders, dict)):
2378    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2379   time.sleep(sleep);
2380   geturls_opener.addheaders = httpheaders;
2381   geturls_opener.set_cookiejar(httpcookie);
2382   geturls_opener.set_handle_robots(False);
2383   if(postdata is not None and not isinstance(postdata, dict)):
2384    postdata = urlencode(postdata);
2385   try:
2386    if(httpmethod=="GET"):
2387     geturls_text = geturls_opener.open(httpurl);
2388    elif(httpmethod=="POST"):
2389     geturls_text = geturls_opener.open(httpurl, data=postdata);
2390    else:
2391     geturls_text = geturls_opener.open(httpurl);
2392   except mechanize.HTTPError as geturls_text_error:
2393    geturls_text = geturls_text_error;
2394    log.info("Error With URL "+httpurl);
2395   except URLError:
2396    log.info("Error With URL "+httpurl);
2397    return False;
2398   except socket.timeout:
2399    log.info("Error With URL "+httpurl);
2400    return False;
2401   downloadsize = int(geturls_text.info().get('Content-Length'));
2402   if(downloadsize is not None):
2403    downloadsize = int(downloadsize);
2404   if downloadsize is None: downloadsize = 0;
2405   fulldatasize = 0;
2406   prevdownsize = 0;
2407   log.info("Downloading URL "+httpurl);
2408   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2409    tmpfilename = f.name;
2410    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2411    while True:
2412     databytes = geturls_text.read(buffersize);
2413     if not databytes: break;
2414     datasize = len(databytes);
2415     fulldatasize = datasize + fulldatasize;
2416     percentage = "";
2417     if(downloadsize>0):
2418      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2419     downloaddiff = fulldatasize - prevdownsize;
2420     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2421     prevdownsize = fulldatasize;
2422     f.write(databytes);
2423    f.close();
2424   geturls_text.close();
2425   exec_time_end = time.time();
2426   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2427   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2428   return returnval;
2429
2430 if(not havemechanize):
2431  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2432   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2433   return returnval;
2434
2435 if(havemechanize):
2436  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2437   global geturls_download_sleep;
2438   if(sleep<0):
2439    sleep = geturls_download_sleep;
2440   if(not outfile=="-"):
2441    outpath = outpath.rstrip(os.path.sep);
2442    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2443    if(not os.path.exists(outpath)):
2444     os.makedirs(outpath);
2445    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2446     return False;
2447    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2448     return False;
2449    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2450    if(not pretmpfilename):
2451     return False;
2452    tmpfilename = pretmpfilename['Filename'];
2453    downloadsize = os.path.getsize(tmpfilename);
2454    fulldatasize = 0;
2455    log.info("Moving file "+tmpfilename+" to "+filepath);
2456    exec_time_start = time.time();
2457    shutil.move(tmpfilename, filepath);
2458    exec_time_end = time.time();
2459    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2460    if(os.path.exists(tmpfilename)):
2461     os.remove(tmpfilename);
2462    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2463   if(outfile=="-" and sys.version[0]=="2"):
2464    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2465    if(not pretmpfilename):
2466     return False;
2467    tmpfilename = pretmpfilename['Filename'];
2468    downloadsize = os.path.getsize(tmpfilename);
2469    fulldatasize = 0;
2470    prevdownsize = 0;
2471    exec_time_start = time.time();
2472    with open(tmpfilename, 'rb') as ft:
2473     f = StringIO();
2474     while True:
2475      databytes = ft.read(buffersize[1]);
2476      if not databytes: break;
2477      datasize = len(databytes);
2478      fulldatasize = datasize + fulldatasize;
2479      percentage = "";
2480      if(downloadsize>0):
2481       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2482      downloaddiff = fulldatasize - prevdownsize;
2483      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2484      prevdownsize = fulldatasize;
2485      f.write(databytes);
2486     f.seek(0);
2487     fdata = f.getvalue();
2488     f.close();
2489     ft.close();
2490     os.remove(tmpfilename);
2491     exec_time_end = time.time();
2492     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2493    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2494   if(outfile=="-" and sys.version[0]>="3"):
2495    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2496    tmpfilename = pretmpfilename['Filename'];
2497    downloadsize = os.path.getsize(tmpfilename);
2498    fulldatasize = 0;
2499    prevdownsize = 0;
2500    exec_time_start = time.time();
2501    with open(tmpfilename, 'rb') as ft:
2502     f = BytesIO();
2503     while True:
2504      databytes = ft.read(buffersize[1]);
2505      if not databytes: break;
2506      datasize = len(databytes);
2507      fulldatasize = datasize + fulldatasize;
2508      percentage = "";
2509      if(downloadsize>0):
2510       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2511      downloaddiff = fulldatasize - prevdownsize;
2512      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2513      prevdownsize = fulldatasize;
2514      f.write(databytes);
2515     f.seek(0);
2516     fdata = f.getvalue();
2517     f.close();
2518     ft.close();
2519     os.remove(tmpfilename);
2520     exec_time_end = time.time();
2521     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2522    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2523   return returnval;
2524
2525 if(not havemechanize):
2526  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2527   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2528   return returnval;
2529
2530 def download_file_from_ftp_file(url):
2531  urlparts = urlparse.urlparse(url);
2532  file_name = os.path.basename(urlparts.path);
2533  file_dir = os.path.dirname(urlparts.path);
2534  if(urlparts.username is not None):
2535   ftp_username = urlparts.username;
2536  else:
2537   ftp_username = "anonymous";
2538  if(urlparts.password is not None):
2539   ftp_password = urlparts.password;
2540  elif(urlparts.password is None and urlparts.username=="anonymous"):
2541   ftp_password = "anonymous";
2542  else:
2543   ftp_password = "";
2544  if(urlparts.scheme=="ftp"):
2545   ftp = FTP();
2546  elif(urlparts.scheme=="ftps"):
2547   ftp = FTP_TLS();
2548  else:
2549   return False;
2550  ftp.connect(urlparts.hostname, urlparts.port);
2551  ftp.login(urlparts.username, urlparts.password);
2552  if(urlparts.scheme=="ftps"):
2553   ftp.prot_p();
2554  ftpfile = BytesIO();
2555  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2556  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2557  ftp.close();
2558  ftpfile.seek(0, 0);
2559  return ftpfile;
2560
2561 def download_file_from_ftp_string(url):
2562  ftpfile = download_file_from_ftp_file(url);
2563  return ftpfile.read();
2564
2565 def upload_file_to_ftp_file(ftpfile, url):
2566  urlparts = urlparse.urlparse(url);
2567  file_name = os.path.basename(urlparts.path);
2568  file_dir = os.path.dirname(urlparts.path);
2569  if(urlparts.username is not None):
2570   ftp_username = urlparts.username;
2571  else:
2572   ftp_username = "anonymous";
2573  if(urlparts.password is not None):
2574   ftp_password = urlparts.password;
2575  elif(urlparts.password is None and urlparts.username=="anonymous"):
2576   ftp_password = "anonymous";
2577  else:
2578   ftp_password = "";
2579  if(urlparts.scheme=="ftp"):
2580   ftp = FTP();
2581  elif(urlparts.scheme=="ftps"):
2582   ftp = FTP_TLS();
2583  else:
2584   return False;
2585  ftp.connect(urlparts.hostname, urlparts.port);
2586  ftp.login(urlparts.username, urlparts.password);
2587  if(urlparts.scheme=="ftps"):
2588   ftp.prot_p();
2589  ftp.storbinary("STOR "+urlparts.path, ftpfile);
2590  ftp.close();
2591  ftpfile.seek(0, 0);
2592  return ftpfile;
2593
2594 def upload_file_to_ftp_string(ftpstring, url):
2595  ftpfileo = BytesIO(ftpstring);
2596  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2597  ftpfileo.close();
2598  return ftpfile;
2599
2600 if(haveparamiko):
2601  def download_file_from_sftp_file(url):
2602   urlparts = urlparse.urlparse(url);
2603   file_name = os.path.basename(urlparts.path);
2604   file_dir = os.path.dirname(urlparts.path);
2605   sftp_port = urlparts.port;
2606   if(urlparts.port is None):
2607    sftp_port = 22;
2608   else:
2609    sftp_port = urlparts.port;
2610   if(urlparts.username is not None):
2611    sftp_username = urlparts.username;
2612   else:
2613    sftp_username = "anonymous";
2614   if(urlparts.password is not None):
2615    sftp_password = urlparts.password;
2616   elif(urlparts.password is None and urlparts.username=="anonymous"):
2617    sftp_password = "anonymous";
2618   else:
2619    sftp_password = "";
2620   if(urlparts.scheme!="sftp"):
2621    return False;
2622   ssh = paramiko.SSHClient();
2623   ssh.load_system_host_keys();
2624   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2625   try:
2626    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2627   except paramiko.ssh_exception.SSHException:
2628    return False;
2629   sftp = ssh.open_sftp();
2630   sftpfile = BytesIO();
2631   sftp.getfo(urlparts.path, sftpfile);
2632   sftp.close();
2633   ssh.close();
2634   sftpfile.seek(0, 0);
2635   return sftpfile;
2636 else:
2637  def download_file_from_sftp_file(url):
2638   return False;
2639
2640 if(haveparamiko):
2641  def download_file_from_sftp_string(url):
2642   sftpfile = download_file_from_sftp_file(url);
2643   return sftpfile.read();
2644 else:
2645  def download_file_from_ftp_string(url):
2646   return False;
2647
2648 if(haveparamiko):
2649  def upload_file_to_sftp_file(sftpfile, url):
2650   urlparts = urlparse.urlparse(url);
2651   file_name = os.path.basename(urlparts.path);
2652   file_dir = os.path.dirname(urlparts.path);
2653   sftp_port = urlparts.port;
2654   if(urlparts.port is None):
2655    sftp_port = 22;
2656   else:
2657    sftp_port = urlparts.port;
2658   if(urlparts.username is not None):
2659    sftp_username = urlparts.username;
2660   else:
2661    sftp_username = "anonymous";
2662   if(urlparts.password is not None):
2663    sftp_password = urlparts.password;
2664   elif(urlparts.password is None and urlparts.username=="anonymous"):
2665    sftp_password = "anonymous";
2666   else:
2667    sftp_password = "";
2668   if(urlparts.scheme!="sftp"):
2669    return False;
2670   ssh = paramiko.SSHClient();
2671   ssh.load_system_host_keys();
2672   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2673   try:
2674    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2675   except paramiko.ssh_exception.SSHException:
2676    return False;
2677   sftp = ssh.open_sftp();
2678   sftp.putfo(sftpfile, urlparts.path);
2679   sftp.close();
2680   ssh.close();
2681   sftpfile.seek(0, 0);
2682   return sftpfile;
2683 else:
2684  def upload_file_to_sftp_file(sftpfile, url):
2685   return False;
2686
2687 if(haveparamiko):
2688  def upload_file_to_sftp_string(sftpstring, url):
2689   sftpfileo = BytesIO(sftpstring);
2690   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2691   sftpfileo.close();
2692   return sftpfile;
2693 else:
2694  def upload_file_to_sftp_string(url):
2695   return False;