pymotherless.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2016 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2016 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: pymotherless.py - Last Update: 6/16/2016 Ver. 0.4.7 RC 1 - Author: cooldude2k $
  17 '''
  18
  19 from __future__ import division, absolute_import, print_function;
  20 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, imp;
  21 import logging as log;
  22 haverequests = False;
  23 try:
  24  imp.find_module('requests');
  25  haverequests = True;
  26  import requests;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  imp.find_module('mechanize');
  32  havemechanize = True;
  33  import mechanize;
  34 except ImportError:
  35  havemechanize = False;
  36 if(sys.version[0]=="2"):
  37  try:
  38   from cStringIO import StringIO;
  39  except ImportError:
  40   from StringIO import StringIO;
  41  # From http://python-future.org/compatible_idioms.html
  42  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  43  from urllib import urlencode;
  44  from urllib2 import urlopen, Request, HTTPError;
  45  import urllib2, urlparse, cookielib;
  46 if(sys.version[0]>="3"):
  47  from io import StringIO, BytesIO;
  48  # From http://python-future.org/compatible_idioms.html
  49  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  50  from urllib.request import urlopen, Request;
  51  from urllib.error import HTTPError;
  52  import urllib.request as urllib2;
  53  import urllib.parse as urlparse;
  54  import http.cookiejar as cookielib;
  55
  56 __program_name__ = "PyMotherless";
  57 __project__ = __program_name__;
  58 __project_url__ = "https://github.com/GameMaker2k/PyMotherless";
  59 __version_info__ = (0, 4, 7, "RC 1", 1);
  60 __version_date_info__ = (2016, 6, 16, "RC 1", 1);
  61 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  62 if(__version_info__[4]!=None):
  63  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  64 if(__version_info__[4]==None):
  65  __version_date_plusrc__ = __version_date__;
  66 if(__version_info__[3]!=None):
  67  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  68 if(__version_info__[3]==None):
  69  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  70
  71 geturls_cj = cookielib.CookieJar();
  72 geturls_ua_firefox_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:44.0) Gecko/20100101 Firefox/44.0";
  73 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:42.0) Gecko/20100101 Firefox/42.0 SeaMonkey/2.39";
  74 geturls_ua_chrome_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
  75 geturls_ua_chromium_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chromium/48.0.2564.116 Chrome/48.0.2564.116 Safari/537.36";
  76 geturls_ua_midori_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/538.15 (KHTML, like Gecko) Chrome/18.0.1025.133 Safari/538.15 Midori/0.5";
  77 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko";
  78 geturls_ua_pymotherless_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
  79 if(platform.python_implementation()!=""):
  80  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=platform.python_implementation(), pyver=platform.python_version(), proname=__project__, prover=__version__);
  81 if(platform.python_implementation()==""):
  82  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp="Python", pyver=platform.python_version(), proname=__project__, prover=__version__);
  83 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
  84 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
  85 geturls_ua = geturls_ua_firefox_windows7;
  86 geturls_headers_firefox_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  87 geturls_headers_seamonkey_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  88 geturls_headers_chrome_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  89 geturls_headers_chromium_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  90 geturls_headers_midori_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_midori_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  91 geturls_headers_internet_explorer_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  92 geturls_headers_pymotherless_python = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  93 geturls_headers_pymotherless_python_alt = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python_alt, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  94 geturls_headers_googlebot_google = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  95 geturls_headers_googlebot_google_old = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  96 geturls_headers = geturls_headers_firefox_windows7;
  97 geturls_download_sleep = 0;
  98
  99 def add_url_param(url, **params):
 100  n=3;
 101  parts = list(urlparse.urlsplit(url));
 102  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 103  d.update(params);
 104  parts[n]=urlencode(d);
 105  return urlparse.urlunsplit(parts);
 106
 107 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 108 def which_exec(execfile):
 109  for path in os.environ["PATH"].split(":"):
 110   if os.path.exists(path + "/" + execfile):
 111    return path + "/" + execfile;
 112
 113 def listize(varlist):
 114  il = 0;
 115  ix = len(varlist);
 116  ilx = 1;
 117  newlistreg = {};
 118  newlistrev = {};
 119  newlistfull = {};
 120  while(il < ix):
 121   newlistreg.update({ilx: varlist[il]});
 122   newlistrev.update({varlist[il]: ilx});
 123   ilx = ilx + 1;
 124   il = il + 1;
 125  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 126  return newlistfull;
 127
 128 def twolistize(varlist):
 129  il = 0;
 130  ix = len(varlist);
 131  ilx = 1;
 132  newlistnamereg = {};
 133  newlistnamerev = {};
 134  newlistdescreg = {};
 135  newlistdescrev = {};
 136  newlistfull = {};
 137  while(il < ix):
 138   newlistnamereg.update({ilx: varlist[il][0].strip()});
 139   newlistnamerev.update({varlist[il][0].strip(): ilx});
 140   newlistdescreg.update({ilx: varlist[il][1].strip()});
 141   newlistdescrev.update({varlist[il][1].strip(): ilx});
 142   ilx = ilx + 1;
 143   il = il + 1;
 144  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 145  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 146  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 147  return newlistfull;
 148
 149 def arglistize(proexec, *varlist):
 150  il = 0;
 151  ix = len(varlist);
 152  ilx = 1;
 153  newarglist = [proexec];
 154  while(il < ix):
 155   if varlist[il][0] is not None:
 156    newarglist.append(varlist[il][0]);
 157   if varlist[il][1] is not None:
 158    newarglist.append(varlist[il][1]);
 159   il = il + 1;
 160  return newarglist;
 161
 162 # hms_string by ArcGIS Python Recipes
 163 # https://arcpy.wordpress.com/2012/04/20/146/
 164 def hms_string(sec_elapsed):
 165  h = int(sec_elapsed / (60 * 60));
 166  m = int((sec_elapsed % (60 * 60)) / 60);
 167  s = sec_elapsed % 60.0;
 168  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 169
 170 # get_readable_size by Lipis
 171 # http://stackoverflow.com/posts/14998888/revisions
 172 def get_readable_size(bytes, precision=1, unit="IEC"):
 173  unit = unit.upper();
 174  if(unit!="IEC" and unit!="SI"):
 175   unit = "IEC";
 176  if(unit=="IEC"):
 177   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 178   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 179   unitsize = 1024.0;
 180  if(unit=="SI"):
 181   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 182   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 183   unitsize = 1000.0;
 184  return_val = {};
 185  orgbytes = bytes;
 186  for unit in units:
 187   if abs(bytes) < unitsize:
 188    strformat = "%3."+str(precision)+"f%s";
 189    pre_return_val = (strformat % (bytes, unit));
 190    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 191    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 192    alt_return_val = pre_return_val.split();
 193    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 194    return return_val;
 195   bytes /= unitsize;
 196  strformat = "%."+str(precision)+"f%s";
 197  pre_return_val = (strformat % (bytes, "YiB"));
 198  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 199  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 200  alt_return_val = pre_return_val.split();
 201  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 202  return return_val;
 203
 204 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 205  unit = unit.upper();
 206  usehashtypes = usehashtypes.lower();
 207  getfilesize = os.path.getsize(infile);
 208  return_val = get_readable_size(getfilesize, precision, unit);
 209  if(usehashes==True):
 210   hashtypelist = usehashtypes.split(",");
 211   openfile = open(infile, "rb");
 212   filecontents = openfile.read();
 213   openfile.close();
 214   listnumcount = 0;
 215   listnumend = len(hashtypelist);
 216   while(listnumcount < listnumend):
 217    hashtypelistlow = hashtypelist[listnumcount].strip();
 218    hashtypelistup = hashtypelistlow.upper();
 219    filehash = hashlib.new(hashtypelistup);
 220    filehash.update(filecontents);
 221    filegethash = filehash.hexdigest();
 222    return_val.update({hashtypelistup: filegethash});
 223    listnumcount += 1;
 224  return return_val;
 225
 226 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 227  unit = unit.upper();
 228  usehashtypes = usehashtypes.lower();
 229  getfilesize = len(instring);
 230  return_val = get_readable_size(getfilesize, precision, unit);
 231  if(usehashes==True):
 232   hashtypelist = usehashtypes.split(",");
 233   listnumcount = 0;
 234   listnumend = len(hashtypelist);
 235   while(listnumcount < listnumend):
 236    hashtypelistlow = hashtypelist[listnumcount].strip();
 237    hashtypelistup = hashtypelistlow.upper();
 238    filehash = hashlib.new(hashtypelistup);
 239    if(sys.version[0]=="2"):
 240     filehash.update(instring);
 241    if(sys.version[0]>="3"):
 242     filehash.update(instring.encode('utf-8'));
 243    filegethash = filehash.hexdigest();
 244    return_val.update({hashtypelistup: filegethash});
 245    listnumcount += 1;
 246  return return_val;
 247
 248 def make_http_headers_from_dict_to_list(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 249  if isinstance(headers, dict):
 250   returnval = [];
 251   if(sys.version[0]=="2"):
 252    for headkey, headvalue in headers.iteritems():
 253     returnval.append((headkey, headvalue));
 254   if(sys.version[0]>="3"):
 255    for headkey, headvalue in headers.items():
 256     returnval.append((headkey, headvalue));
 257  elif isinstance(headers, list):
 258   returnval = headers;
 259  else:
 260   returnval = False;
 261  return returnval;
 262
 263 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 264  if isinstance(headers, dict):
 265   returnval = [];
 266   if(sys.version[0]=="2"):
 267    for headkey, headvalue in headers.iteritems():
 268     returnval.append(headkey+": "+headvalue);
 269   if(sys.version[0]>="3"):
 270    for headkey, headvalue in headers.items():
 271     returnval.append(headkey+": "+headvalue);
 272  elif isinstance(headers, list):
 273   returnval = headers;
 274  else:
 275   returnval = False;
 276  return returnval;
 277
 278 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://motherless.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 279  if isinstance(headers, list):
 280   returnval = {};
 281   mli = 0;
 282   mlil = len(headers);
 283   while(mli<mlil):
 284    returnval.update({headers[mli][0]: headers[mli][1]});
 285    mli = mli + 1;
 286  elif isinstance(headers, dict):
 287   returnval = headers;
 288  else:
 289   returnval = False;
 290  return returnval;
 291
 292 def get_httplib_support(checkvalue=None):
 293  global haverequests, havemechanize;
 294  returnval = ();
 295  returnval.append("urllib");
 296  if(haverequests==True):
 297   returnval.append("requests");
 298  if(havemechanize==True):
 299   returnval.append("mechanize");
 300  if(not checkvalue==None):
 301   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 302    checkvalue = "urllib";
 303   if(checkvalue in returnval):
 304    returnval = True;
 305   else:
 306    returnval = False;
 307  return returnval;
 308
 309 def check_httplib_support(checkvalue="urllib"):
 310  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 311   checkvalue = "urllib";
 312  returnval = get_httplib_support(checkvalue);
 313  return returnval;
 314
 315 def download_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1):
 316  global geturls_download_sleep, haverequests, havemechanize;
 317  if(sleep<0):
 318   sleep = geturls_download_sleep;
 319  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 320   httplibuse = "urllib";
 321  if(haverequests==False and httplibuse=="requests"):
 322   httplibuse = "urllib";
 323  if(havemechanize==False and httplibuse=="mechanize"):
 324   httplibuse = "urllib";
 325  if(httplibuse=="urllib"):
 326   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep);
 327  elif(httplibuse=="requests"):
 328   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep);
 329  elif(httplibuse=="mechanize"):
 330   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep);
 331  else:
 332   returnval = False;
 333  return returnval;
 334
 335 def download_from_url_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", buffersize=524288, sleep=-1):
 336  global geturls_download_sleep, haverequests, havemechanize;
 337  if(sleep<0):
 338   sleep = geturls_download_sleep;
 339  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 340   httplibuse = "urllib";
 341  if(haverequests==False and httplibuse=="requests"):
 342   httplibuse = "urllib";
 343  if(havemechanize==False and httplibuse=="mechanize"):
 344   httplibuse = "urllib";
 345  if(httplibuse=="urllib"):
 346   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep);
 347  elif(httplibuse=="requests"):
 348   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize, sleep);
 349  elif(httplibuse=="mechanize"):
 350   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize, sleep);
 351  else:
 352   returnval = False;
 353  return returnval;
 354
 355 def download_from_url_to_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 356  global geturls_download_sleep, haverequests, havemechanize;
 357  if(sleep<0):
 358   sleep = geturls_download_sleep;
 359  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 360   httplibuse = "urllib";
 361  if(haverequests==False and httplibuse=="requests"):
 362   httplibuse = "urllib";
 363  if(havemechanize==False and httplibuse=="mechanize"):
 364   httplibuse = "urllib";
 365  if(httplibuse=="urllib"):
 366   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 367  elif(httplibuse=="requests"):
 368   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 369  elif(httplibuse=="mechanize"):
 370   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 371  else:
 372   returnval = False;
 373  return returnval;
 374
 375 def download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep=-1):
 376  global geturls_download_sleep;
 377  if(sleep<0):
 378   sleep = geturls_download_sleep;
 379  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 380  if(isinstance(httpheaders, dict)):
 381   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 382  geturls_opener.addheaders = httpheaders;
 383  time.sleep(sleep);
 384  geturls_text = geturls_opener.open(httpurl);
 385  log.info("Downloading URL "+httpurl);
 386  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 387   if(sys.version[0]=="2"):
 388    strbuf = StringIO(geturls_text.read());
 389   if(sys.version[0]>="3"):
 390    strbuf = BytesIO(geturls_text.read());
 391   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 392   returnval_content = gzstrbuf.read()[:];
 393  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 394   returnval_content = geturls_text.read()[:];
 395  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 396  geturls_text.close();
 397  return returnval;
 398
 399 def download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize=524288, sleep=-1):
 400  global geturls_download_sleep;
 401  exec_time_start = time.time();
 402  if(sleep<0):
 403   sleep = geturls_download_sleep;
 404  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 405  if(isinstance(httpheaders, dict)):
 406   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 407  geturls_opener.addheaders = httpheaders;
 408  time.sleep(sleep);
 409  geturls_text = geturls_opener.open(httpurl);
 410  downloadsize = geturls_text.info().get('Content-Length');
 411  if(downloadsize is not None):
 412   downloadsize = int(downloadsize);
 413  if downloadsize is None: downloadsize = 0;
 414  fulldatasize = 0;
 415  prevdownsize = 0;
 416  log.info("Downloading URL "+httpurl);
 417  with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 418   tmpfilename = f.name;
 419   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 420   while True:
 421    databytes = geturls_text.read(buffersize);
 422    if not databytes: break;
 423    datasize = len(databytes);
 424    fulldatasize = datasize + fulldatasize;
 425    percentage = "";
 426    if(downloadsize>0):
 427     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 428    downloaddiff = fulldatasize - prevdownsize;
 429    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 430    prevdownsize = fulldatasize;
 431    f.write(databytes);
 432   f.close();
 433  geturls_text.close();
 434  exec_time_end = time.time();
 435  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 436  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 437  return returnval;
 438
 439 def download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 440  global geturls_download_sleep;
 441  if(sleep<0):
 442   sleep = geturls_download_sleep;
 443  if(not outfile=="-"):
 444   outpath = outpath.rstrip(os.path.sep);
 445   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 446   if(not os.path.exists(outpath)):
 447    os.makedirs(outpath);
 448   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 449    return False;
 450   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 451    return False;
 452   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 453   tmpfilename = pretmpfilename['Filename'];
 454   downloadsize = os.path.getsize(tmpfilename);
 455   fulldatasize = 0;
 456   log.info("Moving file "+tmpfilename+" to "+filepath);
 457   exec_time_start = time.time();
 458   shutil.move(tmpfilename, filepath);
 459   exec_time_end = time.time();
 460   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 461   if(os.path.exists(tmpfilename)==True):
 462    os.remove(tmpfilename);
 463   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 464  if(outfile=="-" and sys.version[0]=="2"):
 465   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 466   tmpfilename = pretmpfilename['Filename'];
 467   downloadsize = os.path.getsize(tmpfilename);
 468   fulldatasize = 0;
 469   prevdownsize = 0;
 470   exec_time_start = time.time();
 471   with open(tmpfilename, 'rb') as ft:
 472    f = StringIO();
 473    while True:
 474     databytes = ft.read(buffersize[1]);
 475     if not databytes: break;
 476     datasize = len(databytes);
 477     fulldatasize = datasize + fulldatasize;
 478     percentage = "";
 479     if(downloadsize>0):
 480      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 481     downloaddiff = fulldatasize - prevdownsize;
 482     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 483     prevdownsize = fulldatasize;
 484     f.write(databytes);
 485    f.seek(0);
 486    fdata = f.getvalue();
 487    f.close();
 488    ft.close();
 489    os.remove(tmpfilename);
 490    exec_time_end = time.time();
 491    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 492   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 493  if(outfile=="-" and sys.version[0]>="3"):
 494   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 495   tmpfilename = pretmpfilename['Filename'];
 496   downloadsize = os.path.getsize(tmpfilename);
 497   fulldatasize = 0;
 498   prevdownsize = 0;
 499   exec_time_start = time.time();
 500   with open(tmpfilename, 'rb') as ft:
 501    f = BytesIO();
 502    while True:
 503     databytes = ft.read(buffersize[1]);
 504     if not databytes: break;
 505     datasize = len(databytes);
 506     fulldatasize = datasize + fulldatasize;
 507     percentage = "";
 508     if(downloadsize>0):
 509      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 510     downloaddiff = fulldatasize - prevdownsize;
 511     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 512     prevdownsize = fulldatasize;
 513     f.write(databytes);
 514    f.seek(0);
 515    fdata = f.getvalue();
 516    f.close();
 517    ft.close();
 518    os.remove(tmpfilename);
 519    exec_time_end = time.time();
 520    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 521   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 522  return returnval;
 523
 524 if(haverequests==True):
 525  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 526   global geturls_download_sleep;
 527   if(sleep<0):
 528    sleep = geturls_download_sleep;
 529   if(isinstance(httpheaders, list)):
 530    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 531   time.sleep(sleep);
 532   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
 533   log.info("Downloading URL "+httpurl);
 534   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
 535    if(sys.version[0]=="2"):
 536     strbuf = StringIO(geturls_text.content);
 537    if(sys.version[0]>="3"):
 538     strbuf = BytesIO(geturls_text.content);
 539    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 540    returnval_content = gzstrbuf.content[:];
 541   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
 542    returnval_content = geturls_text.content[:];
 543   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 544   geturls_text.close();
 545   return returnval;
 546
 547 if(haverequests==False):
 548  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 549   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 550   return returnval;
 551
 552 if(haverequests==True):
 553  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=524288, sleep=-1):
 554   global geturls_download_sleep;
 555   exec_time_start = time.time();
 556   if(sleep<0):
 557    sleep = geturls_download_sleep;
 558   if(isinstance(httpheaders, list)):
 559    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 560   time.sleep(sleep);
 561   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
 562   downloadsize = int(geturls_text.headers.get('Content-Length'));
 563   if(downloadsize is not None):
 564    downloadsize = int(downloadsize);
 565   if downloadsize is None: downloadsize = 0;
 566   fulldatasize = 0;
 567   prevdownsize = 0;
 568   log.info("Downloading URL "+httpurl);
 569   with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 570    tmpfilename = f.name;
 571    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 572    for databytes in geturls_text.iter_content(chunk_size=buffersize):
 573     datasize = len(databytes);
 574     fulldatasize = datasize + fulldatasize;
 575     percentage = "";
 576     if(downloadsize>0):
 577      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 578     downloaddiff = fulldatasize - prevdownsize;
 579     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 580     prevdownsize = fulldatasize;
 581     f.write(databytes);
 582    f.close();
 583   geturls_text.close();
 584   exec_time_end = time.time();
 585   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 586   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 587   return returnval;
 588
 589 if(haverequests==False):
 590  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=524288, sleep=-1):
 591   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 592   return returnval;
 593
 594 if(haverequests==True):
 595  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 596   global geturls_download_sleep;
 597   if(sleep<0):
 598    sleep = geturls_download_sleep;
 599   if(not outfile=="-"):
 600    outpath = outpath.rstrip(os.path.sep);
 601    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 602    if(not os.path.exists(outpath)):
 603     os.makedirs(outpath);
 604    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 605     return False;
 606    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 607     return False;
 608    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 609    tmpfilename = pretmpfilename['Filename'];
 610    downloadsize = os.path.getsize(tmpfilename);
 611    fulldatasize = 0;
 612    log.info("Moving file "+tmpfilename+" to "+filepath);
 613    exec_time_start = time.time();
 614    shutil.move(tmpfilename, filepath);
 615    exec_time_end = time.time();
 616    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 617    if(os.path.exists(tmpfilename)==True):
 618     os.remove(tmpfilename);
 619    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 620   if(outfile=="-" and sys.version[0]=="2"):
 621    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 622    tmpfilename = pretmpfilename['Filename'];
 623    downloadsize = os.path.getsize(tmpfilename);
 624    fulldatasize = 0;
 625    prevdownsize = 0;
 626    exec_time_start = time.time();
 627    with open(tmpfilename, 'rb') as ft:
 628     f = StringIO();
 629     while True:
 630      databytes = ft.read(buffersize[1]);
 631      if not databytes: break;
 632      datasize = len(databytes);
 633      fulldatasize = datasize + fulldatasize;
 634      percentage = "";
 635      if(downloadsize>0):
 636       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 637      downloaddiff = fulldatasize - prevdownsize;
 638      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 639      prevdownsize = fulldatasize;
 640      f.write(databytes);
 641     f.seek(0);
 642     fdata = f.getvalue();
 643     f.close();
 644     ft.close();
 645     os.remove(tmpfilename);
 646     exec_time_end = time.time();
 647     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 648    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 649   if(outfile=="-" and sys.version[0]>="3"):
 650    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 651    tmpfilename = pretmpfilename['Filename'];
 652    downloadsize = os.path.getsize(tmpfilename);
 653    fulldatasize = 0;
 654    prevdownsize = 0;
 655    exec_time_start = time.time();
 656    with open(tmpfilename, 'rb') as ft:
 657     f = BytesIO();
 658     while True:
 659      databytes = ft.read(buffersize[1]);
 660      if not databytes: break;
 661      datasize = len(databytes);
 662      fulldatasize = datasize + fulldatasize;
 663      percentage = "";
 664      if(downloadsize>0):
 665       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 666      downloaddiff = fulldatasize - prevdownsize;
 667      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 668      prevdownsize = fulldatasize;
 669      f.write(databytes);
 670     f.seek(0);
 671     fdata = f.getvalue();
 672     f.close();
 673     ft.close();
 674     os.remove(tmpfilename);
 675     exec_time_end = time.time();
 676     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 677    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 678   return returnval;
 679
 680 if(haverequests==False):
 681  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 682   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 683   return returnval;
 684
 685 if(havemechanize==True):
 686  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 687   global geturls_download_sleep;
 688   if(sleep<0):
 689    sleep = geturls_download_sleep;
 690   geturls_opener = mechanize.Browser();
 691   if(isinstance(httpheaders, dict)):
 692    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 693   time.sleep(sleep);
 694   geturls_opener.addheaders = httpheaders;
 695   geturls_opener.set_cookiejar(httpcookie);
 696   geturls_opener.set_handle_robots(False);
 697   geturls_text = geturls_opener.open(httpurl);
 698   log.info("Downloading URL "+httpurl);
 699   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 700    if(sys.version[0]=="2"):
 701     strbuf = StringIO(geturls_text.read());
 702    if(sys.version[0]>="3"):
 703     strbuf = BytesIO(geturls_text.read());
 704    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 705    returnval_content = gzstrbuf.read()[:];
 706   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 707    returnval_content = geturls_text.read()[:];
 708   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 709   geturls_text.close();
 710   return returnval;
 711
 712 if(havemechanize==False):
 713  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 714   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 715   return returnval;
 716
 717 if(havemechanize==True):
 718  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=524288, sleep=-1):
 719   global geturls_download_sleep;
 720   exec_time_start = time.time();
 721   if(sleep<0):
 722    sleep = geturls_download_sleep;
 723   geturls_opener = mechanize.Browser();
 724   if(isinstance(httpheaders, dict)):
 725    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 726   time.sleep(sleep);
 727   geturls_opener.addheaders = httpheaders;
 728   geturls_opener.set_cookiejar(httpcookie);
 729   geturls_opener.set_handle_robots(False);
 730   geturls_text = geturls_opener.open(httpurl);
 731   downloadsize = int(geturls_text.info().get('Content-Length'));
 732   if(downloadsize is not None):
 733    downloadsize = int(downloadsize);
 734   if downloadsize is None: downloadsize = 0;
 735   fulldatasize = 0;
 736   prevdownsize = 0;
 737   log.info("Downloading URL "+httpurl);
 738   with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 739    tmpfilename = f.name;
 740    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 741    while True:
 742     databytes = geturls_text.read(buffersize);
 743     if not databytes: break;
 744     datasize = len(databytes);
 745     fulldatasize = datasize + fulldatasize;
 746     percentage = "";
 747     if(downloadsize>0):
 748      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 749     downloaddiff = fulldatasize - prevdownsize;
 750     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 751     prevdownsize = fulldatasize;
 752     f.write(databytes);
 753    f.close();
 754   geturls_text.close();
 755   exec_time_end = time.time();
 756   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 757   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 758   return returnval;
 759
 760 if(havemechanize==False):
 761  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=524288, sleep=-1):
 762   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 763   return returnval;
 764
 765 if(havemechanize==True):
 766  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 767   global geturls_download_sleep;
 768   if(sleep<0):
 769    sleep = geturls_download_sleep;
 770   if(not outfile=="-"):
 771    outpath = outpath.rstrip(os.path.sep);
 772    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 773    if(not os.path.exists(outpath)):
 774     os.makedirs(outpath);
 775    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 776     return False;
 777    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 778     return False;
 779    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 780    tmpfilename = pretmpfilename['Filename'];
 781    downloadsize = os.path.getsize(tmpfilename);
 782    fulldatasize = 0;
 783    log.info("Moving file "+tmpfilename+" to "+filepath);
 784    exec_time_start = time.time();
 785    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 786    shutil.move(tmpfilename, filepath);
 787    if(os.path.exists(tmpfilename)==True):
 788     os.remove(tmpfilename);
 789    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 790   if(outfile=="-" and sys.version[0]=="2"):
 791    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 792    tmpfilename = pretmpfilename['Filename'];
 793    downloadsize = os.path.getsize(tmpfilename);
 794    fulldatasize = 0;
 795    prevdownsize = 0;
 796    exec_time_start = time.time();
 797    with open(tmpfilename, 'rb') as ft:
 798     f = StringIO();
 799     while True:
 800      databytes = ft.read(buffersize[1]);
 801      if not databytes: break;
 802      datasize = len(databytes);
 803      fulldatasize = datasize + fulldatasize;
 804      percentage = "";
 805      if(downloadsize>0):
 806       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 807      downloaddiff = fulldatasize - prevdownsize;
 808      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 809      prevdownsize = fulldatasize;
 810      f.write(databytes);
 811     f.seek(0);
 812     fdata = f.getvalue();
 813     f.close();
 814     ft.close();
 815     os.remove(tmpfilename);
 816     exec_time_end = time.time();
 817     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 818    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 819   if(outfile=="-" and sys.version[0]>="3"):
 820    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 821    tmpfilename = pretmpfilename['Filename'];
 822    downloadsize = os.path.getsize(tmpfilename);
 823    fulldatasize = 0;
 824    prevdownsize = 0;
 825    exec_time_start = time.time();
 826    with open(tmpfilename, 'rb') as ft:
 827     f = BytesIO();
 828     while True:
 829      databytes = ft.read(buffersize[1]);
 830      if not databytes: break;
 831      datasize = len(databytes);
 832      fulldatasize = datasize + fulldatasize;
 833      percentage = "";
 834      if(downloadsize>0):
 835       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 836      downloaddiff = fulldatasize - prevdownsize;
 837      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 838      prevdownsize = fulldatasize;
 839      f.write(databytes);
 840     f.seek(0);
 841     fdata = f.getvalue();
 842     f.close();
 843     ft.close();
 844     os.remove(tmpfilename);
 845     exec_time_end = time.time();
 846     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 847    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 848   return returnval;
 849
 850 if(havemechanize==False):
 851  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 852   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 853   return returnval;
 854
 855 def check_motherless_url(httpurl):
 856  returnval = {};
 857  httpurl.replace("https://", "http://");
 858  if(not httpurl.startswith("http")):
 859   httpurl = "http://"+httpurl;
 860  urlparts = urlsplit(httpurl);
 861  urltype = None;
 862  domaintype = None;
 863  if(urlparts.netloc=="motherless.com" or urlparts.netloc=="www.motherless.com"):
 864   urltype = "motherless-link";
 865   domaintype = "motherless";
 866  if(urlparts.netloc=="cdn.avatars.motherlessmedia.com" or urlparts.netloc=="cdn1.avatars.motherlessmedia.com" or urlparts.netloc=="cdn2.avatars.motherlessmedia.com" or urlparts.netloc=="cdn3.avatars.motherlessmedia.com" or urlparts.netloc=="cdn4.avatars.motherlessmedia.com"):
 867   urltype = "motherless-avatar";
 868   domaintype = "motherlessmedia";
 869  if(urlparts.netloc=="cdn.images.motherlessmedia.com" or urlparts.netloc=="cdn1.images.motherlessmedia.com" or urlparts.netloc=="cdn2.images.motherlessmedia.com" or urlparts.netloc=="cdn3.images.motherlessmedia.com" or urlparts.netloc=="cdn4.images.motherlessmedia.com"):
 870   urltype = "motherless-image";
 871   domaintype = "motherlessmedia";
 872  if(urlparts.netloc=="cdn.videos.motherlessmedia.com" or urlparts.netloc=="cdn1.videos.motherlessmedia.com" or urlparts.netloc=="cdn2.videos.motherlessmedia.com" or urlparts.netloc=="cdn3.videos.motherlessmedia.com" or urlparts.netloc=="cdn4.videos.motherlessmedia.com"):
 873   urltype = "motherless-video";
 874   domaintype = "motherlessmedia";
 875  if(urlparts.netloc=="cdn.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn1.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn2.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn3.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 876   urltype = "motherless-thumbnail";
 877   domaintype = "motherlessmedia";
 878  if(urltype==None or domaintype==None):
 879   return False;
 880  returnval.update({'url': httpurl, 'urltype': urltype, 'domaintype': domaintype, 'urlinfo': {'scheme': urlparts.scheme, 'netloc': urlparts.netloc, 'path': urlparts.path, 'query': urlparts.query, 'fragment': urlparts.fragment}});
 881  return returnval;
 882
 883 def fix_motherless_url(httpurl):
 884  returnval = check_motherless_url(httpurl)['url'];
 885  return returnval;
 886
 887 def get_motherless_link_type(httpurl):
 888  httpurl = fix_motherless_url(httpurl);
 889  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
 890  httpurl = re.sub(mregex_gettitle, "http://cdn.", httpurl);
 891  mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(httpurl).query);
 892  mlessvidid_parts = urlparse.urlparse(httpurl);
 893  mlessvidid = mlessvidid_parts.path.split("/");
 894  returnval = False;
 895  if(mlessvidid[1]=="videos" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 896   returnval = "gallery";
 897  if(mlessvidid[1]=="images" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 898   returnval = "gallery";
 899  if(mlessvidid[1]=="galleries" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 900   returnval = "gallery";
 901  if(mlessvidid[1]=="videos" and len(mlessvidid)==2):
 902   returnval = "sample-videos";
 903  if(mlessvidid[1]=="images" and len(mlessvidid)==2):
 904   returnval = "sample-images";
 905  if(mlessvidid[1]=="galleries" and len(mlessvidid)==2):
 906   returnval = "sample-galleries";
 907  if(mlessvidid[1]=="" and len(mlessvidid)==2):
 908   returnval = "sample";
 909  if(mlessvidid[1]=="groups" and len(mlessvidid)==2):
 910   returnval = "group";
 911  if(mlessvidid[1]=="groups" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 912   returnval = "group";
 913  if(mlessvidid[1]=="groups" and len(mlessvidid)==4 and mlessvidid[2]=="category"):
 914   returnval = "group";
 915  if(mlessvidid[1]=="live" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images")):
 916   returnval = "gallery";
 917  if(mlessvidid[1]=="u" and len(mlessvidid)==3):
 918   returnval = "gallery";
 919  if(mlessvidid[1]=="f" and len(mlessvidid)==4 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 920   returnval = "gallery";
 921  if(mlessvidid[1]=="galleries" and len(mlessvidid)==4 and mlessvidid[2]=="member"):
 922   returnval = "gallery";
 923  if(mlessvidid[1]=="galleries" and len(mlessvidid)==5 and mlessvidid[2]=="member" and (mlessvidid[4]=="created" or mlessvidid[4]=="viewed" or mlessvidid[4]=="favorited" or mlessvidid[4]=="commented")):
 924   returnval = "gallery";
 925  if(mlessvidid[1]=="gv" and len(mlessvidid)==3):
 926   returnval = "gallery";
 927  if(mlessvidid[1]=="gi" and len(mlessvidid)==3):
 928   returnval = "gallery";
 929  if(mlessvidid[1]=="gm" and len(mlessvidid)==3):
 930   returnval = "member";
 931  if(mlessvidid[1]=="term" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 932   returnval = "gallery";
 933  if(mlessvidid[1]=="g" and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 934   returnval = "link";
 935  if(mlessvidid[1]=="random" and len(mlessvidid)==3 and (mlessvidid[2]=="video" or mlessvidid[2]=="image")):
 936   returnval = "link";
 937  if(re.match("^V", mlessvidid[1]) and len(mlessvidid)==2):
 938   returnval = "board";
 939  if(re.match("^H", mlessvidid[1]) and len(mlessvidid)==2):
 940   returnval = "gallery";
 941  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==2):
 942   returnval = "gallery";
 943  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==3):
 944   returnval = "link";
 945  if(re.match("^g", mlessvidid[1]) and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 946   returnval = "link";
 947  if(mlessvidid[1]=="members" and len(mlessvidid)==2):
 948   returnval = "member";
 949  if(mlessvidid[1]=="members" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 950   returnval = "member";
 951  if(mlessvidid[1]=="members" and len(mlessvidid)==4 and (mlessvidid[2]=="uploader" or mlessvidid[2]=="viewed" or mlessvidid[2]=="social" or mlessvidid[2]=="favorited" or mlessvidid[2]=="commented" or mlessvidid[2]=="mentioned" or mlessvidid[2]=="verified")):
 952   returnval = "member";
 953  if(mlessvidid[1]=="girls" and len(mlessvidid)==2):
 954   returnval = "girls";
 955  if(mlessvidid[1]=="referers" and len(mlessvidid)==2):
 956   returnval = "referers";
 957  if(mlessvidid[1]=="about" and len(mlessvidid)==2):
 958   returnval = "team";
 959  if(mlessvidid_parts.netloc=="cdn.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.thumbs.motherlessmedia.com"):
 960   returnval = "download";
 961  if(mlessvidid_parts.netloc=="cdn1.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.thumbs.motherlessmedia.com"):
 962   returnval = "download";
 963  if(mlessvidid_parts.netloc=="cdn2.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.thumbs.motherlessmedia.com"):
 964   returnval = "download";
 965  if(mlessvidid_parts.netloc=="cdn3.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.thumbs.motherlessmedia.com"):
 966   returnval = "download";
 967  if(mlessvidid_parts.netloc=="cdn4.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 968   returnval = "download";
 969  if(returnval==False and len(mlessvidid)==2 and re.match("^([0-9A-F]+)$", mlessvidid[1])):
 970   returnval = "link";
 971  return returnval;
 972
 973 def get_motherless_link_type_alt(httpurl):
 974  httpurl = fix_motherless_url(httpurl);
 975  returnval = {'urlinfo': check_motherless_url(httpurl), 'motherlessinfo': get_motherless_link_type(httpurl)};
 976  return returnval;
 977
 978 def get_motherless_number_of_pages(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
 979  httpurl = fix_motherless_url(httpurl);
 980  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
 981  mrtext = premrtext['Content'];
 982  if(sys.version[0]>="3"):
 983   mrtext = mrtext.decode('ascii', 'replace');
 984  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
 985  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
 986  try:
 987   returnval = int(mlesspagenum[-1][0]);
 988  except:
 989   returnval = 1;
 990  return returnval;
 991
 992 def get_motherless_user_info(username, motherless_serv=None):
 993  motherless_subdomain = "cdn.";
 994  if(motherless_serv<5 or motherless_serv>0):
 995   motherless_subdomain = "cdn"+str(motherless_serv)+".";
 996  returnval = {'username': username};
 997  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg").path));
 998  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
 999  returnval.update({'orginurl': "http://motherless.com/m/"+username});
1000  returnval.update({'orginurltype': "profile"});
1001  returnval.update({'url': "http://motherless.com/m/"+username});
1002  returnval.update({'urltype': "profile"});
1003  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg"});
1004  returnval.update({'avatarfullfilename': avatarfilenameext});
1005  returnval.update({'avatarfilename': avatarfilename});
1006  returnval.update({'avatarextension': avatarfileextension});
1007  return returnval;
1008
1009 def get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", motherless_serv=None):
1010  httpurl = fix_motherless_url(httpurl);
1011  motherless_subdomain = "cdn.";
1012  if(motherless_serv<5 or motherless_serv>0):
1013   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1014  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1015  mrtext = premrtext['Content'];
1016  if(sys.version[0]>="3"):
1017   mrtext = mrtext.decode('ascii', 'replace');
1018  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
1019  mrtext = re.sub(mregex_gettitle, "http://cdn.", mrtext);
1020  mregex_gettitle = re.escape("<title>")+"(.*)"+re.escape(" - MOTHERLESS.COM</title>");
1021  mlesstitle = re.findall(mregex_gettitle, mrtext);
1022  mregex_geturlone = re.escape("__fileurl = '")+'?\'?([^"\'>]*)'+re.escape("';");
1023  mlesslinkone = re.findall(mregex_geturlone, mrtext);
1024  mregex_geturlonetype = re.escape("http://cdn.")+"(images|videos)"+re.escape(".motherlessmedia.com/")+"(images|videos)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1025  mlesslinkonetype = re.findall(mregex_geturlonetype, mrtext);
1026  mregex_geturltwo = re.escape("<meta property=\"og:image\" content=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1027  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1028  mregex_geturltwotype = re.escape("http://cdn.")+"(images|thumbs)"+re.escape(".motherlessmedia.com/")+"(images|thumbs)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1029  mlesslinktwotype = re.findall(mregex_geturltwotype, mrtext);
1030  filenameext = os.path.basename(urlparse.urljoin(mlesslinkone[0], urlparse.urlparse(mlesslinkone[0]).path));
1031  filename, fileextension = os.path.splitext(filenameext);
1032  thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[0], urlparse.urlparse(mlesslinktwo[0]).path));
1033  thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1034  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1035  mlessuname = re.findall(mregex_getuname, mrtext);
1036  mlessuname = mlessuname[0];
1037  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1038  mlessurlname = re.findall(mregex_geturlname, mrtext);
1039  mlessurlname = mlessurlname[0].replace("/m/", "");
1040  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1041  mlessavatar = re.findall(mregex_getavatar, mrtext);
1042  mlessavatar = mlessavatar[0];
1043  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar, urlparse.urlparse(mlessavatar).path));
1044  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1045  mregex_getviews = re.escape("<strong>Views</strong>\n")+"(\s+|\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1046  mlessviews = re.findall(mregex_getviews, mrtext);
1047  mregex_getfavorited = re.escape("<strong>Favorited</strong>\n")+"(\s+\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1048  mlessfavorited = re.findall(mregex_getfavorited, mrtext);
1049  if(mlesslinkonetype[0][1]=="images"):
1050   thumbnailaltpart = thumbfilename+"-zoom"+thumbfileextension;
1051   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1052   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1053   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1054  if(mlesslinkonetype[0][1]=="videos"):
1055   thumbnailaltpart = thumbfilename+"-small"+thumbfileextension;
1056   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1057   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1058   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1059  returnval = False;
1060  mlessurltype = get_motherless_link_type(mlesslinkone[0]);
1061  if(mlesslinkonetype[0][1]=="images"):
1062   returnval = {'type': mlesslinkonetype[0][1], 'urltype': mlessurltype, 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1063  if(mlesslinkonetype[0][1]=="videos"):
1064   returnval = {'type': mlesslinkonetype[0][1], 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1065  return returnval;
1066
1067 def get_motherless_links_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1068  httpurl = fix_motherless_url(httpurl);
1069  returnval = False;
1070  if(get_motherless_link_type(httpurl)=="download"):
1071   urlparts = urlparse.urlparse(httpurl);
1072   filewithext = os.path.split(urlparts.path);
1073   if(filewithext[0]=="/images"):
1074    filewithoutext = os.path.splitext(filewithext[1])[0];
1075    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1076   if(filewithext[0]=="/thumbs"):
1077    filewithoutext = os.path.splitext(filewithext[1])[0];
1078    filewithoutext = filewithoutext.replace("-zoom", "");
1079    filewithoutext = filewithoutext.replace("-small", "");
1080    filewithoutext = filewithoutext.replace("-strip", "");
1081    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1082  return returnval;
1083
1084 def get_motherless_external_links(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1085  httpurl = fix_motherless_url(httpurl);
1086  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1087  mrtext = premrtext['Content'];
1088  if(sys.version[0]>="3"):
1089   mrtext = mrtext.decode('ascii', 'replace');
1090  mregex_geturlinternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"motherless link\">");
1091  mlesslinkinternal = re.findall(mregex_geturlinternal, mrtext);
1092  mregex_geturlexternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"external link\">");
1093  mlesslinkexternal = re.findall(mregex_geturlexternal, mrtext);
1094  returnvalone = None;
1095  if(len(mlesslinkinternal)>1):
1096   mli = 0;
1097   mlil = len(mlesslinkinternal);
1098   returnvalone = {'numoflinks': mlil};
1099   returnvalone.update({'numofalllinks': len(mlesslinkinternal)});
1100   returnvalone.update({'orginurl': httpurl});
1101   returnvalone.update({'orginurltype': get_motherless_link_type(httpurl)});
1102   mlessrooturltype = get_motherless_link_type(httpurl);
1103   returnvalone.update({'urltype': mlessrooturltype});
1104   while(mli<mlil):
1105    mlessurltype = get_motherless_link_type(mlesslinkinternal[mli]);
1106    returnvalone.update({mli: {'urltype': mlessurltype, 'url': mlesslinkinternal[mli]} });
1107    mli = mli + 1;
1108  returnvaltwo = None;
1109  if(len(mlesslinkexternal)>1):
1110   mli = 0;
1111   mlil = len(mlesslinkexternal);
1112   returnvaltwo = {'numoflinks': mlil};
1113   returnvaltwo.update({'numofalllinks': len(mlesslinkexternal)});
1114   returnvaltwo.update({'orginurl': httpurl});
1115   returnvaltwo.update({'orginurltype': get_motherless_link_type(httpurl)});
1116   mlessrooturltype = get_motherless_link_type(httpurl);
1117   returnvaltwo.update({'urltype': mlessrooturltype});
1118   while(mli<mlil):
1119    returnvaltwo.update({mli: {'urltype': "external", 'url': mlesslinkexternal[mli]} });
1120    mli = mli + 1;
1121  if(returnvalone==None and returnvaltwo==None):
1122   returnval = {'internal': None, 'external': None};
1123  if(not returnvalone==None and not returnvaltwo==None):
1124   returnval = {'internal': returnvalone, 'external': returnvaltwo};
1125  if(not returnvalone==None and returnvaltwo==None):
1126   returnval = {'internal': returnvalone, 'external': None};
1127  if(returnvalone==None and not returnvaltwo==None):
1128   returnval = {'internal': None, 'external': returnvaltwo};
1129  return returnval;
1130
1131 def get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1], motherless_serv=None):
1132  httpurl = fix_motherless_url(httpurl);
1133  motherless_subdomain = "cdn.";
1134  if(motherless_serv<5 or motherless_serv>0):
1135   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1136  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1137  mrtext = premrtext['Content'];
1138  if(sys.version[0]>="3"):
1139   mrtext = mrtext.decode('ascii', 'replace');
1140  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1141  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1142  try:
1143   lastpage = int(mlesspagenum[-1][0]);
1144  except:
1145   lastpage = 1;
1146  if(page>lastpage):
1147   page = lastpage;
1148  httpurl = add_url_param(httpurl, page=str(page));
1149  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1150  mrtext = premrtext['Content'];
1151  if(sys.version[0]>="3"):
1152   mrtext = mrtext.decode('ascii', 'replace');
1153  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"img-container\" target=\"_self\">");
1154  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1155  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1156  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1157  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1158  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1159  mregex_geturltwo = re.escape("<img class=\"static\" src=\"")+'?\'?([^"\'>]*)'+re.escape("\" data-strip-src=\"")+'?\'?([^"\'>]*)'+re.escape("\" alt=\"")+'?\'?([^">]*)'+re.escape("\" />");
1160  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1161  mregex_getcaption = re.escape("<h2 class=\"caption title\">")+"(.*)"+re.escape("</h2>");
1162  mlesscaption = re.findall(mregex_getcaption, mrtext);
1163  mregex_gethits = re.escape("<div class=\"caption right\">")+"([0-9\,]+)"+re.escape(" Hits</div>");
1164  mlesshits = re.findall(mregex_gethits, mrtext);
1165  mregex_getuserinfo = re.escape("<a class=\"caption left\" href=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1166  mlessuname = re.findall(mregex_getuserinfo, mrtext);
1167  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1168   getlinks[1] = len(mlesslinkone);
1169  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1170   tmpgetlinks0 = getlinks[0];
1171   tmpgetlinks1 = getlinks[1];
1172   getlinks[0] = tmpgetlinks1;
1173   getlinks[1] = tmpgetlinks0;
1174  if(getlinks[0]<0):
1175   getlinks[0] = 0;
1176  mli = getlinks[0];
1177  mlil = getlinks[1];
1178  returnval = {'pages': lastpage};
1179  returnval.update({'curpage': page});
1180  returnval.update({'numoflinks': mlil});
1181  returnval.update({'numofalllinks': len(mlesslinkone)});
1182  returnval.update({'orginurl': httpurl});
1183  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1184  mlessrooturltype = get_motherless_link_type(httpurl);
1185  returnval.update({'urltype': mlessrooturltype});
1186  while(mli<mlil):
1187   mlessuname[mli] = mlessuname[mli].replace("/m/", "");
1188   stripfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1189   stripfilename, stripfileextension = os.path.splitext(stripfilenameext);
1190   thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][0], urlparse.urlparse(mlesslinktwo[mli][0]).path));
1191   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1192   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlesslinkone[mli]);
1193   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1194   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1195   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'strip': mlesslinktwo[mli][1], 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'stripfullfilename': stripfilenameext, 'stripfilename': stripfilename, 'stripextension': stripfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1196   mli = mli + 1;
1197  return returnval;
1198
1199 def get_motherless_random_links(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1200  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1201   tmpgetlinks0 = getlinks[0];
1202   tmpgetlinks1 = getlinks[1];
1203   getlinks[0] = tmpgetlinks1;
1204   getlinks[1] = tmpgetlinks0;
1205  if(getlinks[0]<0):
1206   getlinks[0] = 0;
1207  mli = getlinks[0];
1208  mlil = getlinks[1];
1209  if(linktype=="image"):
1210   returnval = {'pages': 1};
1211   returnval.update({'curpage': 1});
1212   returnval.update({'numoflinks': 80});
1213   returnval.update({'numofalllinks': mlil});
1214   returnval.update({'orginurl': "http://motherless.com/random/image"});
1215   returnval.update({'orginurltype': "gallery"});
1216   returnval.update({'urltype': "gallery"});
1217   while(mli<mlil):
1218    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1219    returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1220    mli = mli + 1;
1221  if(linktype=="video"):
1222   returnval = {'pages': 1};
1223   returnval.update({'curpage': 1});
1224   returnval.update({'numoflinks': 80});
1225   returnval.update({'numofalllinks': mlil});
1226   returnval.update({'orginurl': "http://motherless.com/random/video"});
1227   returnval.update({'orginurltype': "gallery"});
1228   returnval.update({'urltype': "gallery"});
1229   while(mli<mlil):
1230    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1231    returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1232    mli = mli + 1;
1233  return returnval;
1234
1235 def get_motherless_random_links_alt(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1236  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1237   tmpgetlinks0 = getlinks[0];
1238   tmpgetlinks1 = getlinks[1];
1239   getlinks[0] = tmpgetlinks1;
1240   getlinks[1] = tmpgetlinks0;
1241  if(getlinks[0]<0):
1242   getlinks[0] = 0;
1243  mli = getlinks[0];
1244  mlil = getlinks[1];
1245  if(linktype=="image"):
1246   returnval = {'pages': 1};
1247   returnval.update({'curpage': 1});
1248   returnval.update({'numoflinks': 80});
1249   returnval.update({'numofalllinks': mlil});
1250   returnval.update({'orginurl': "http://motherless.com/random/image"});
1251   returnval.update({'orginurltype': "gallery"});
1252   returnval.update({'urltype': "gallery"});
1253   while(mli<mlil):
1254    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1255    returnval.update({mli: get_links});
1256    mli = mli + 1;
1257  if(linktype=="video"):
1258   returnval = {'pages': 1};
1259   returnval.update({'curpage': 1});
1260   returnval.update({'numoflinks': 80});
1261   returnval.update({'numofalllinks': mlil});
1262   returnval.update({'orginurl': "http://motherless.com/random/video"});
1263   returnval.update({'orginurltype': "gallery"});
1264   returnval.update({'urltype': "gallery"});
1265   while(mli<mlil):
1266    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1267    returnval.update({mli: get_links});
1268    mli = mli + 1;
1269  return returnval;
1270
1271 def get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1], motherless_serv=None):
1272  httpurl = fix_motherless_url(httpurl);
1273  motherless_subdomain = "cdn.";
1274  if(motherless_serv<5 or motherless_serv>0):
1275   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1276  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1277  mrtext = premrtext['Content'];
1278  if(sys.version[0]>="3"):
1279   mrtext = mrtext.decode('ascii', 'replace');
1280  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1281  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1282  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1283  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1284  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" title=\"motherless link\">");
1285  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1286  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1287  mlessurlname = re.findall(mregex_geturlname, mrtext);
1288  mlessavaturl = "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessurlname[0][1]+"-avatar.jpg";
1289  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavaturl, urlparse.urlparse(mlessavaturl).path));
1290  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1291  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1292   getlinks[1] = len(mlesslinkone);
1293  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1294   tmpgetlinks0 = getlinks[0];
1295   tmpgetlinks1 = getlinks[1];
1296   getlinks[0] = tmpgetlinks1;
1297   getlinks[1] = tmpgetlinks0;
1298  if(getlinks[0]<0):
1299   getlinks[0] = 0;
1300  mli = getlinks[0];
1301  mlil = getlinks[1];
1302  returnval = {'numoflinks': mlil};
1303  returnval.update({'numofalllinks': len(mlesslinkone)});
1304  returnval.update({'orginurl': httpurl});
1305  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1306  mlessrooturltype = get_motherless_link_type(httpurl);
1307  returnval.update({'urltype': mlessrooturltype});
1308  returnval.update({'userinfo': get_motherless_user_info(mlessurlname[0][1])});
1309  returnval.update({'username': mlessurlname[0][1]});
1310  returnval.update({'avatarurl': mlessavaturl});
1311  returnval.update({'avatarfullfilename': avatarfilenameext});
1312  returnval.update({'avatarfilename': avatarfilename});
1313  returnval.update({'avatarextension': avatarfileextension});
1314  while(mli<mlil):
1315   mlessurltype = get_motherless_link_type("http://motherless.com"+mlesslinkone[mli]);
1316   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli]} });
1317   mli = mli + 1;
1318  return returnval;
1319
1320 def get_motherless_boards_posts(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1321  httpurl = fix_motherless_url(httpurl);
1322  motherless_subdomain = "cdn.";
1323  if(motherless_serv<5 or motherless_serv>0):
1324   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1325  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1326  mrtext = premrtext['Content'];
1327  if(sys.version[0]>="3"):
1328   mrtext = mrtext.decode('ascii', 'replace');
1329  mregex_getposts = "(\t+)"+re.escape("<p>")+"(.+?)"+re.escape("</p>")+"(\t+)";
1330  mlessposts = re.findall(mregex_getposts, mrtext);
1331  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1332  mlessuname = re.findall(mregex_getuname, mrtext);
1333  mregex_getopuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1334  mlessopuname = re.findall(mregex_getopuname, mrtext);
1335  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1336   getposts[1] = len(mlessposts);
1337  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1338   tmpgetposts0 = getposts[0];
1339   tmpgetposts1 = getposts[1];
1340   getposts[0] = tmpgetposts1;
1341   getposts[1] = tmpgetposts0;
1342  if(getposts[0]<0):
1343   getposts[0] = 0;
1344  mli = getposts[0];
1345  mlil = getposts[1];
1346  returnval = {'numofposts': mlil};
1347  returnval.update({'numofallposts': len(mlessposts)});
1348  returnval.update({'orginurl': httpurl});
1349  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1350  mlessrooturltype = get_motherless_link_type(httpurl);
1351  returnval.update({'urltype': mlessrooturltype});
1352  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0][1])});
1353  returnval.update({'username': mlessopuname[0][1]});
1354  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg").path));
1355  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1356  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg"});
1357  returnval.update({'avatarfullfilename': avatarfilenameext});
1358  returnval.update({'avatarfilename': avatarfilename});
1359  returnval.update({'avatarextension': avatarfileextension});
1360  while(mli<mlil):
1361   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1362   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1363   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1364   mli = mli + 1;
1365  return returnval;
1366
1367 def get_motherless_links_comments(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1368  httpurl = fix_motherless_url(httpurl);
1369  motherless_subdomain = "cdn.";
1370  if(motherless_serv<5 or motherless_serv>0):
1371   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1372  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1373  mrtext = premrtext['Content'];
1374  if(sys.version[0]>="3"):
1375   mrtext = mrtext.decode('ascii', 'replace');
1376  mregex_getposts = re.escape("<div style=\"text-align: justify;\">\n")+"(\t+)(.+?)"+re.escape("</div>");
1377  mlessposts = re.findall(mregex_getposts, mrtext);
1378  mregex_getuname = re.escape("<div class=\"media-comment\" id=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"0\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1379  mlessuname = re.findall(mregex_getuname, mrtext);
1380  mregex_getopuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1381  mlessopuname = re.findall(mregex_getopuname, mrtext);
1382  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1383   getposts[1] = len(mlessposts);
1384  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1385   tmpgetposts0 = getposts[0];
1386   tmpgetposts1 = getposts[1];
1387   getposts[0] = tmpgetposts1;
1388   getposts[1] = tmpgetposts0;
1389  if(getposts[0]<0):
1390   getposts[0] = 0;
1391  mli = getposts[0];
1392  mlil = getposts[1];
1393  returnval = {'numofposts': mlil};
1394  returnval.update({'numofallposts': len(mlessposts)});
1395  returnval.update({'orginurl': httpurl});
1396  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1397  mlessrooturltype = get_motherless_link_type(httpurl);
1398  returnval.update({'urltype': mlessrooturltype});
1399  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0])});
1400  returnval.update({'username': mlessopuname[0]});
1401  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg").path));
1402  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1403  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg"});
1404  returnval.update({'avatarfullfilename': avatarfilenameext});
1405  returnval.update({'avatarfilename': avatarfilename});
1406  returnval.update({'avatarextension': avatarfileextension});
1407  while(mli<mlil):
1408   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg").path));
1409   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1410   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1411   mli = mli + 1;
1412  return returnval;
1413
1414 def get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1415  httpurl = fix_motherless_url(httpurl);
1416  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1417  mrtext = premrtext['Content'];
1418  if(sys.version[0]>="3"):
1419   mrtext = mrtext.decode('ascii', 'replace');
1420  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1421  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1422  try:
1423   lastpage = int(mlesspagenum[-1][0]);
1424  except:
1425   lastpage = 1;
1426  if(page>lastpage):
1427   page = lastpage;
1428  httpurl = add_url_param(httpurl, page=str(page));
1429  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1430  mrtext = premrtext['Content'];
1431  if(sys.version[0]>="3"):
1432   mrtext = mrtext.decode('ascii', 'replace');
1433  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1434  mlessuname = re.findall(mregex_getuname, mrtext);
1435  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1436  mlessurlname = re.findall(mregex_geturlname, mrtext);
1437  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1438  mlessavatar = re.findall(mregex_getavatar, mrtext);
1439  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1440   getlinks[1] = len(mlessuname);
1441  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1442   tmpgetlinks0 = getlinks[0];
1443   tmpgetlinks1 = getlinks[1];
1444   getlinks[0] = tmpgetlinks1;
1445   getlinks[1] = tmpgetlinks0;
1446  if(getlinks[0]<0):
1447   getlinks[0] = 0;
1448  mli = getlinks[0];
1449  mlil = getlinks[1];
1450  returnval = {'numoflinks': mlil};
1451  returnval.update({'numofalllinks': len(mlessuname)});
1452  returnval.update({'pages': lastpage});
1453  returnval.update({'curpage': page});
1454  returnval.update({'orginurl': httpurl});
1455  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1456  mlessrooturltype = get_motherless_link_type(httpurl);
1457  returnval.update({'urltype': mlessrooturltype});
1458  while(mli<mlil):
1459   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli], urlparse.urlparse(mlessavatar[mli]).path));
1460   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1461   mlessurlname[mli] = mlessurlname[mli].replace("/m/", "");
1462   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessurlname[mli]);
1463   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessurlname[mli], 'userinfo': get_motherless_user_info(mlessurlname[mli]), 'username': mlessuname[mli], 'avatarurl': mlessavatar[mli], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1464   mli = mli + 1;
1465  return returnval;
1466
1467 def get_motherless_girls(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1468  premrtext = download_from_url("http://motherless.com/girls", httpheaders, httpcookie, httplibuse);
1469  mrtext = premrtext['Content'];
1470  if(sys.version[0]>="3"):
1471   mrtext = mrtext.decode('ascii', 'replace');
1472  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1473  mlessuname = re.findall(mregex_getuname, mrtext);
1474  mregex_geturlname = re.escape("\n\t\t\t\t\t\t<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">");
1475  mlessurlname = re.findall(mregex_geturlname, mrtext);
1476  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1477   getlinks[1] = len(mlessuname);
1478  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1479   tmpgetlinks0 = getlinks[0];
1480   tmpgetlinks1 = getlinks[1];
1481   getlinks[0] = tmpgetlinks1;
1482   getlinks[1] = tmpgetlinks0;
1483  if(getlinks[0]<0):
1484   getlinks[0] = 0;
1485  mli = getlinks[0];
1486  mlil = getlinks[1];
1487  returnval = {'numoflinks': mlil};
1488  returnval.update({'numofalllinks': len(mlessuname)});
1489  returnval.update({'orginurl': "http://motherless.com/girls"});
1490  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/girls")});
1491  mlessrooturltype = get_motherless_link_type("http://motherless.com/girls");
1492  returnval.update({'urltype': mlessrooturltype});
1493  while(mli<mlil):
1494   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessuname[mli][0], urlparse.urlparse(mlessuname[mli][0]).path));
1495   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1496   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname[mli][1]);
1497   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': mlessuname[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1498   mli = mli + 1;
1499  return returnval;
1500
1501 def get_motherless_team(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1502  premrtext = download_from_url("http://motherless.com/about", httpheaders, httpcookie, httplibuse);
1503  mrtext = premrtext['Content'];
1504  if(sys.version[0]>="3"):
1505   mrtext = mrtext.decode('ascii', 'replace');
1506  mregex_getuname = re.escape("<div class=\"about-us-member\">\n                         <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1507  mlessuname = re.findall(mregex_getuname, mrtext);
1508  mlessuname_odd = mlessuname[1::2];
1509  mlessuname_even = mlessuname[::2];
1510  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1511  mlessavatar = re.findall(mregex_getavatar, mrtext);
1512  if(getlinks[1]>len(mlessuname_odd) or getlinks[1]==-1):
1513   getlinks[1] = len(mlessuname_odd);
1514  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1515   tmpgetlinks0 = getlinks[0];
1516   tmpgetlinks1 = getlinks[1];
1517   getlinks[0] = tmpgetlinks1;
1518   getlinks[1] = tmpgetlinks0;
1519  if(getlinks[0]<0):
1520   getlinks[0] = 0;
1521  mli = getlinks[0];
1522  mlil = getlinks[1];
1523  returnval = {'numoflinks': mlil};
1524  returnval.update({'numofalllinks': len(mlessuname)});
1525  returnval.update({'orginurl': "http://motherless.com/about"});
1526  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/about")});
1527  mlessrooturltype = get_motherless_link_type("http://motherless.com/about");
1528  returnval.update({'urltype': mlessrooturltype});
1529  while(mli<mlil):
1530   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1531   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1532   mlessuname_odd[mli] = mlessuname_odd[mli].replace("/m/", "");
1533   mlessuname_even[mli] = mlessuname_even[mli].replace("/m/", "");
1534   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname_odd[mli]);
1535   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname_odd[mli], 'userinfo': get_motherless_user_info(mlessuname_odd[mli]), 'username': mlessuname_odd[mli], 'avatarurl': mlessavatar[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1536   mli = mli + 1;
1537  return returnval;
1538
1539 def get_motherless_top_referrers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1540  premrtext = download_from_url("http://motherless.com/referers", httpheaders, httpcookie, httplibuse);
1541  mrtext = premrtext['Content'];
1542  if(sys.version[0]>="3"):
1543   mrtext = mrtext.decode('ascii', 'replace');
1544  mregex_geturlname = "([0-9]+)"+re.escape(". <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"pop\" target=\"_blank\" rel=\"nofollow\">\n                                            ")+"(.*)"+re.escape("                                   </a>");
1545  mlessurlname = re.findall(mregex_geturlname, mrtext);
1546  if(getlinks[1]>len(mlessurlname) or getlinks[1]==-1):
1547   getlinks[1] = len(mlessurlname);
1548  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1549   tmpgetlinks0 = getlinks[0];
1550   tmpgetlinks1 = getlinks[1];
1551   getlinks[0] = tmpgetlinks1;
1552   getlinks[1] = tmpgetlinks0;
1553  if(getlinks[0]<0):
1554   getlinks[0] = 0;
1555  mli = getlinks[0];
1556  mlil = getlinks[1];
1557  returnval = {'numoflinks': mlil};
1558  returnval.update({'numofalllinks': len(mlessurlname)});
1559  returnval.update({'orginurl': "http://motherless.com/referers"});
1560  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/referers")});
1561  mlessrooturltype = get_motherless_link_type("http://motherless.com/referers");
1562  returnval.update({'urltype': mlessrooturltype});
1563  while(mli<mlil):
1564   returnval.update({mli: {'urltype': "referer-links", 'url': mlessurlname[mli][1], 'title': mlessurlname[mli][2]} });
1565   mli = mli + 1;
1566  return returnval;
1567
1568 def get_motherless_top_referers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1569  return get_motherless_top_referrers(httpheaders, httpcookie, httplibuse, getlinks);
1570
1571 def get_motherless_groups(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1572  httpurl = fix_motherless_url(httpurl);
1573  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1574  mrtext = premrtext['Content'];
1575  if(sys.version[0]>="3"):
1576   mrtext = mrtext.decode('ascii', 'replace');
1577  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1578  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1579  try:
1580   lastpage = int(mlesspagenum[-1][0]);
1581  except:
1582   lastpage = 1;
1583  if(page>lastpage):
1584   page = lastpage;
1585  httpurl = add_url_param(httpurl, page=str(page));
1586  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1587  mrtext = premrtext['Content'];
1588  if(sys.version[0]>="3"):
1589   mrtext = mrtext.decode('ascii', 'replace');
1590  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1591  mlessavatar = re.findall(mregex_getavatar, mrtext);
1592  mregex_getgroups = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"grunge motherless-red\">\n")+"(.*)"+re.escape("</a>");
1593  mlessgroups = re.findall(mregex_getgroups, mrtext);
1594  if(getlinks[1]>len(mlessgroups) or getlinks[1]==-1):
1595   getlinks[1] = len(mlessgroups);
1596  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1597   tmpgetlinks0 = getlinks[0];
1598   tmpgetlinks1 = getlinks[1];
1599   getlinks[0] = tmpgetlinks1;
1600   getlinks[1] = tmpgetlinks0;
1601  if(getlinks[0]<0):
1602   getlinks[0] = 0;
1603  mli = getlinks[0];
1604  mlil = getlinks[1];
1605  returnval = {'pages': lastpage};
1606  returnval.update({'curpage': page});
1607  returnval.update({'numoflinks': mlil});
1608  returnval.update({'numofalllinks': len(mlessgroups)});
1609  returnval.update({'orginurl': httpurl});
1610  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1611  while(mli<mlil):
1612   thumbfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1613   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1614   returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0]), 'url': "http://motherless.com"+mlessgroups[mli][0], 'urltype-image': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gi/")), 'url-image': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gi/"), 'urltype-video': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gv/")), 'url-video': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gv/"), 'urltype-member': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gm/")), 'url-member': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gm/"), 'thumbnail': mlessavatar[mli][0], 'title': mlessgroups[mli][1].strip(), 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension} });
1615   mli = mli + 1;
1616  return returnval;
1617
1618 def get_motherless_sample_links(httpheaders, httpcookie, httplibuse="urllib", numoflinks=10, urltype="video"):
1619  if(urltype=="video"):
1620   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) } };
1621  if(urltype=="image"):
1622   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) } };
1623  if(urltype=="gallery"):
1624   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/galleries", 'orginurltype': get_motherless_link_type("http://motherless.com/galleries"), 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1625  if(urltype=="all"):
1626   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1627  return returnval;
1628
1629 def get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1630  httpurl = fix_motherless_url(httpurl);
1631  returnval = False;
1632  if(get_motherless_link_type(httpurl)=="link"):
1633   returnval = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1634  if(get_motherless_link_type(httpurl)=="gallery"):
1635   returnval = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page);
1636  if(get_motherless_link_type(httpurl)=="sample-videos"):
1637   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "video");
1638  if(get_motherless_link_type(httpurl)=="sample-images"):
1639   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "image");
1640  if(get_motherless_link_type(httpurl)=="sample-galleries"):
1641   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "gallery");
1642  if(get_motherless_link_type(httpurl)=="sample" or get_motherless_link_type(httpurl)=="sample-all"):
1643   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "all");
1644  if(get_motherless_link_type(httpurl)=="board"):
1645   returnval = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse);
1646  if(get_motherless_link_type(httpurl)=="member"):
1647   returnval = get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse, page);
1648  if(get_motherless_link_type(httpurl)=="girls"):
1649   returnval = get_motherless_girls(httpheaders, httpcookie, httplibuse);
1650  if(get_motherless_link_type(httpurl)=="download"):
1651   returnval = httpurl;
1652  return returnval;
1653
1654 def view_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", viewerpro="mpv", prearg=[], proarg=[]):
1655  httpurl = fix_motherless_url(httpurl);
1656  commandlist = [viewerpro] + prearg;
1657  commandlist = commandlist + [get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse)['url']];
1658  commandlist = commandlist + proarg;
1659  mpvplaylistp = subprocess.Popen(commandlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
1660  mpvplayout, mpvplayerr = mpvplaylistp.communicate();
1661  return True;
1662
1663 def download_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[524288, 524288], outfile="-", outpath=os.getcwd(), usetitlename=False):
1664  global geturls_download_sleep;
1665  httpurl = fix_motherless_url(httpurl);
1666  if(sleep<0):
1667   sleep = geturls_download_sleep;
1668  mlessurl = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1669  outputname = mlessurl['fullfilename'];
1670  outpath = outpath.rstrip(os.path.sep);
1671  if(usetitlename==True):
1672   outputname = mlessurl['title'];
1673  if(usetitlename=="-" and outfile=="-"):
1674   outputname = "-";
1675  if(usetitlename=="-" and not outfile=="-"):
1676   outputname = outfile;
1677  returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep);
1678  return returnval;
1679
1680 def download_motherless_links_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[524288, 524288], outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1681  global geturls_download_sleep;
1682  httpurl = fix_motherless_url(httpurl);
1683  if(sleep<0):
1684   sleep = geturls_download_sleep;
1685  mlessurl = get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse, page);
1686  if(mlessurl['urltype']=="download"):
1687   outputname = mlessurl['fullfilename'];
1688   outpathname = outpath.rstrip(os.path.sep);
1689   if(usetitlename==True):
1690    outputname = mlessurl['title'];
1691   if(usetitlename=="-" and outfile[mli]=="-"):
1692    outputname = "-";
1693   if(usetitlename=="-" and not outfile[mli]=="-"):
1694    outputname = outfile;
1695   returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpathname, buffersize, sleep);
1696  if(not mlessurl['urltype']=="download"):
1697   returnval = mlessurl;
1698  return returnval;
1699
1700 def download_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[524288, 524288], outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1701  global geturls_download_sleep;
1702  httpurl = fix_motherless_url(httpurl);
1703  if(sleep<0):
1704   sleep = geturls_download_sleep;
1705  mlessgalleries = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks);
1706  mli = 0;
1707  mlil = mlessgalleries['numoflinks'];
1708  returnval = {'pages': mlessgalleries['pages']};
1709  returnval.update({'numoflists': mlessgalleries['numoflinks']});
1710  returnval.update({'curpage': mlessgalleries['curpage']});
1711  returnval.update({'numoflinks': mlessgalleries['numoflinks']});
1712  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1713  returnval.update({'orginurl': httpurl});
1714  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1715  while(mli<mlil):
1716   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1717   outputname = mlesslink['fullfilename'];
1718   outpath = outpath.rstrip(os.path.sep);
1719   if(usetitlename==True):
1720    outputname = mlesslink['title'];
1721   if(usetitlename=="-" and outfile=="-"):
1722    outputname = "-";
1723   if(usetitlename=="-" and not outfile=="-"):
1724    outputname = outfile;
1725   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1726   mli = mli + 1;
1727  return returnval;
1728
1729 def download_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[524288, 524288], outfile="-", outpath=os.getcwd(), usetitlename=False, getlinks=[0, -1]):
1730  global geturls_download_sleep;
1731  httpurl = fix_motherless_url(httpurl);
1732  if(sleep<0):
1733   sleep = geturls_download_sleep;
1734  mlessgalleries = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse, getlinks);
1735  mli = 0;
1736  mlil = mlessgalleries['numoflinks'];
1737  returnval = {'numoflists': mlessgalleries['numoflinks']};
1738  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1739  returnval.update({'orginurl': httpurl});
1740  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1741  while(mli<mlil):
1742   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1743   outputname = mlesslink['fullfilename'];
1744   outpath = outpath.rstrip(os.path.sep);
1745   if(usetitlename==True):
1746    outputname = mlesslink['title'];
1747   if(usetitlename=="-" and outfile=="-"):
1748    outputname = "-";
1749   if(usetitlename=="-" and not outfile=="-"):
1750    outputname = outfile;
1751   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1752   mli = mli + 1;
1753  return returnval;
1754