pymotherless.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2016 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2016 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: pymotherless.py - Last Update: 6/16/2016 Ver. 0.4.7 RC 1 - Author: cooldude2k $
  17 '''
  18
  19 from __future__ import division, absolute_import, print_function;
  20 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, imp;
  21 import logging as log;
  22 haverequests = False;
  23 try:
  24  imp.find_module('requests');
  25  haverequests = True;
  26  import requests;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  imp.find_module('mechanize');
  32  havemechanize = True;
  33  import mechanize;
  34 except ImportError:
  35  havemechanize = False;
  36 if(sys.version[0]=="2"):
  37  try:
  38   from cStringIO import StringIO;
  39  except ImportError:
  40   from StringIO import StringIO;
  41  # From http://python-future.org/compatible_idioms.html
  42  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  43  from urllib import urlencode;
  44  from urllib2 import urlopen, Request, HTTPError;
  45  import urllib2, urlparse, cookielib;
  46 if(sys.version[0]>="3"):
  47  from io import StringIO, BytesIO;
  48  # From http://python-future.org/compatible_idioms.html
  49  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  50  from urllib.request import urlopen, Request;
  51  from urllib.error import HTTPError;
  52  import urllib.request as urllib2;
  53  import urllib.parse as urlparse;
  54  import http.cookiejar as cookielib;
  55
  56 __program_name__ = "PyMotherless";
  57 __project__ = __program_name__;
  58 __project_url__ = "https://github.com/GameMaker2k/PyMotherless";
  59 __version_info__ = (0, 4, 7, "RC 1", 1);
  60 __version_date_info__ = (2016, 6, 16, "RC 1", 1);
  61 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  62 if(__version_info__[4]!=None):
  63  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  64 if(__version_info__[4]==None):
  65  __version_date_plusrc__ = __version_date__;
  66 if(__version_info__[3]!=None):
  67  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  68 if(__version_info__[3]==None):
  69  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  70
  71 tmpfileprefix = "py"+str(sys.version_info[0])+"motherless"+str(__version_info__[0])+"-";
  72 tmpfilesuffix = "-";
  73 pytempdir = tempfile.gettempdir();
  74 defbuffersize = 524288;
  75 defbuffersizedub = [defbuffersize, defbuffersize];
  76
  77 geturls_cj = cookielib.CookieJar();
  78 geturls_ua_firefox_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0";
  79 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0 SeaMonkey/2.49.3";
  80 geturls_ua_chrome_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
  81 geturls_ua_chromium_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chromium/67.0.3396.99 Chrome/67.0.3396.99 Safari/537.36";
  82 geturls_ua_palemoon_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:52.9) Gecko/20100101 Goanna/3.4 Firefox/52.9 PaleMoon/27.9.3";
  83 geturls_ua_opera_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36 OPR/54.0.2952.54";
  84 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 Vivaldi/1.2.490.43";
  85 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko";
  86 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134";
  87 geturls_ua_pymotherless_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
  88 if(platform.python_implementation()!=""):
  89  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=platform.python_implementation(), pyver=platform.python_version(), proname=__project__, prover=__version__);
  90 if(platform.python_implementation()==""):
  91  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp="Python", pyver=platform.python_version(), proname=__project__, prover=__version__);
  92 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
  93 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
  94 geturls_ua = geturls_ua_firefox_windows7;
  95 geturls_headers_firefox_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  96 geturls_headers_seamonkey_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  97 geturls_headers_chrome_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  98 geturls_headers_chromium_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  99 geturls_headers_palemoon_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 100 geturls_headers_opera_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 101 geturls_headers_vivaldi_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 102 geturls_headers_internet_explorer_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 103 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 104 geturls_headers_pymotherless_python = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 105 geturls_headers_pymotherless_python_alt = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python_alt, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 106 geturls_headers_googlebot_google = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 107 geturls_headers_googlebot_google_old = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 108 geturls_headers = geturls_headers_firefox_windows7;
 109 geturls_download_sleep = 0;
 110
 111 def add_url_param(url, **params):
 112  n=3;
 113  parts = list(urlparse.urlsplit(url));
 114  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 115  d.update(params);
 116  parts[n]=urlencode(d);
 117  return urlparse.urlunsplit(parts);
 118
 119 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 120 def which_exec(execfile):
 121  for path in os.environ["PATH"].split(":"):
 122   if os.path.exists(path + "/" + execfile):
 123    return path + "/" + execfile;
 124
 125 def listize(varlist):
 126  il = 0;
 127  ix = len(varlist);
 128  ilx = 1;
 129  newlistreg = {};
 130  newlistrev = {};
 131  newlistfull = {};
 132  while(il < ix):
 133   newlistreg.update({ilx: varlist[il]});
 134   newlistrev.update({varlist[il]: ilx});
 135   ilx = ilx + 1;
 136   il = il + 1;
 137  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 138  return newlistfull;
 139
 140 def twolistize(varlist):
 141  il = 0;
 142  ix = len(varlist);
 143  ilx = 1;
 144  newlistnamereg = {};
 145  newlistnamerev = {};
 146  newlistdescreg = {};
 147  newlistdescrev = {};
 148  newlistfull = {};
 149  while(il < ix):
 150   newlistnamereg.update({ilx: varlist[il][0].strip()});
 151   newlistnamerev.update({varlist[il][0].strip(): ilx});
 152   newlistdescreg.update({ilx: varlist[il][1].strip()});
 153   newlistdescrev.update({varlist[il][1].strip(): ilx});
 154   ilx = ilx + 1;
 155   il = il + 1;
 156  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 157  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 158  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 159  return newlistfull;
 160
 161 def arglistize(proexec, *varlist):
 162  il = 0;
 163  ix = len(varlist);
 164  ilx = 1;
 165  newarglist = [proexec];
 166  while(il < ix):
 167   if varlist[il][0] is not None:
 168    newarglist.append(varlist[il][0]);
 169   if varlist[il][1] is not None:
 170    newarglist.append(varlist[il][1]);
 171   il = il + 1;
 172  return newarglist;
 173
 174 # hms_string by ArcGIS Python Recipes
 175 # https://arcpy.wordpress.com/2012/04/20/146/
 176 def hms_string(sec_elapsed):
 177  h = int(sec_elapsed / (60 * 60));
 178  m = int((sec_elapsed % (60 * 60)) / 60);
 179  s = sec_elapsed % 60.0;
 180  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 181
 182 # get_readable_size by Lipis
 183 # http://stackoverflow.com/posts/14998888/revisions
 184 def get_readable_size(bytes, precision=1, unit="IEC"):
 185  unit = unit.upper();
 186  if(unit!="IEC" and unit!="SI"):
 187   unit = "IEC";
 188  if(unit=="IEC"):
 189   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 190   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 191   unitsize = 1024.0;
 192  if(unit=="SI"):
 193   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 194   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 195   unitsize = 1000.0;
 196  return_val = {};
 197  orgbytes = bytes;
 198  for unit in units:
 199   if abs(bytes) < unitsize:
 200    strformat = "%3."+str(precision)+"f%s";
 201    pre_return_val = (strformat % (bytes, unit));
 202    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 203    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 204    alt_return_val = pre_return_val.split();
 205    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 206    return return_val;
 207   bytes /= unitsize;
 208  strformat = "%."+str(precision)+"f%s";
 209  pre_return_val = (strformat % (bytes, "YiB"));
 210  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 211  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 212  alt_return_val = pre_return_val.split();
 213  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 214  return return_val;
 215
 216 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 217  unit = unit.upper();
 218  usehashtypes = usehashtypes.lower();
 219  getfilesize = os.path.getsize(infile);
 220  return_val = get_readable_size(getfilesize, precision, unit);
 221  if(usehashes==True):
 222   hashtypelist = usehashtypes.split(",");
 223   openfile = open(infile, "rb");
 224   filecontents = openfile.read();
 225   openfile.close();
 226   listnumcount = 0;
 227   listnumend = len(hashtypelist);
 228   while(listnumcount < listnumend):
 229    hashtypelistlow = hashtypelist[listnumcount].strip();
 230    hashtypelistup = hashtypelistlow.upper();
 231    filehash = hashlib.new(hashtypelistup);
 232    filehash.update(filecontents);
 233    filegethash = filehash.hexdigest();
 234    return_val.update({hashtypelistup: filegethash});
 235    listnumcount += 1;
 236  return return_val;
 237
 238 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 239  unit = unit.upper();
 240  usehashtypes = usehashtypes.lower();
 241  getfilesize = len(instring);
 242  return_val = get_readable_size(getfilesize, precision, unit);
 243  if(usehashes==True):
 244   hashtypelist = usehashtypes.split(",");
 245   listnumcount = 0;
 246   listnumend = len(hashtypelist);
 247   while(listnumcount < listnumend):
 248    hashtypelistlow = hashtypelist[listnumcount].strip();
 249    hashtypelistup = hashtypelistlow.upper();
 250    filehash = hashlib.new(hashtypelistup);
 251    if(sys.version[0]=="2"):
 252     filehash.update(instring);
 253    if(sys.version[0]>="3"):
 254     filehash.update(instring.encode('utf-8'));
 255    filegethash = filehash.hexdigest();
 256    return_val.update({hashtypelistup: filegethash});
 257    listnumcount += 1;
 258  return return_val;
 259
 260 def make_http_headers_from_dict_to_list(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 261  if isinstance(headers, dict):
 262   returnval = [];
 263   if(sys.version[0]=="2"):
 264    for headkey, headvalue in headers.iteritems():
 265     returnval.append((headkey, headvalue));
 266   if(sys.version[0]>="3"):
 267    for headkey, headvalue in headers.items():
 268     returnval.append((headkey, headvalue));
 269  elif isinstance(headers, list):
 270   returnval = headers;
 271  else:
 272   returnval = False;
 273  return returnval;
 274
 275 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 276  if isinstance(headers, dict):
 277   returnval = [];
 278   if(sys.version[0]=="2"):
 279    for headkey, headvalue in headers.iteritems():
 280     returnval.append(headkey+": "+headvalue);
 281   if(sys.version[0]>="3"):
 282    for headkey, headvalue in headers.items():
 283     returnval.append(headkey+": "+headvalue);
 284  elif isinstance(headers, list):
 285   returnval = headers;
 286  else:
 287   returnval = False;
 288  return returnval;
 289
 290 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://motherless.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 291  if isinstance(headers, list):
 292   returnval = {};
 293   mli = 0;
 294   mlil = len(headers);
 295   while(mli<mlil):
 296    returnval.update({headers[mli][0]: headers[mli][1]});
 297    mli = mli + 1;
 298  elif isinstance(headers, dict):
 299   returnval = headers;
 300  else:
 301   returnval = False;
 302  return returnval;
 303
 304 def get_httplib_support(checkvalue=None):
 305  global haverequests, havemechanize;
 306  returnval = [];
 307  returnval.append("urllib");
 308  if(haverequests==True):
 309   returnval.append("requests");
 310  if(havemechanize==True):
 311   returnval.append("mechanize");
 312  if(not checkvalue==None):
 313   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 314    checkvalue = "urllib";
 315   if(checkvalue in returnval):
 316    returnval = True;
 317   else:
 318    returnval = False;
 319  return returnval;
 320
 321 def check_httplib_support(checkvalue="urllib"):
 322  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 323   checkvalue = "urllib";
 324  returnval = get_httplib_support(checkvalue);
 325  return returnval;
 326
 327 def get_httplib_support_list():
 328  returnval = get_httplib_support(None);
 329  return returnval;
 330
 331 def download_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1):
 332  global geturls_download_sleep, haverequests, havemechanize;
 333  if(sleep<0):
 334   sleep = geturls_download_sleep;
 335  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 336   httplibuse = "urllib";
 337  if(haverequests==False and httplibuse=="requests"):
 338   httplibuse = "urllib";
 339  if(havemechanize==False and httplibuse=="mechanize"):
 340   httplibuse = "urllib";
 341  if(httplibuse=="urllib"):
 342   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep);
 343  elif(httplibuse=="requests"):
 344   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep);
 345  elif(httplibuse=="mechanize"):
 346   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep);
 347  else:
 348   returnval = False;
 349  return returnval;
 350
 351 def download_from_url_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", buffersize=defbuffersize, sleep=-1):
 352  global geturls_download_sleep, haverequests, havemechanize;
 353  if(sleep<0):
 354   sleep = geturls_download_sleep;
 355  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 356   httplibuse = "urllib";
 357  if(haverequests==False and httplibuse=="requests"):
 358   httplibuse = "urllib";
 359  if(havemechanize==False and httplibuse=="mechanize"):
 360   httplibuse = "urllib";
 361  if(httplibuse=="urllib"):
 362   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep);
 363  elif(httplibuse=="requests"):
 364   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize, sleep);
 365  elif(httplibuse=="mechanize"):
 366   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize, sleep);
 367  else:
 368   returnval = False;
 369  return returnval;
 370
 371 def download_from_url_to_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 372  global geturls_download_sleep, haverequests, havemechanize;
 373  if(sleep<0):
 374   sleep = geturls_download_sleep;
 375  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 376   httplibuse = "urllib";
 377  if(haverequests==False and httplibuse=="requests"):
 378   httplibuse = "urllib";
 379  if(havemechanize==False and httplibuse=="mechanize"):
 380   httplibuse = "urllib";
 381  if(httplibuse=="urllib"):
 382   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 383  elif(httplibuse=="requests"):
 384   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 385  elif(httplibuse=="mechanize"):
 386   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 387  else:
 388   returnval = False;
 389  return returnval;
 390
 391 def download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep=-1):
 392  global geturls_download_sleep;
 393  if(sleep<0):
 394   sleep = geturls_download_sleep;
 395  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 396  if(isinstance(httpheaders, dict)):
 397   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 398  geturls_opener.addheaders = httpheaders;
 399  time.sleep(sleep);
 400  geturls_text = geturls_opener.open(httpurl);
 401  log.info("Downloading URL "+httpurl);
 402  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 403   if(sys.version[0]=="2"):
 404    strbuf = StringIO(geturls_text.read());
 405   if(sys.version[0]>="3"):
 406    strbuf = BytesIO(geturls_text.read());
 407   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 408   returnval_content = gzstrbuf.read()[:];
 409  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 410   returnval_content = geturls_text.read()[:];
 411  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 412  geturls_text.close();
 413  return returnval;
 414
 415 def download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 416  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 417  exec_time_start = time.time();
 418  myhash = hashlib.new("sha1");
 419  if(sys.version[0]=="2"):
 420   myhash.update(httpurl);
 421   myhash.update(str(buffersize));
 422   myhash.update(str(exec_time_start));
 423  if(sys.version[0]>="3"):
 424   myhash.update(httpurl.encode('utf-8'));
 425   myhash.update(str(buffersize).encode('utf-8'));
 426   myhash.update(str(exec_time_start).encode('utf-8'));
 427  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 428  if(sleep<0):
 429   sleep = geturls_download_sleep;
 430  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 431  if(isinstance(httpheaders, dict)):
 432   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 433  geturls_opener.addheaders = httpheaders;
 434  time.sleep(sleep);
 435  geturls_text = geturls_opener.open(httpurl);
 436  downloadsize = geturls_text.info().get('Content-Length');
 437  if(downloadsize is not None):
 438   downloadsize = int(downloadsize);
 439  if downloadsize is None: downloadsize = 0;
 440  fulldatasize = 0;
 441  prevdownsize = 0;
 442  log.info("Downloading URL "+httpurl);
 443  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 444   tmpfilename = f.name;
 445   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 446   while True:
 447    databytes = geturls_text.read(buffersize);
 448    if not databytes: break;
 449    datasize = len(databytes);
 450    fulldatasize = datasize + fulldatasize;
 451    percentage = "";
 452    if(downloadsize>0):
 453     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 454    downloaddiff = fulldatasize - prevdownsize;
 455    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 456    prevdownsize = fulldatasize;
 457    f.write(databytes);
 458   f.close();
 459  geturls_text.close();
 460  exec_time_end = time.time();
 461  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 462  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 463  return returnval;
 464
 465 def download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 466  global geturls_download_sleep;
 467  if(sleep<0):
 468   sleep = geturls_download_sleep;
 469  if(not outfile=="-"):
 470   outpath = outpath.rstrip(os.path.sep);
 471   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 472   if(not os.path.exists(outpath)):
 473    os.makedirs(outpath);
 474   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 475    return False;
 476   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 477    return False;
 478   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 479   tmpfilename = pretmpfilename['Filename'];
 480   downloadsize = os.path.getsize(tmpfilename);
 481   fulldatasize = 0;
 482   log.info("Moving file "+tmpfilename+" to "+filepath);
 483   exec_time_start = time.time();
 484   shutil.move(tmpfilename, filepath);
 485   exec_time_end = time.time();
 486   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 487   if(os.path.exists(tmpfilename)==True):
 488    os.remove(tmpfilename);
 489   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 490  if(outfile=="-" and sys.version[0]=="2"):
 491   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 492   tmpfilename = pretmpfilename['Filename'];
 493   downloadsize = os.path.getsize(tmpfilename);
 494   fulldatasize = 0;
 495   prevdownsize = 0;
 496   exec_time_start = time.time();
 497   with open(tmpfilename, 'rb') as ft:
 498    f = StringIO();
 499    while True:
 500     databytes = ft.read(buffersize[1]);
 501     if not databytes: break;
 502     datasize = len(databytes);
 503     fulldatasize = datasize + fulldatasize;
 504     percentage = "";
 505     if(downloadsize>0):
 506      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 507     downloaddiff = fulldatasize - prevdownsize;
 508     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 509     prevdownsize = fulldatasize;
 510     f.write(databytes);
 511    f.seek(0);
 512    fdata = f.getvalue();
 513    f.close();
 514    ft.close();
 515    os.remove(tmpfilename);
 516    exec_time_end = time.time();
 517    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 518   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 519  if(outfile=="-" and sys.version[0]>="3"):
 520   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 521   tmpfilename = pretmpfilename['Filename'];
 522   downloadsize = os.path.getsize(tmpfilename);
 523   fulldatasize = 0;
 524   prevdownsize = 0;
 525   exec_time_start = time.time();
 526   with open(tmpfilename, 'rb') as ft:
 527    f = BytesIO();
 528    while True:
 529     databytes = ft.read(buffersize[1]);
 530     if not databytes: break;
 531     datasize = len(databytes);
 532     fulldatasize = datasize + fulldatasize;
 533     percentage = "";
 534     if(downloadsize>0):
 535      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 536     downloaddiff = fulldatasize - prevdownsize;
 537     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 538     prevdownsize = fulldatasize;
 539     f.write(databytes);
 540    f.seek(0);
 541    fdata = f.getvalue();
 542    f.close();
 543    ft.close();
 544    os.remove(tmpfilename);
 545    exec_time_end = time.time();
 546    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 547   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 548  return returnval;
 549
 550 if(haverequests==True):
 551  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 552   global geturls_download_sleep;
 553   if(sleep<0):
 554    sleep = geturls_download_sleep;
 555   if(isinstance(httpheaders, list)):
 556    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 557   time.sleep(sleep);
 558   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
 559   log.info("Downloading URL "+httpurl);
 560   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
 561    if(sys.version[0]=="2"):
 562     strbuf = StringIO(geturls_text.content);
 563    if(sys.version[0]>="3"):
 564     strbuf = BytesIO(geturls_text.content);
 565    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 566    returnval_content = gzstrbuf.content[:];
 567   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
 568    returnval_content = geturls_text.content[:];
 569   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 570   geturls_text.close();
 571   return returnval;
 572
 573 if(haverequests==False):
 574  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 575   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 576   return returnval;
 577
 578 if(haverequests==True):
 579  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 580   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 581   exec_time_start = time.time();
 582   myhash = hashlib.new("sha1");
 583   if(sys.version[0]=="2"):
 584    myhash.update(httpurl);
 585    myhash.update(str(buffersize));
 586    myhash.update(str(exec_time_start));
 587   if(sys.version[0]>="3"):
 588    myhash.update(httpurl.encode('utf-8'));
 589    myhash.update(str(buffersize).encode('utf-8'));
 590    myhash.update(str(exec_time_start).encode('utf-8'));
 591   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 592   if(sleep<0):
 593    sleep = geturls_download_sleep;
 594   if(isinstance(httpheaders, list)):
 595    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 596   time.sleep(sleep);
 597   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
 598   downloadsize = int(geturls_text.headers.get('Content-Length'));
 599   if(downloadsize is not None):
 600    downloadsize = int(downloadsize);
 601   if downloadsize is None: downloadsize = 0;
 602   fulldatasize = 0;
 603   prevdownsize = 0;
 604   log.info("Downloading URL "+httpurl);
 605   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 606    tmpfilename = f.name;
 607    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 608    for databytes in geturls_text.iter_content(chunk_size=buffersize):
 609     datasize = len(databytes);
 610     fulldatasize = datasize + fulldatasize;
 611     percentage = "";
 612     if(downloadsize>0):
 613      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 614     downloaddiff = fulldatasize - prevdownsize;
 615     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 616     prevdownsize = fulldatasize;
 617     f.write(databytes);
 618    f.close();
 619   geturls_text.close();
 620   exec_time_end = time.time();
 621   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 622   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 623   return returnval;
 624
 625 if(haverequests==False):
 626  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 627   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 628   return returnval;
 629
 630 if(haverequests==True):
 631  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 632   global geturls_download_sleep;
 633   if(sleep<0):
 634    sleep = geturls_download_sleep;
 635   if(not outfile=="-"):
 636    outpath = outpath.rstrip(os.path.sep);
 637    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 638    if(not os.path.exists(outpath)):
 639     os.makedirs(outpath);
 640    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 641     return False;
 642    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 643     return False;
 644    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 645    tmpfilename = pretmpfilename['Filename'];
 646    downloadsize = os.path.getsize(tmpfilename);
 647    fulldatasize = 0;
 648    log.info("Moving file "+tmpfilename+" to "+filepath);
 649    exec_time_start = time.time();
 650    shutil.move(tmpfilename, filepath);
 651    exec_time_end = time.time();
 652    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 653    if(os.path.exists(tmpfilename)==True):
 654     os.remove(tmpfilename);
 655    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 656   if(outfile=="-" and sys.version[0]=="2"):
 657    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 658    tmpfilename = pretmpfilename['Filename'];
 659    downloadsize = os.path.getsize(tmpfilename);
 660    fulldatasize = 0;
 661    prevdownsize = 0;
 662    exec_time_start = time.time();
 663    with open(tmpfilename, 'rb') as ft:
 664     f = StringIO();
 665     while True:
 666      databytes = ft.read(buffersize[1]);
 667      if not databytes: break;
 668      datasize = len(databytes);
 669      fulldatasize = datasize + fulldatasize;
 670      percentage = "";
 671      if(downloadsize>0):
 672       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 673      downloaddiff = fulldatasize - prevdownsize;
 674      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 675      prevdownsize = fulldatasize;
 676      f.write(databytes);
 677     f.seek(0);
 678     fdata = f.getvalue();
 679     f.close();
 680     ft.close();
 681     os.remove(tmpfilename);
 682     exec_time_end = time.time();
 683     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 684    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 685   if(outfile=="-" and sys.version[0]>="3"):
 686    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 687    tmpfilename = pretmpfilename['Filename'];
 688    downloadsize = os.path.getsize(tmpfilename);
 689    fulldatasize = 0;
 690    prevdownsize = 0;
 691    exec_time_start = time.time();
 692    with open(tmpfilename, 'rb') as ft:
 693     f = BytesIO();
 694     while True:
 695      databytes = ft.read(buffersize[1]);
 696      if not databytes: break;
 697      datasize = len(databytes);
 698      fulldatasize = datasize + fulldatasize;
 699      percentage = "";
 700      if(downloadsize>0):
 701       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 702      downloaddiff = fulldatasize - prevdownsize;
 703      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 704      prevdownsize = fulldatasize;
 705      f.write(databytes);
 706     f.seek(0);
 707     fdata = f.getvalue();
 708     f.close();
 709     ft.close();
 710     os.remove(tmpfilename);
 711     exec_time_end = time.time();
 712     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 713    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 714   return returnval;
 715
 716 if(haverequests==False):
 717  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 718   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 719   return returnval;
 720
 721 if(havemechanize==True):
 722  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 723   global geturls_download_sleep;
 724   if(sleep<0):
 725    sleep = geturls_download_sleep;
 726   geturls_opener = mechanize.Browser();
 727   if(isinstance(httpheaders, dict)):
 728    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 729   time.sleep(sleep);
 730   geturls_opener.addheaders = httpheaders;
 731   geturls_opener.set_cookiejar(httpcookie);
 732   geturls_opener.set_handle_robots(False);
 733   geturls_text = geturls_opener.open(httpurl);
 734   log.info("Downloading URL "+httpurl);
 735   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 736    if(sys.version[0]=="2"):
 737     strbuf = StringIO(geturls_text.read());
 738    if(sys.version[0]>="3"):
 739     strbuf = BytesIO(geturls_text.read());
 740    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 741    returnval_content = gzstrbuf.read()[:];
 742   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 743    returnval_content = geturls_text.read()[:];
 744   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 745   geturls_text.close();
 746   return returnval;
 747
 748 if(havemechanize==False):
 749  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 750   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 751   return returnval;
 752
 753 if(havemechanize==True):
 754  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 755   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 756   exec_time_start = time.time();
 757   myhash = hashlib.new("sha1");
 758   if(sys.version[0]=="2"):
 759    myhash.update(httpurl);
 760    myhash.update(str(buffersize));
 761    myhash.update(str(exec_time_start));
 762   if(sys.version[0]>="3"):
 763    myhash.update(httpurl.encode('utf-8'));
 764    myhash.update(str(buffersize).encode('utf-8'));
 765    myhash.update(str(exec_time_start).encode('utf-8'));
 766   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 767   if(sleep<0):
 768    sleep = geturls_download_sleep;
 769   geturls_opener = mechanize.Browser();
 770   if(isinstance(httpheaders, dict)):
 771    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 772   time.sleep(sleep);
 773   geturls_opener.addheaders = httpheaders;
 774   geturls_opener.set_cookiejar(httpcookie);
 775   geturls_opener.set_handle_robots(False);
 776   geturls_text = geturls_opener.open(httpurl);
 777   downloadsize = int(geturls_text.info().get('Content-Length'));
 778   if(downloadsize is not None):
 779    downloadsize = int(downloadsize);
 780   if downloadsize is None: downloadsize = 0;
 781   fulldatasize = 0;
 782   prevdownsize = 0;
 783   log.info("Downloading URL "+httpurl);
 784   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 785    tmpfilename = f.name;
 786    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 787    while True:
 788     databytes = geturls_text.read(buffersize);
 789     if not databytes: break;
 790     datasize = len(databytes);
 791     fulldatasize = datasize + fulldatasize;
 792     percentage = "";
 793     if(downloadsize>0):
 794      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 795     downloaddiff = fulldatasize - prevdownsize;
 796     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 797     prevdownsize = fulldatasize;
 798     f.write(databytes);
 799    f.close();
 800   geturls_text.close();
 801   exec_time_end = time.time();
 802   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 803   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 804   return returnval;
 805
 806 if(havemechanize==False):
 807  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 808   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 809   return returnval;
 810
 811 if(havemechanize==True):
 812  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 813   global geturls_download_sleep;
 814   if(sleep<0):
 815    sleep = geturls_download_sleep;
 816   if(not outfile=="-"):
 817    outpath = outpath.rstrip(os.path.sep);
 818    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 819    if(not os.path.exists(outpath)):
 820     os.makedirs(outpath);
 821    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 822     return False;
 823    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 824     return False;
 825    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 826    tmpfilename = pretmpfilename['Filename'];
 827    downloadsize = os.path.getsize(tmpfilename);
 828    fulldatasize = 0;
 829    log.info("Moving file "+tmpfilename+" to "+filepath);
 830    exec_time_start = time.time();
 831    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 832    shutil.move(tmpfilename, filepath);
 833    if(os.path.exists(tmpfilename)==True):
 834     os.remove(tmpfilename);
 835    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 836   if(outfile=="-" and sys.version[0]=="2"):
 837    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 838    tmpfilename = pretmpfilename['Filename'];
 839    downloadsize = os.path.getsize(tmpfilename);
 840    fulldatasize = 0;
 841    prevdownsize = 0;
 842    exec_time_start = time.time();
 843    with open(tmpfilename, 'rb') as ft:
 844     f = StringIO();
 845     while True:
 846      databytes = ft.read(buffersize[1]);
 847      if not databytes: break;
 848      datasize = len(databytes);
 849      fulldatasize = datasize + fulldatasize;
 850      percentage = "";
 851      if(downloadsize>0):
 852       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 853      downloaddiff = fulldatasize - prevdownsize;
 854      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 855      prevdownsize = fulldatasize;
 856      f.write(databytes);
 857     f.seek(0);
 858     fdata = f.getvalue();
 859     f.close();
 860     ft.close();
 861     os.remove(tmpfilename);
 862     exec_time_end = time.time();
 863     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 864    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 865   if(outfile=="-" and sys.version[0]>="3"):
 866    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 867    tmpfilename = pretmpfilename['Filename'];
 868    downloadsize = os.path.getsize(tmpfilename);
 869    fulldatasize = 0;
 870    prevdownsize = 0;
 871    exec_time_start = time.time();
 872    with open(tmpfilename, 'rb') as ft:
 873     f = BytesIO();
 874     while True:
 875      databytes = ft.read(buffersize[1]);
 876      if not databytes: break;
 877      datasize = len(databytes);
 878      fulldatasize = datasize + fulldatasize;
 879      percentage = "";
 880      if(downloadsize>0):
 881       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 882      downloaddiff = fulldatasize - prevdownsize;
 883      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 884      prevdownsize = fulldatasize;
 885      f.write(databytes);
 886     f.seek(0);
 887     fdata = f.getvalue();
 888     f.close();
 889     ft.close();
 890     os.remove(tmpfilename);
 891     exec_time_end = time.time();
 892     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 893    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 894   return returnval;
 895
 896 if(havemechanize==False):
 897  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 898   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 899   return returnval;
 900
 901 def check_motherless_url(httpurl):
 902  returnval = {};
 903  httpurl.replace("https://", "http://");
 904  if(not httpurl.startswith("http")):
 905   httpurl = "http://"+httpurl;
 906  urlparts = urlsplit(httpurl);
 907  urltype = None;
 908  domaintype = None;
 909  urlservtype = None;
 910  if(urlparts.netloc=="motherless.com" or urlparts.netloc=="www.motherless.com"):
 911   urltype = "motherless-link";
 912   domaintype = "motherless";
 913   urlservtype = "www";
 914  if(urlparts.netloc=="cdn.avatars.motherlessmedia.com" or urlparts.netloc=="cdn1.avatars.motherlessmedia.com" or urlparts.netloc=="cdn2.avatars.motherlessmedia.com" or urlparts.netloc=="cdn3.avatars.motherlessmedia.com" or urlparts.netloc=="cdn4.avatars.motherlessmedia.com"):
 915   urltype = "motherless-avatar";
 916   domaintype = "motherlessmedia";
 917  if(urlparts.netloc=="cdn.images.motherlessmedia.com" or urlparts.netloc=="cdn1.images.motherlessmedia.com" or urlparts.netloc=="cdn2.images.motherlessmedia.com" or urlparts.netloc=="cdn3.images.motherlessmedia.com" or urlparts.netloc=="cdn4.images.motherlessmedia.com"):
 918   urltype = "motherless-image";
 919   domaintype = "motherlessmedia";
 920  if(urlparts.netloc=="cdn.videos.motherlessmedia.com" or urlparts.netloc=="cdn1.videos.motherlessmedia.com" or urlparts.netloc=="cdn2.videos.motherlessmedia.com" or urlparts.netloc=="cdn3.videos.motherlessmedia.com" or urlparts.netloc=="cdn4.videos.motherlessmedia.com"):
 921   urltype = "motherless-video";
 922   domaintype = "motherlessmedia";
 923  if(urlparts.netloc=="cdn.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn1.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn2.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn3.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 924   urltype = "motherless-thumbnail";
 925   domaintype = "motherlessmedia";
 926  if(urlparts.netloc=="cdn.avatars.motherlessmedia.com" or urlparts.netloc=="cdn.images.motherlessmedia.com" or urlparts.netloc=="cdn.videos.motherlessmedia.com" or urlparts.netloc=="cdn.thumbs.motherlessmedia.com"):
 927   urlservtype = "cdn";
 928  if(urlparts.netloc=="cdn1.avatars.motherlessmedia.com" or urlparts.netloc=="cdn1.images.motherlessmedia.com" or urlparts.netloc=="cdn1.videos.motherlessmedia.com" or urlparts.netloc=="cdn1.thumbs.motherlessmedia.com"):
 929   urlservtype = "cdn1";
 930  if(urlparts.netloc=="cdn2.avatars.motherlessmedia.com" or urlparts.netloc=="cdn2.images.motherlessmedia.com" or urlparts.netloc=="cdn2.videos.motherlessmedia.com" or urlparts.netloc=="cdn2.thumbs.motherlessmedia.com"):
 931   urlservtype = "cdn2";
 932  if(urlparts.netloc=="cdn3.avatars.motherlessmedia.com" or urlparts.netloc=="cdn3.images.motherlessmedia.com" or urlparts.netloc=="cdn3.videos.motherlessmedia.com" or urlparts.netloc=="cdn3.thumbs.motherlessmedia.com"):
 933   urlservtype = "cdn3";
 934  if(urlparts.netloc=="cdn4.avatars.motherlessmedia.com" or urlparts.netloc=="cdn4.images.motherlessmedia.com" or urlparts.netloc=="cdn4.videos.motherlessmedia.com" or urlparts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 935   urlservtype = "cdn4";
 936  if(urltype==None or domaintype==None or urlservtype==None):
 937   return False;
 938  returnval.update({'url': urlparts.geturl(), 'originalurl': httpurl, 'urltype': urltype, 'urlserv': urlservtype, 'domaintype': domaintype, 'urlinfo': {'scheme': urlparts.scheme, 'netloc': urlparts.netloc, 'path': urlparts.path, 'query': urlparts.query, 'fragment': urlparts.fragment}});
 939  return returnval;
 940
 941 def fix_motherless_url(httpurl):
 942  tmpreturnval = check_motherless_url(httpurl);
 943  if(tmpreturnval==False):
 944   returnval = False
 945  else:
 946   returnval = tmpreturnval['url'];
 947  return returnval;
 948
 949 def get_motherless_link_type(httpurl):
 950  httpurl = fix_motherless_url(httpurl);
 951  if(httpurl==False):
 952   return False;
 953  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
 954  httpurl = re.sub(mregex_gettitle, "http://cdn.", httpurl);
 955  mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(httpurl).query);
 956  mlessvidid_parts = urlparse.urlparse(httpurl);
 957  mlessvidid = mlessvidid_parts.path.split("/");
 958  returnval = False;
 959  if(mlessvidid[1]=="videos" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 960   returnval = "gallery";
 961  if(mlessvidid[1]=="images" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 962   returnval = "gallery";
 963  if(mlessvidid[1]=="galleries" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 964   returnval = "gallery";
 965  if(mlessvidid[1]=="videos" and len(mlessvidid)==2):
 966   returnval = "sample-videos";
 967  if(mlessvidid[1]=="images" and len(mlessvidid)==2):
 968   returnval = "sample-images";
 969  if(mlessvidid[1]=="galleries" and len(mlessvidid)==2):
 970   returnval = "sample-galleries";
 971  if(mlessvidid[1]=="" and len(mlessvidid)==2):
 972   returnval = "sample";
 973  if(mlessvidid[1]=="groups" and len(mlessvidid)==2):
 974   returnval = "group";
 975  if(mlessvidid[1]=="groups" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 976   returnval = "group";
 977  if(mlessvidid[1]=="groups" and len(mlessvidid)==4 and mlessvidid[2]=="category"):
 978   returnval = "group";
 979  if(mlessvidid[1]=="live" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images")):
 980   returnval = "gallery";
 981  if(mlessvidid[1]=="u" and len(mlessvidid)==3):
 982   returnval = "gallery";
 983  if(mlessvidid[1]=="f" and len(mlessvidid)==4 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 984   returnval = "gallery";
 985  if(mlessvidid[1]=="galleries" and len(mlessvidid)==4 and mlessvidid[2]=="member"):
 986   returnval = "gallery";
 987  if(mlessvidid[1]=="galleries" and len(mlessvidid)==5 and mlessvidid[2]=="member" and (mlessvidid[4]=="created" or mlessvidid[4]=="viewed" or mlessvidid[4]=="favorited" or mlessvidid[4]=="commented")):
 988   returnval = "gallery";
 989  if(mlessvidid[1]=="gv" and len(mlessvidid)==3):
 990   returnval = "gallery";
 991  if(mlessvidid[1]=="gi" and len(mlessvidid)==3):
 992   returnval = "gallery";
 993  if(mlessvidid[1]=="gm" and len(mlessvidid)==3):
 994   returnval = "member";
 995  if(mlessvidid[1]=="term" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 996   returnval = "gallery";
 997  if(mlessvidid[1]=="g" and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 998   returnval = "link";
 999  if(mlessvidid[1]=="random" and len(mlessvidid)==3 and (mlessvidid[2]=="video" or mlessvidid[2]=="image")):
1000   returnval = "link";
1001  if(re.match("^V", mlessvidid[1]) and len(mlessvidid)==2):
1002   returnval = "board";
1003  if(re.match("^H", mlessvidid[1]) and len(mlessvidid)==2):
1004   returnval = "gallery";
1005  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==2):
1006   returnval = "gallery";
1007  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==3):
1008   returnval = "link";
1009  if(re.match("^g", mlessvidid[1]) and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
1010   returnval = "link";
1011  if(mlessvidid[1]=="members" and len(mlessvidid)==2):
1012   returnval = "member";
1013  if(mlessvidid[1]=="members" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
1014   returnval = "member";
1015  if(mlessvidid[1]=="members" and len(mlessvidid)==4 and (mlessvidid[2]=="uploader" or mlessvidid[2]=="viewed" or mlessvidid[2]=="social" or mlessvidid[2]=="favorited" or mlessvidid[2]=="commented" or mlessvidid[2]=="mentioned" or mlessvidid[2]=="verified")):
1016   returnval = "member";
1017  if(mlessvidid[1]=="girls" and len(mlessvidid)==2):
1018   returnval = "girls";
1019  if(mlessvidid[1]=="referers" and len(mlessvidid)==2):
1020   returnval = "referers";
1021  if(mlessvidid[1]=="about" and len(mlessvidid)==2):
1022   returnval = "team";
1023  if(mlessvidid_parts.netloc=="cdn.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.thumbs.motherlessmedia.com"):
1024   returnval = "download";
1025  if(mlessvidid_parts.netloc=="cdn1.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.thumbs.motherlessmedia.com"):
1026   returnval = "download";
1027  if(mlessvidid_parts.netloc=="cdn2.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.thumbs.motherlessmedia.com"):
1028   returnval = "download";
1029  if(mlessvidid_parts.netloc=="cdn3.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.thumbs.motherlessmedia.com"):
1030   returnval = "download";
1031  if(mlessvidid_parts.netloc=="cdn4.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.thumbs.motherlessmedia.com"):
1032   returnval = "download";
1033  if(returnval==False and len(mlessvidid)==2 and re.match("^([0-9A-F]+)$", mlessvidid[1])):
1034   returnval = "link";
1035  return returnval;
1036
1037 def get_motherless_link_type_alt(httpurl):
1038  httpurl = fix_motherless_url(httpurl);
1039  returnval = {'urlinfo': check_motherless_url(httpurl), 'motherlessinfo': get_motherless_link_type(httpurl)};
1040  return returnval;
1041
1042 def get_motherless_number_of_pages(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1043  httpurl = fix_motherless_url(httpurl);
1044  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1045  mrtext = premrtext['Content'];
1046  if(sys.version[0]>="3"):
1047   mrtext = mrtext.decode('ascii', 'replace');
1048  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1049  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1050  try:
1051   returnval = int(mlesspagenum[-1][0]);
1052  except:
1053   returnval = 1;
1054  return returnval;
1055
1056 def get_motherless_user_info(username, motherless_serv=None):
1057  motherless_subdomain = "cdn.";
1058  if(motherless_serv==None):
1059   motherless_serv = "";
1060  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1061   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1062  if(motherless_serv==""):
1063   motherless_subdomain = "cdn.";
1064  returnval = {'username': username};
1065  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg").path));
1066  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1067  returnval.update({'orginurl': "http://motherless.com/m/"+username});
1068  returnval.update({'orginurltype': "profile"});
1069  returnval.update({'url': "http://motherless.com/m/"+username});
1070  returnval.update({'urltype': "profile"});
1071  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg"});
1072  returnval.update({'avatarfullfilename': avatarfilenameext});
1073  returnval.update({'avatarfilename': avatarfilename});
1074  returnval.update({'avatarextension': avatarfileextension});
1075  return returnval;
1076
1077 def get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", motherless_serv=None):
1078  httpurl = fix_motherless_url(httpurl);
1079  if(get_motherless_link_type(httpurl)!="link"):
1080   returnval = False;
1081   return returnval;
1082  motherless_subdomain = "cdn.";
1083  if(motherless_serv==None):
1084   motherless_serv = "";
1085  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1086   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1087  if(motherless_serv==""):
1088   motherless_subdomain = "cdn.";
1089  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1090  mrtext = premrtext['Content'];
1091  if(sys.version[0]>="3"):
1092   mrtext = mrtext.decode('ascii', 'replace');
1093  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
1094  mrtext = re.sub(mregex_gettitle, "http://cdn.", mrtext);
1095  mregex_gettitle = re.escape("<title>")+"(.*)"+re.escape(" - MOTHERLESS.COM</title>");
1096  mlesstitle = re.findall(mregex_gettitle, mrtext);
1097  mregex_geturlone = re.escape("__fileurl = '")+'?\'?([^"\'>]*)'+re.escape("';");
1098  mlesslinkone = re.findall(mregex_geturlone, mrtext);
1099  mregex_geturlonetype = re.escape("http://cdn.")+"(images|videos)"+re.escape(".motherlessmedia.com/")+"(images|videos)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1100  mlesslinkonetype = re.findall(mregex_geturlonetype, mrtext);
1101  mregex_geturltwo = re.escape("<meta property=\"og:image\" content=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1102  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1103  mregex_geturltwotype = re.escape("http://cdn.")+"(images|thumbs)"+re.escape(".motherlessmedia.com/")+"(images|thumbs)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1104  mlesslinktwotype = re.findall(mregex_geturltwotype, mrtext);
1105  filenameext = os.path.basename(urlparse.urljoin(mlesslinkone[0], urlparse.urlparse(mlesslinkone[0]).path));
1106  filename, fileextension = os.path.splitext(filenameext);
1107  thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[0], urlparse.urlparse(mlesslinktwo[0]).path));
1108  thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1109  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1110  mlessuname = re.findall(mregex_getuname, mrtext);
1111  mlessuname = mlessuname[0];
1112  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1113  mlessurlname = re.findall(mregex_geturlname, mrtext);
1114  mlessurlname = mlessurlname[0].replace("/m/", "");
1115  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1116  mlessavatar = re.findall(mregex_getavatar, mrtext);
1117  mlessavatar = mlessavatar[0];
1118  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar, urlparse.urlparse(mlessavatar).path));
1119  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1120  mregex_getviews = re.escape("<strong>Views</strong>\n")+"(\s+|\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1121  mlessviews = re.findall(mregex_getviews, mrtext);
1122  mregex_getfavorited = re.escape("<strong>Favorited</strong>\n")+"(\s+\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1123  mlessfavorited = re.findall(mregex_getfavorited, mrtext);
1124  if(mlesslinkonetype[0][1]=="images"):
1125   thumbnailaltpart = thumbfilename+"-zoom"+thumbfileextension;
1126   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1127   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1128   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1129  if(mlesslinkonetype[0][1]=="videos"):
1130   thumbnailaltpart = thumbfilename+"-small"+thumbfileextension;
1131   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1132   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1133   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1134   stripimgpart = thumbfilename+"-strip"+thumbfileextension;
1135   stripimg = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+stripimgpart;
1136   stripimgfilenameext = os.path.basename(urlparse.urljoin(stripimg, urlparse.urlparse(stripimg).path));
1137   stripimgfilename, stripimgfileextension = os.path.splitext(stripimgfilenameext);
1138  returnval = False;
1139  mlessurltype = get_motherless_link_type(mlesslinkone[0]);
1140  if(mlesslinkonetype[0][1]=="images"):
1141   regex_getimgsize = re.escape("style=\"width: ")+"([0-9]+)"+re.escape("px; height: ")+"([0-9]+)"+re.escape("px;\"");
1142   mlessgetimgsize = re.findall(regex_getimgsize, mrtext);
1143   returnval = {'type': mlesslinkonetype[0][1], 'urltype': mlessurltype, 'url': mlesslinkone[0], 'height': int(mlessgetimgsize[0][1]), 'width': int(mlessgetimgsize[0][0]), 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1144  if(mlesslinkonetype[0][1]=="videos"):
1145   returnval = {'type': mlesslinkonetype[0][1], 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': thumbnailalt+"?from_helper", 'thumbnailalt': mlesslinktwo[0].replace("images", "thumbs"), 'strip': stripimg+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbnailaltpart, 'thumbfilename': thumbnailaltfilename, 'thumbextension': thumbnailaltfileextension, 'thumbnailaltfullfilename': thumbfilenameext, 'thumbnailaltfilename': thumbfilename, 'thumbnailaltextension': thumbfileextension, 'stripfullfilename': stripimgfilenameext, 'stripfilename': stripimgfilename, 'stripextension': stripimgfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1146  return returnval;
1147
1148 def get_motherless_links_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1149  httpurl = fix_motherless_url(httpurl);
1150  if(get_motherless_link_type(httpurl)!="download"):
1151   returnval = False;
1152   return returnval;
1153  returnval = False;
1154  if(get_motherless_link_type(httpurl)=="download"):
1155   urlparts = urlparse.urlparse(httpurl);
1156   filewithext = os.path.split(urlparts.path);
1157   if(filewithext[0]=="/images"):
1158    filewithoutext = os.path.splitext(filewithext[1])[0];
1159    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1160   if(filewithext[0]=="/thumbs"):
1161    filewithoutext = os.path.splitext(filewithext[1])[0];
1162    filewithoutext = filewithoutext.replace("-zoom", "");
1163    filewithoutext = filewithoutext.replace("-small", "");
1164    filewithoutext = filewithoutext.replace("-strip", "");
1165    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1166  return returnval;
1167
1168 def get_motherless_external_links(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1169  httpurl = fix_motherless_url(httpurl);
1170  if(get_motherless_link_type(httpurl)!="board" and get_motherless_link_type(httpurl)!="link"):
1171   returnval = False;
1172   return returnval;
1173  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1174  mrtext = premrtext['Content'];
1175  if(sys.version[0]>="3"):
1176   mrtext = mrtext.decode('ascii', 'replace');
1177  mregex_geturlinternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"motherless link\">");
1178  mlesslinkinternal = re.findall(mregex_geturlinternal, mrtext);
1179  mregex_geturlexternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"external link\">");
1180  mlesslinkexternal = re.findall(mregex_geturlexternal, mrtext);
1181  returnvalone = None;
1182  if(len(mlesslinkinternal)>1):
1183   mli = 0;
1184   mlil = len(mlesslinkinternal);
1185   returnvalone = {'numoflinks': mlil};
1186   returnvalone.update({'numofalllinks': len(mlesslinkinternal)});
1187   returnvalone.update({'orginurl': httpurl});
1188   returnvalone.update({'orginurltype': get_motherless_link_type(httpurl)});
1189   mlessrooturltype = get_motherless_link_type(httpurl);
1190   returnvalone.update({'urltype': mlessrooturltype});
1191   while(mli<mlil):
1192    mlessurltype = get_motherless_link_type(mlesslinkinternal[mli]);
1193    returnvalone.update({mli: {'urltype': mlessurltype, 'url': mlesslinkinternal[mli]} });
1194    mli = mli + 1;
1195  returnvaltwo = None;
1196  if(len(mlesslinkexternal)>1):
1197   mli = 0;
1198   mlil = len(mlesslinkexternal);
1199   returnvaltwo = {'numoflinks': mlil};
1200   returnvaltwo.update({'numofalllinks': len(mlesslinkexternal)});
1201   returnvaltwo.update({'orginurl': httpurl});
1202   returnvaltwo.update({'orginurltype': get_motherless_link_type(httpurl)});
1203   mlessrooturltype = get_motherless_link_type(httpurl);
1204   returnvaltwo.update({'urltype': mlessrooturltype});
1205   while(mli<mlil):
1206    returnvaltwo.update({mli: {'urltype': "external", 'url': mlesslinkexternal[mli]} });
1207    mli = mli + 1;
1208  if(returnvalone==None and returnvaltwo==None):
1209   returnval = {'internal': None, 'external': None};
1210  if(not returnvalone==None and not returnvaltwo==None):
1211   returnval = {'internal': returnvalone, 'external': returnvaltwo};
1212  if(not returnvalone==None and returnvaltwo==None):
1213   returnval = {'internal': returnvalone, 'external': None};
1214  if(returnvalone==None and not returnvaltwo==None):
1215   returnval = {'internal': None, 'external': returnvaltwo};
1216  return returnval;
1217
1218 def get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1], motherless_serv=None):
1219  httpurl = fix_motherless_url(httpurl);
1220  if(get_motherless_link_type(httpurl)!="gallery"):
1221   returnval = False;
1222   return returnval;
1223  motherless_subdomain = "cdn.";
1224  if(motherless_serv==None):
1225   motherless_serv = "";
1226  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1227   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1228  if(motherless_serv==""):
1229   motherless_subdomain = "cdn.";
1230  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1231  mrtext = premrtext['Content'];
1232  if(sys.version[0]>="3"):
1233   mrtext = mrtext.decode('ascii', 'replace');
1234  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1235  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1236  try:
1237   lastpage = int(mlesspagenum[-1][0]);
1238  except:
1239   lastpage = 1;
1240  if(page>lastpage):
1241   page = lastpage;
1242  httpurl = add_url_param(httpurl, page=str(page));
1243  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1244  mrtext = premrtext['Content'];
1245  if(sys.version[0]>="3"):
1246   mrtext = mrtext.decode('ascii', 'replace');
1247  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"img-container\" target=\"_self\">");
1248  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1249  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1250  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1251  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1252  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1253  mregex_geturltwo = re.escape("<img class=\"static\" src=\"")+'?\'?([^"\'>]*)'+re.escape("\" data-strip-src=\"")+'?\'?([^"\'>]*)'+re.escape("\" alt=\"")+'?\'?([^">]*)'+re.escape("\" />");
1254  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1255  mregex_getcaption = re.escape("<h2 class=\"caption title\">")+"(.*)"+re.escape("</h2>");
1256  mlesscaption = re.findall(mregex_getcaption, mrtext);
1257  mregex_gethits = re.escape("<div class=\"caption right\">")+"([0-9\,]+)"+re.escape(" Hits</div>");
1258  mlesshits = re.findall(mregex_gethits, mrtext);
1259  if(len(mlesshits)<1):
1260   mregex_gethits = re.escape("<div class=\"caption right\">\n        ")+"([0-9\,]+)"+re.escape(" Hits\n    </div>");
1261   mlesshits = re.findall(mregex_gethits, mrtext);
1262  mregex_getuserinfo = re.escape("<a class=\"caption left\" href=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1263  mlessuname = re.findall(mregex_getuserinfo, mrtext);
1264  mregex_getmediainfo = re.escape("data-size=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-is-premium=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-codename=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-mediatype=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-gallery-codename=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-frames=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1265  mlessmediainfo = re.findall(mregex_getmediainfo, mrtext);
1266  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1267   getlinks[1] = len(mlesslinkone);
1268  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1269   tmpgetlinks0 = getlinks[0];
1270   tmpgetlinks1 = getlinks[1];
1271   getlinks[0] = tmpgetlinks1;
1272   getlinks[1] = tmpgetlinks0;
1273  if(getlinks[0]<0):
1274   getlinks[0] = 0;
1275  mli = getlinks[0];
1276  mlil = getlinks[1];
1277  returnval = {'pages': lastpage};
1278  returnval.update({'curpage': page});
1279  returnval.update({'numoflinks': mlil});
1280  returnval.update({'numofalllinks': len(mlesslinkone)});
1281  returnval.update({'orginurl': httpurl});
1282  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1283  mlessrooturltype = get_motherless_link_type(httpurl);
1284  returnval.update({'urltype': mlessrooturltype});
1285  while(mli<mlil):
1286   mlessuname[mli] = mlessuname[mli].replace("/m/", "");
1287   stripfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1288   stripfilename, stripfileextension = os.path.splitext(stripfilenameext);
1289   thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][0], urlparse.urlparse(mlesslinktwo[mli][0]).path));
1290   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1291   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlesslinkone[mli]);
1292   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1293   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1294   if(mlessmediainfo[mli][3]=="image"):
1295    returnval.update({mli: {'type': "images", 'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'thumbnailalt': mlesslinktwo[mli][1], 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': stripfilenameext, 'thumbnailaltfilename': stripfilename, 'thumbnailaltextension': stripfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1296   if(mlessmediainfo[mli][3]=="video"):
1297    altthumbnail = mlesslinktwo[mli][1].replace('-strip', '');
1298    altfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1299    altfilename, altfileextension = os.path.splitext(altfilenameext);
1300    returnval.update({mli: {'type': "videos", 'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'strip': mlesslinktwo[mli][1], 'thumbnailalt': altthumbnail, 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'stripfullfilename': stripfilenameext, 'stripfilename': stripfilename, 'stripextension': stripfileextension, 'thumbnailaltfullfilename': altfilenameext, 'thumbnailaltfilename': altfilename, 'thumbnailaltextension': altfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1301   mli = mli + 1;
1302  return returnval;
1303
1304 def get_motherless_random_links(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1305  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1306   tmpgetlinks0 = getlinks[0];
1307   tmpgetlinks1 = getlinks[1];
1308   getlinks[0] = tmpgetlinks1;
1309   getlinks[1] = tmpgetlinks0;
1310  if(getlinks[0]<0):
1311   getlinks[0] = 0;
1312  mli = getlinks[0];
1313  mlil = getlinks[1];
1314  if(linktype=="image"):
1315   returnval = {'pages': 1};
1316   returnval.update({'curpage': 1});
1317   returnval.update({'numoflinks': 80});
1318   returnval.update({'numofalllinks': mlil});
1319   returnval.update({'orginurl': "http://motherless.com/random/image"});
1320   returnval.update({'orginurltype': "gallery"});
1321   returnval.update({'urltype': "gallery"});
1322   while(mli<mlil):
1323    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1324    returnval.update({mli: {'type': "images", 'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1325    mli = mli + 1;
1326  if(linktype=="video"):
1327   returnval = {'pages': 1};
1328   returnval.update({'curpage': 1});
1329   returnval.update({'numoflinks': 80});
1330   returnval.update({'numofalllinks': mlil});
1331   returnval.update({'orginurl': "http://motherless.com/random/video"});
1332   returnval.update({'orginurltype': "gallery"});
1333   returnval.update({'urltype': "gallery"});
1334   while(mli<mlil):
1335    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1336    returnval.update({mli: {'type': "videos", 'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1337    mli = mli + 1;
1338  return returnval;
1339
1340 def get_motherless_random_links_alt(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1341  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1342   tmpgetlinks0 = getlinks[0];
1343   tmpgetlinks1 = getlinks[1];
1344   getlinks[0] = tmpgetlinks1;
1345   getlinks[1] = tmpgetlinks0;
1346  if(getlinks[0]<0):
1347   getlinks[0] = 0;
1348  mli = getlinks[0];
1349  mlil = getlinks[1];
1350  if(linktype=="image"):
1351   returnval = {'pages': 1};
1352   returnval.update({'curpage': 1});
1353   returnval.update({'numoflinks': 80});
1354   returnval.update({'numofalllinks': mlil});
1355   returnval.update({'orginurl': "http://motherless.com/random/image"});
1356   returnval.update({'orginurltype': "gallery"});
1357   returnval.update({'urltype': "gallery"});
1358   while(mli<mlil):
1359    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1360    returnval.update({mli: get_links});
1361    mli = mli + 1;
1362  if(linktype=="video"):
1363   returnval = {'pages': 1};
1364   returnval.update({'curpage': 1});
1365   returnval.update({'numoflinks': 80});
1366   returnval.update({'numofalllinks': mlil});
1367   returnval.update({'orginurl': "http://motherless.com/random/video"});
1368   returnval.update({'orginurltype': "gallery"});
1369   returnval.update({'urltype': "gallery"});
1370   while(mli<mlil):
1371    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1372    returnval.update({mli: get_links});
1373    mli = mli + 1;
1374  return returnval;
1375
1376 def get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", geturlinfo=False, getlinks=[0, -1], motherless_serv=None):
1377  httpurl = fix_motherless_url(httpurl);
1378  if(get_motherless_link_type(httpurl)!="board"):
1379   returnval = False;
1380   return returnval;
1381  motherless_subdomain = "cdn.";
1382  if(motherless_serv==None):
1383   motherless_serv = "";
1384  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1385   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1386  if(motherless_serv==""):
1387   motherless_subdomain = "cdn.";
1388  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1389  mrtext = premrtext['Content'];
1390  if(sys.version[0]>="3"):
1391   mrtext = mrtext.decode('ascii', 'replace');
1392  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1393  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1394  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1395  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1396  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" title=\"motherless link\">");
1397  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1398  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1399  mlessurlname = re.findall(mregex_geturlname, mrtext);
1400  mregex_getdatasrc = re.escape("data-src=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1401  mlessdatasrc = re.findall(mregex_getdatasrc, mrtext_tmp);
1402  print(mlessdatasrc);
1403  mlessavaturl = "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessurlname[0][1]+"-avatar.jpg";
1404  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavaturl, urlparse.urlparse(mlessavaturl).path));
1405  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1406  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1407   getlinks[1] = len(mlesslinkone);
1408  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1409   tmpgetlinks0 = getlinks[0];
1410   tmpgetlinks1 = getlinks[1];
1411   getlinks[0] = tmpgetlinks1;
1412   getlinks[1] = tmpgetlinks0;
1413  if(getlinks[0]<0):
1414   getlinks[0] = 0;
1415  mli = getlinks[0];
1416  mlil = getlinks[1];
1417  returnval = {'numoflinks': mlil};
1418  returnval.update({'numofalllinks': len(mlesslinkone)});
1419  returnval.update({'orginurl': httpurl});
1420  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1421  mlessrooturltype = get_motherless_link_type(httpurl);
1422  returnval.update({'urltype': mlessrooturltype});
1423  returnval.update({'userinfo': get_motherless_user_info(mlessurlname[0][1])});
1424  returnval.update({'username': mlessurlname[0][1]});
1425  returnval.update({'avatarurl': mlessavaturl});
1426  returnval.update({'avatarfullfilename': avatarfilenameext});
1427  returnval.update({'avatarfilename': avatarfilename});
1428  returnval.update({'avatarextension': avatarfileextension});
1429  while(mli<mlil):
1430   mlessurltype = get_motherless_link_type("http://motherless.com"+mlesslinkone[mli]);
1431   mlessdlurltype = get_motherless_link_type(mlessdatasrc[mli]);
1432   if(geturlinfo==False):
1433    returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'downloadurl': mlessdatasrc[mli], 'downloadurltype': mlessdlurltype} });
1434   if(geturlinfo==True):
1435    returnval.update({mli: {'urltype': mlessurltype, 'urlinfo': get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse, motherless_serv), 'url': "http://motherless.com"+mlesslinkone[mli], 'downloadurl': mlessdatasrc[mli], 'downloadurltype': mlessdlurltype} });
1436   mli = mli + 1;
1437  return returnval;
1438
1439 def get_motherless_boards_posts(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1440  httpurl = fix_motherless_url(httpurl);
1441  if(get_motherless_link_type(httpurl)!="board"):
1442   returnval = False;
1443   return returnval;
1444  motherless_subdomain = "cdn.";
1445  if(motherless_serv==None):
1446   motherless_serv = "";
1447  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1448   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1449  if(motherless_serv==""):
1450   motherless_subdomain = "cdn.";
1451  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1452  mrtext = premrtext['Content'];
1453  if(sys.version[0]>="3"):
1454   mrtext = mrtext.decode('ascii', 'replace');
1455  mregex_getposts = "(\t+)"+re.escape("<p>")+"(.+?)"+re.escape("</p>")+"(\t+)";
1456  mlessposts = re.findall(mregex_getposts, mrtext);
1457  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1458  mlessuname = re.findall(mregex_getuname, mrtext);
1459  mregex_getopuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1460  mlessopuname = re.findall(mregex_getopuname, mrtext);
1461  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1462   getposts[1] = len(mlessposts);
1463  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1464   tmpgetposts0 = getposts[0];
1465   tmpgetposts1 = getposts[1];
1466   getposts[0] = tmpgetposts1;
1467   getposts[1] = tmpgetposts0;
1468  if(getposts[0]<0):
1469   getposts[0] = 0;
1470  mli = getposts[0];
1471  mlil = getposts[1];
1472  returnval = {'numofposts': mlil};
1473  returnval.update({'numofallposts': len(mlessposts)});
1474  returnval.update({'orginurl': httpurl});
1475  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1476  mlessrooturltype = get_motherless_link_type(httpurl);
1477  returnval.update({'urltype': mlessrooturltype});
1478  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0][1])});
1479  returnval.update({'username': mlessopuname[0][1]});
1480  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg").path));
1481  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1482  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg"});
1483  returnval.update({'avatarfullfilename': avatarfilenameext});
1484  returnval.update({'avatarfilename': avatarfilename});
1485  returnval.update({'avatarextension': avatarfileextension});
1486  while(mli<mlil):
1487   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1488   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1489   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1490   mli = mli + 1;
1491  return returnval;
1492
1493 def get_motherless_links_comments(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1494  httpurl = fix_motherless_url(httpurl);
1495  if(get_motherless_link_type(httpurl)!="link"):
1496   returnval = False;
1497   return returnval;
1498  motherless_subdomain = "cdn.";
1499  if(motherless_serv==None):
1500   motherless_serv = "";
1501  if(motherless_serv!="" and (motherless_serv<5 or motherless_serv>0)):
1502   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1503  if(motherless_serv==""):
1504   motherless_subdomain = "cdn.";
1505  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1506  mrtext = premrtext['Content'];
1507  if(sys.version[0]>="3"):
1508   mrtext = mrtext.decode('ascii', 'replace');
1509  mregex_getposts = re.escape("<div style=\"text-align: justify;\">\n")+"(\t+)(.+?)"+re.escape("</div>");
1510  mlessposts = re.findall(mregex_getposts, mrtext);
1511  mregex_getuname = re.escape("<div class=\"media-comment\" id=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"0\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1512  mlessuname = re.findall(mregex_getuname, mrtext);
1513  mregex_getopuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1514  mlessopuname = re.findall(mregex_getopuname, mrtext);
1515  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1516   getposts[1] = len(mlessposts);
1517  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1518   tmpgetposts0 = getposts[0];
1519   tmpgetposts1 = getposts[1];
1520   getposts[0] = tmpgetposts1;
1521   getposts[1] = tmpgetposts0;
1522  if(getposts[0]<0):
1523   getposts[0] = 0;
1524  mli = getposts[0];
1525  mlil = getposts[1];
1526  returnval = {'numofposts': mlil};
1527  returnval.update({'numofallposts': len(mlessposts)});
1528  returnval.update({'orginurl': httpurl});
1529  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1530  mlessrooturltype = get_motherless_link_type(httpurl);
1531  returnval.update({'urltype': mlessrooturltype});
1532  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0])});
1533  returnval.update({'username': mlessopuname[0]});
1534  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg").path));
1535  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1536  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg"});
1537  returnval.update({'avatarfullfilename': avatarfilenameext});
1538  returnval.update({'avatarfilename': avatarfilename});
1539  returnval.update({'avatarextension': avatarfileextension});
1540  while(mli<mlil):
1541   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg").path));
1542   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1543   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1544   mli = mli + 1;
1545  return returnval;
1546
1547 def get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1548  httpurl = fix_motherless_url(httpurl);
1549  if(get_motherless_link_type(httpurl)!="member"):
1550   returnval = False;
1551   return returnval;
1552  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1553  mrtext = premrtext['Content'];
1554  if(sys.version[0]>="3"):
1555   mrtext = mrtext.decode('ascii', 'replace');
1556  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1557  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1558  try:
1559   lastpage = int(mlesspagenum[-1][0]);
1560  except:
1561   lastpage = 1;
1562  if(page>lastpage):
1563   page = lastpage;
1564  httpurl = add_url_param(httpurl, page=str(page));
1565  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1566  mrtext = premrtext['Content'];
1567  if(sys.version[0]>="3"):
1568   mrtext = mrtext.decode('ascii', 'replace');
1569  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1570  mlessuname = re.findall(mregex_getuname, mrtext);
1571  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1572  mlessurlname = re.findall(mregex_geturlname, mrtext);
1573  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1574  mlessavatar = re.findall(mregex_getavatar, mrtext);
1575  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1576   getlinks[1] = len(mlessuname);
1577  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1578   tmpgetlinks0 = getlinks[0];
1579   tmpgetlinks1 = getlinks[1];
1580   getlinks[0] = tmpgetlinks1;
1581   getlinks[1] = tmpgetlinks0;
1582  if(getlinks[0]<0):
1583   getlinks[0] = 0;
1584  mli = getlinks[0];
1585  mlil = getlinks[1];
1586  returnval = {'numoflinks': mlil};
1587  returnval.update({'numofalllinks': len(mlessuname)});
1588  returnval.update({'pages': lastpage});
1589  returnval.update({'curpage': page});
1590  returnval.update({'orginurl': httpurl});
1591  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1592  mlessrooturltype = get_motherless_link_type(httpurl);
1593  returnval.update({'urltype': mlessrooturltype});
1594  while(mli<mlil):
1595   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli], urlparse.urlparse(mlessavatar[mli]).path));
1596   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1597   mlessurlname[mli] = mlessurlname[mli].replace("/m/", "");
1598   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessurlname[mli]);
1599   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessurlname[mli], 'userinfo': get_motherless_user_info(mlessurlname[mli]), 'username': mlessuname[mli], 'avatarurl': mlessavatar[mli], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1600   mli = mli + 1;
1601  return returnval;
1602
1603 def get_motherless_girls(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1604  premrtext = download_from_url("http://motherless.com/girls", httpheaders, httpcookie, httplibuse);
1605  mrtext = premrtext['Content'];
1606  if(sys.version[0]>="3"):
1607   mrtext = mrtext.decode('ascii', 'replace');
1608  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1609  mlessuname = re.findall(mregex_getuname, mrtext);
1610  mregex_geturlname = re.escape("\n\t\t\t\t\t\t<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">");
1611  mlessurlname = re.findall(mregex_geturlname, mrtext);
1612  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1613   getlinks[1] = len(mlessuname);
1614  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1615   tmpgetlinks0 = getlinks[0];
1616   tmpgetlinks1 = getlinks[1];
1617   getlinks[0] = tmpgetlinks1;
1618   getlinks[1] = tmpgetlinks0;
1619  if(getlinks[0]<0):
1620   getlinks[0] = 0;
1621  mli = getlinks[0];
1622  mlil = getlinks[1];
1623  returnval = {'numoflinks': mlil};
1624  returnval.update({'numofalllinks': len(mlessuname)});
1625  returnval.update({'orginurl': "http://motherless.com/girls"});
1626  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/girls")});
1627  mlessrooturltype = get_motherless_link_type("http://motherless.com/girls");
1628  returnval.update({'urltype': mlessrooturltype});
1629  while(mli<mlil):
1630   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessuname[mli][0], urlparse.urlparse(mlessuname[mli][0]).path));
1631   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1632   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname[mli][1]);
1633   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': mlessuname[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1634   mli = mli + 1;
1635  return returnval;
1636
1637 def get_motherless_team(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1638  premrtext = download_from_url("http://motherless.com/about", httpheaders, httpcookie, httplibuse);
1639  mrtext = premrtext['Content'];
1640  if(sys.version[0]>="3"):
1641   mrtext = mrtext.decode('ascii', 'replace');
1642  mregex_getuname = re.escape("<div class=\"about-us-member\">\n                         <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1643  mlessuname = re.findall(mregex_getuname, mrtext);
1644  mlessuname_odd = mlessuname[1::2];
1645  mlessuname_even = mlessuname[::2];
1646  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1647  mlessavatar = re.findall(mregex_getavatar, mrtext);
1648  if(getlinks[1]>len(mlessuname_odd) or getlinks[1]==-1):
1649   getlinks[1] = len(mlessuname_odd);
1650  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1651   tmpgetlinks0 = getlinks[0];
1652   tmpgetlinks1 = getlinks[1];
1653   getlinks[0] = tmpgetlinks1;
1654   getlinks[1] = tmpgetlinks0;
1655  if(getlinks[0]<0):
1656   getlinks[0] = 0;
1657  mli = getlinks[0];
1658  mlil = getlinks[1];
1659  returnval = {'numoflinks': mlil};
1660  returnval.update({'numofalllinks': len(mlessuname)});
1661  returnval.update({'orginurl': "http://motherless.com/about"});
1662  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/about")});
1663  mlessrooturltype = get_motherless_link_type("http://motherless.com/about");
1664  returnval.update({'urltype': mlessrooturltype});
1665  while(mli<mlil):
1666   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1667   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1668   mlessuname_odd[mli] = mlessuname_odd[mli].replace("/m/", "");
1669   mlessuname_even[mli] = mlessuname_even[mli].replace("/m/", "");
1670   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname_odd[mli]);
1671   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname_odd[mli], 'userinfo': get_motherless_user_info(mlessuname_odd[mli]), 'username': mlessuname_odd[mli], 'avatarurl': mlessavatar[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1672   mli = mli + 1;
1673  return returnval;
1674
1675 def get_motherless_top_referrers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1676  premrtext = download_from_url("http://motherless.com/referers", httpheaders, httpcookie, httplibuse);
1677  mrtext = premrtext['Content'];
1678  if(sys.version[0]>="3"):
1679   mrtext = mrtext.decode('ascii', 'replace');
1680  mregex_geturlname = "([0-9]+)"+re.escape(". <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"pop\" target=\"_blank\" rel=\"nofollow\">\n                                            ")+"(.*)"+re.escape("                                   </a>");
1681  mlessurlname = re.findall(mregex_geturlname, mrtext);
1682  if(getlinks[1]>len(mlessurlname) or getlinks[1]==-1):
1683   getlinks[1] = len(mlessurlname);
1684  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1685   tmpgetlinks0 = getlinks[0];
1686   tmpgetlinks1 = getlinks[1];
1687   getlinks[0] = tmpgetlinks1;
1688   getlinks[1] = tmpgetlinks0;
1689  if(getlinks[0]<0):
1690   getlinks[0] = 0;
1691  mli = getlinks[0];
1692  mlil = getlinks[1];
1693  returnval = {'numoflinks': mlil};
1694  returnval.update({'numofalllinks': len(mlessurlname)});
1695  returnval.update({'orginurl': "http://motherless.com/referers"});
1696  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/referers")});
1697  mlessrooturltype = get_motherless_link_type("http://motherless.com/referers");
1698  returnval.update({'urltype': mlessrooturltype});
1699  while(mli<mlil):
1700   returnval.update({mli: {'urltype': "referer-links", 'url': mlessurlname[mli][1], 'title': mlessurlname[mli][2]} });
1701   mli = mli + 1;
1702  return returnval;
1703
1704 def get_motherless_top_referers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1705  return get_motherless_top_referrers(httpheaders, httpcookie, httplibuse, getlinks);
1706
1707 def get_motherless_groups(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1708  httpurl = fix_motherless_url(httpurl);
1709  if(get_motherless_link_type(httpurl)!="group"):
1710   returnval = False;
1711   return returnval;
1712  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1713  mrtext = premrtext['Content'];
1714  if(sys.version[0]>="3"):
1715   mrtext = mrtext.decode('ascii', 'replace');
1716  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1717  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1718  try:
1719   lastpage = int(mlesspagenum[-1][0]);
1720  except:
1721   lastpage = 1;
1722  if(page>lastpage):
1723   page = lastpage;
1724  httpurl = add_url_param(httpurl, page=str(page));
1725  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1726  mrtext = premrtext['Content'];
1727  if(sys.version[0]>="3"):
1728   mrtext = mrtext.decode('ascii', 'replace');
1729  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1730  mlessavatar = re.findall(mregex_getavatar, mrtext);
1731  mregex_getgroups = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"grunge motherless-red\">\n")+"(.*)"+re.escape("</a>");
1732  mlessgroups = re.findall(mregex_getgroups, mrtext);
1733  if(getlinks[1]>len(mlessgroups) or getlinks[1]==-1):
1734   getlinks[1] = len(mlessgroups);
1735  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1736   tmpgetlinks0 = getlinks[0];
1737   tmpgetlinks1 = getlinks[1];
1738   getlinks[0] = tmpgetlinks1;
1739   getlinks[1] = tmpgetlinks0;
1740  if(getlinks[0]<0):
1741   getlinks[0] = 0;
1742  mli = getlinks[0];
1743  mlil = getlinks[1];
1744  returnval = {'pages': lastpage};
1745  returnval.update({'curpage': page});
1746  returnval.update({'numoflinks': mlil});
1747  returnval.update({'numofalllinks': len(mlessgroups)});
1748  returnval.update({'orginurl': httpurl});
1749  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1750  while(mli<mlil):
1751   thumbfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1752   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1753   returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0]), 'url': "http://motherless.com"+mlessgroups[mli][0], 'urltype-image': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gi/")), 'url-image': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gi/"), 'urltype-video': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gv/")), 'url-video': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gv/"), 'urltype-member': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gm/")), 'url-member': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gm/"), 'thumbnail': mlessavatar[mli][0], 'title': mlessgroups[mli][1].strip(), 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension} });
1754   mli = mli + 1;
1755  return returnval;
1756
1757 def get_motherless_sample_links(httpheaders, httpcookie, httplibuse="urllib", numoflinks=10, urltype="video", enablerand=True):
1758  if(urltype=="video" and enablerand==True):
1759   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) } };
1760  if(urltype=="video" and enablerand==False):
1761   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1762  if(urltype=="image" and enablerand==True):
1763   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) } };
1764  if(urltype=="image" and enablerand==False):
1765   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1766  if(urltype=="gallery"):
1767   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/galleries", 'orginurltype': get_motherless_link_type("http://motherless.com/galleries"), 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1768  if(urltype=="all" and enablerand==True):
1769   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1770  if(urltype=="all" and enablerand==False):
1771   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1772  return returnval;
1773
1774 def get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1], motherless_serv=None):
1775  httpurl = fix_motherless_url(httpurl);
1776  returnval = False;
1777  if(get_motherless_link_type(httpurl)=="link"):
1778   returnval = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse, motherless_serv);
1779  if(get_motherless_link_type(httpurl)=="gallery"):
1780   returnval = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks, motherless_serv);
1781  if(get_motherless_link_type(httpurl)=="sample-videos"):
1782   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "video");
1783  if(get_motherless_link_type(httpurl)=="sample-images"):
1784   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "image");
1785  if(get_motherless_link_type(httpurl)=="sample-galleries"):
1786   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "gallery");
1787  if(get_motherless_link_type(httpurl)=="sample" or get_motherless_link_type(httpurl)=="sample-all"):
1788   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "all");
1789  if(get_motherless_link_type(httpurl)=="board"):
1790   returnval = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse, False, getlinks, motherless_serv);
1791  if(get_motherless_link_type(httpurl)=="member"):
1792   returnval = get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks);
1793  if(get_motherless_link_type(httpurl)=="girls"):
1794   returnval = get_motherless_girls(httpheaders, httpcookie, httplibuse, getlinks);
1795  if(get_motherless_link_type(httpurl)=="download"):
1796   returnval = get_motherless_links_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1797  return returnval;
1798
1799 def view_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", viewerpro="mpv", prearg=[], proarg=[]):
1800  httpurl = fix_motherless_url(httpurl);
1801  commandlist = [viewerpro] + prearg;
1802  commandlist = commandlist + [get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse)['url']];
1803  commandlist = commandlist + proarg;
1804  mpvplaylistp = subprocess.Popen(commandlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
1805  mpvplayout, mpvplayerr = mpvplaylistp.communicate();
1806  return True;
1807
1808 def download_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False):
1809  global geturls_download_sleep;
1810  httpurl = fix_motherless_url(httpurl);
1811  if(sleep<0):
1812   sleep = geturls_download_sleep;
1813  mlessurl = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1814  outputname = mlessurl['fullfilename'];
1815  outpath = outpath.rstrip(os.path.sep);
1816  if(usetitlename==True):
1817   outputname = mlessurl['title'];
1818  if(usetitlename=="-" and outfile=="-"):
1819   outputname = "-";
1820  if(usetitlename=="-" and not outfile=="-"):
1821   outputname = outfile;
1822  returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep);
1823  return returnval;
1824
1825 def download_motherless_links_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1], motherless_serv=None):
1826  global geturls_download_sleep;
1827  httpurl = fix_motherless_url(httpurl);
1828  if(sleep<0):
1829   sleep = geturls_download_sleep;
1830  mlessurl = get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks, motherless_serv);
1831  if(mlessurl['urltype']=="download"):
1832   outputname = mlessurl['fullfilename'];
1833   outpathname = outpath.rstrip(os.path.sep);
1834   if(usetitlename==True):
1835    outputname = mlessurl['title'];
1836   if(usetitlename=="-" and outfile[mli]=="-"):
1837    outputname = "-";
1838   if(usetitlename=="-" and not outfile[mli]=="-"):
1839    outputname = outfile;
1840   returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpathname, buffersize, sleep);
1841  if(not mlessurl['urltype']=="download"):
1842   returnval = mlessurl;
1843  return returnval;
1844
1845 def download_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1], motherless_serv=None):
1846  global geturls_download_sleep;
1847  httpurl = fix_motherless_url(httpurl);
1848  if(sleep<0):
1849   sleep = geturls_download_sleep;
1850  mlessgalleries = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks, motherless_serv);
1851  mli = 0;
1852  mlil = mlessgalleries['numoflinks'];
1853  returnval = {'pages': mlessgalleries['pages']};
1854  returnval.update({'numoflists': mlessgalleries['numoflinks']});
1855  returnval.update({'curpage': mlessgalleries['curpage']});
1856  returnval.update({'numoflinks': mlessgalleries['numoflinks']});
1857  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1858  returnval.update({'orginurl': httpurl});
1859  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1860  while(mli<mlil):
1861   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1862   outputname = mlesslink['fullfilename'];
1863   outpath = outpath.rstrip(os.path.sep);
1864   if(usetitlename==True):
1865    outputname = mlesslink['title'];
1866   if(usetitlename=="-" and outfile=="-"):
1867    outputname = "-";
1868   if(usetitlename=="-" and not outfile=="-"):
1869    outputname = outfile;
1870   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1871   mli = mli + 1;
1872  return returnval;
1873
1874 def download_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, getlinks=[0, -1], motherless_serv=None):
1875  global geturls_download_sleep;
1876  httpurl = fix_motherless_url(httpurl);
1877  if(sleep<0):
1878   sleep = geturls_download_sleep;
1879  mlessgalleries = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse, False, getlinks, motherless_serv);
1880  mli = 0;
1881  mlil = mlessgalleries['numoflinks'];
1882  returnval = {'numoflists': mlessgalleries['numoflinks']};
1883  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1884  returnval.update({'orginurl': httpurl});
1885  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1886  while(mli<mlil):
1887   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1888   outputname = mlesslink['fullfilename'];
1889   outpath = outpath.rstrip(os.path.sep);
1890   if(usetitlename==True):
1891    outputname = mlesslink['title'];
1892   if(usetitlename=="-" and outfile=="-"):
1893    outputname = "-";
1894   if(usetitlename=="-" and not outfile=="-"):
1895    outputname = outfile;
1896   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1897   mli = mli + 1;
1898  return returnval;