lib/snoopy/Snoopy.class.inc

   1 <?php
   2
   3 /*************************************************
   4
   5 Snoopy - the PHP net client
   6 Author: Monte Ohrt <monte@ispi.net>
   7 Copyright (c): 1999-2000 ispi, all rights reserved
   8 Version: 1.01
   9 (Note: v1.2.3 - var $agent="Snoopy v1.2.3";// agent we masquerade as)
  10
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; either
  14  * version 2.1 of the License, or (at your option) any later version.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  24
  25 You may contact the author of Snoopy by e-mail at:
  26 monte@ispi.net
  27
  28 Or, write to:
  29 Monte Ohrt
  30 CTO, ispi
  31 237 S. 70th suite 220
  32 Lincoln, NE 68510
  33
  34 The latest version of Snoopy can be obtained from:
  35 http://snoopy.sourceforge.net/
  36
  37 *************************************************/
  38
  39 class Snoopy
  40 {
  41     /**** Public variables ****/
  42
  43     /* user definable vars */
  44
  45     var $host            =    "www.php.net";        // host name we are connecting to
  46     var $port            =    80;                    // port we are connecting to
  47     var $proxy_host        =    "";                    // proxy host to use
  48     var $proxy_port        =    "";                    // proxy port to use
  49     var $proxy_user        =    "";                    // proxy user to use
  50     var $proxy_pass        =    "";                    // proxy password to use
  51
  52     var $agent            =    "Snoopy v1.2.3";    // agent we masquerade as
  53     var    $referer        =    "";                    // referer info to pass
  54     var $cookies        =    array();            // array of cookies to pass
  55                                                 // $cookies["username"]="joe";
  56     var    $rawheaders        =    array();            // array of raw headers to send
  57                                                 // $rawheaders["Content-type"]="text/html";
  58
  59     var $maxredirs        =    5;                    // http redirection depth maximum. 0 = disallow
  60     var $lastredirectaddr    =    "";                // contains address of last redirected address
  61     var    $offsiteok        =    true;                // allows redirection off-site
  62     var $maxframes        =    0;                    // frame content depth maximum. 0 = disallow
  63     var $expandlinks    =    true;                // expand links to fully qualified URLs.
  64                                                 // this only applies to fetchlinks()
  65                                                 // submitlinks(), and submittext()
  66     var $passcookies    =    true;                // pass set cookies back through redirects
  67                                                 // NOTE: this currently does not respect
  68                                                 // dates, domains or paths.
  69
  70     var    $user            =    "";                    // user for http authentication
  71     var    $pass            =    "";                    // password for http authentication
  72
  73     // http accept types
  74     var $accept            =    "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*";
  75
  76     var $results        =    "";                    // where the content is put
  77
  78     var $error            =    "";                    // error messages sent here
  79     var    $response_code    =    "";                    // response code returned from server
  80     var    $headers        =    array();            // headers returned from server sent here
  81     var    $maxlength        =    500000;                // max return data length (body)
  82     var $read_timeout    =    0;                    // timeout on read operations, in seconds
  83                                                 // supported only since PHP 4 Beta 4
  84                                                 // set to 0 to disallow timeouts
  85     var $timed_out        =    false;                // if a read operation timed out
  86     var    $status            =    0;                    // http request status
  87
  88     var $temp_dir        =    "/tmp";                // temporary directory that the webserver
  89                                                 // has permission to write to.
  90                                                 // under Windows, this should be C:\temp
  91
  92     var    $curl_path        =    "/usr/local/bin/curl";
  93                                                 // Snoopy will use cURL for fetching
  94                                                 // SSL content if a full system path to
  95                                                 // the cURL binary is supplied here.
  96                                                 // set to false if you do not have
  97                                                 // cURL installed. See http://curl.haxx.se
  98                                                 // for details on installing cURL.
  99                                                 // Snoopy does *not* use the cURL
 100                                                 // library functions built into php,
 101                                                 // as these functions are not stable
 102                                                 // as of this Snoopy release.
 103
 104     /**** Private variables ****/
 105
 106     var    $_maxlinelen    =    4096;                // max line length (headers)
 107
 108     var $_httpmethod    =    "GET";                // default http request method
 109     var $_httpversion    =    "HTTP/1.0";            // default http request version
 110     var $_submit_method    =    "POST";                // default submit method
 111     var $_submit_type    =    "application/x-www-form-urlencoded";    // default submit type
 112     var $_mime_boundary    =   "";                    // MIME boundary for multipart/form-data submit type
 113     var $_redirectaddr    =    false;                // will be set if page fetched is a redirect
 114     var $_redirectdepth    =    0;                    // increments on an http redirect
 115     var $_frameurls        =     array();            // frame src urls
 116     var $_framedepth    =    0;                    // increments on frame depth
 117
 118     var $_isproxy        =    false;                // set if using a proxy server
 119     var $_fp_timeout    =    30;                    // timeout for socket connection
 120
 121 /*======================================================================*\
 122     Function:    fetch
 123     Purpose:    fetch the contents of a web page
 124                 (and possibly other protocols in the
 125                 future like ftp, nntp, gopher, etc.)
 126     Input:        $URI    the location of the page to fetch
 127     Output:        $this->results    the output text from the fetch
 128 \*======================================================================*/
 129
 130     function fetch($URI)
 131     {
 132
 133         //preg_match("|^([^:]+)://([^:/]+)(:[\d]+)*(.*)|",$URI,$URI_PARTS);
 134         $URI_PARTS = parse_url($URI);
 135         if (!empty($URI_PARTS["user"]))
 136             $this->user = $URI_PARTS["user"];
 137         if (!empty($URI_PARTS["pass"]))
 138             $this->pass = $URI_PARTS["pass"];
 139         if (empty($URI_PARTS["query"]))
 140             $URI_PARTS["query"] = '';
 141         if (empty($URI_PARTS["path"]))
 142             $URI_PARTS["path"] = '';
 143
 144         switch(strtolower($URI_PARTS["scheme"]))
 145         {
 146             case "http":
 147                 $this->host = $URI_PARTS["host"];
 148                 if(!empty($URI_PARTS["port"]))
 149                     $this->port = $URI_PARTS["port"];
 150                 if($this->_connect($fp))
 151                 {
 152                     if($this->_isproxy)
 153                     {
 154                         // using proxy, send entire URI
 155                         $this->_httprequest($URI,$fp,$URI,$this->_httpmethod);
 156                     }
 157                     else
 158                     {
 159                         $path = $URI_PARTS["path"].($URI_PARTS["query"] ? "?".$URI_PARTS["query"] : "");
 160                         // no proxy, send only the path
 161                         $this->_httprequest($path, $fp, $URI, $this->_httpmethod);
 162                     }
 163
 164                     $this->_disconnect($fp);
 165
 166                     if($this->_redirectaddr)
 167                     {
 168                         /* url was redirected, check if we've hit the max depth */
 169                         if($this->maxredirs > $this->_redirectdepth)
 170                         {
 171                             // only follow redirect if it's on this site, or offsiteok is true
 172                             if(preg_match("|^http://".preg_quote($this->host)."|i",$this->_redirectaddr) || $this->offsiteok)
 173                             {
 174                                 /* follow the redirect */
 175                                 $this->_redirectdepth++;
 176                                 $this->lastredirectaddr=$this->_redirectaddr;
 177                                 $this->fetch($this->_redirectaddr);
 178                             }
 179                         }
 180                     }
 181
 182                     if($this->_framedepth < $this->maxframes && count($this->_frameurls) > 0)
 183                     {
 184                         $frameurls = $this->_frameurls;
 185                         $this->_frameurls = array();
 186
 187                         while(list(,$frameurl) = each($frameurls))
 188                         {
 189                             if($this->_framedepth < $this->maxframes)
 190                             {
 191                                 $this->fetch($frameurl);
 192                                 $this->_framedepth++;
 193                             }
 194                             else
 195                                 break;
 196                         }
 197                     }
 198                 }
 199                 else
 200                 {
 201                     return false;
 202                 }
 203                 return true;
 204                 break;
 205             case "https":
 206                 if(!$this->curl_path)
 207                     return false;
 208                 if(function_exists("is_executable"))
 209                     if (!is_executable($this->curl_path))
 210                         return false;
 211                 $this->host = $URI_PARTS["host"];
 212                 if(!empty($URI_PARTS["port"]))
 213                     $this->port = $URI_PARTS["port"];
 214                 if($this->_isproxy)
 215                 {
 216                     // using proxy, send entire URI
 217                     $this->_httpsrequest($URI,$URI,$this->_httpmethod);
 218                 }
 219                 else
 220                 {
 221                     $path = $URI_PARTS["path"].($URI_PARTS["query"] ? "?".$URI_PARTS["query"] : "");
 222                     // no proxy, send only the path
 223                     $this->_httpsrequest($path, $URI, $this->_httpmethod);
 224                 }
 225
 226                 if($this->_redirectaddr)
 227                 {
 228                     /* url was redirected, check if we've hit the max depth */
 229                     if($this->maxredirs > $this->_redirectdepth)
 230                     {
 231                         // only follow redirect if it's on this site, or offsiteok is true
 232                         if(preg_match("|^http://".preg_quote($this->host)."|i",$this->_redirectaddr) || $this->offsiteok)
 233                         {
 234                             /* follow the redirect */
 235                             $this->_redirectdepth++;
 236                             $this->lastredirectaddr=$this->_redirectaddr;
 237                             $this->fetch($this->_redirectaddr);
 238                         }
 239                     }
 240                 }
 241
 242                 if($this->_framedepth < $this->maxframes && count($this->_frameurls) > 0)
 243                 {
 244                     $frameurls = $this->_frameurls;
 245                     $this->_frameurls = array();
 246
 247                     while(list(,$frameurl) = each($frameurls))
 248                     {
 249                         if($this->_framedepth < $this->maxframes)
 250                         {
 251                             $this->fetch($frameurl);
 252                             $this->_framedepth++;
 253                         }
 254                         else
 255                             break;
 256                     }
 257                 }
 258                 return true;
 259                 break;
 260             default:
 261                 // not a valid protocol
 262                 $this->error    =    'Invalid protocol "'.$URI_PARTS["scheme"]; // moodlefix
 263                 return false;
 264                 break;
 265         }
 266         return true;
 267     }
 268
 269 /*======================================================================*\
 270     Function:    submit
 271     Purpose:    submit an http form
 272     Input:        $URI    the location to post the data
 273                 $formvars    the formvars to use.
 274                     format: $formvars["var"] = "val";
 275                 $formfiles  an array of files to submit
 276                     format: $formfiles["var"] = "/dir/filename.ext";
 277     Output:        $this->results    the text output from the post
 278 \*======================================================================*/
 279
 280     function submit($URI, $formvars="", $formfiles="")
 281     {
 282         unset($postdata);
 283
 284         $postdata = $this->_prepare_post_body($formvars, $formfiles);
 285
 286         $URI_PARTS = parse_url($URI);
 287         if (!empty($URI_PARTS["user"]))
 288             $this->user = $URI_PARTS["user"];
 289         if (!empty($URI_PARTS["pass"]))
 290             $this->pass = $URI_PARTS["pass"];
 291         if (empty($URI_PARTS["query"]))
 292             $URI_PARTS["query"] = '';
 293         if (empty($URI_PARTS["path"]))
 294             $URI_PARTS["path"] = '';
 295
 296         switch(strtolower($URI_PARTS["scheme"]))
 297         {
 298             case "http":
 299                 $this->host = $URI_PARTS["host"];
 300                 if(!empty($URI_PARTS["port"]))
 301                     $this->port = $URI_PARTS["port"];
 302                 if($this->_connect($fp))
 303                 {
 304                     if($this->_isproxy)
 305                     {
 306                         // using proxy, send entire URI
 307                         $this->_httprequest($URI,$fp,$URI,$this->_submit_method,$this->_submit_type,$postdata);
 308                     }
 309                     else
 310                     {
 311                         $path = $URI_PARTS["path"].($URI_PARTS["query"] ? "?".$URI_PARTS["query"] : "");
 312                         // no proxy, send only the path
 313                         $this->_httprequest($path, $fp, $URI, $this->_submit_method, $this->_submit_type, $postdata);
 314                     }
 315
 316                     $this->_disconnect($fp);
 317
 318                     if($this->_redirectaddr)
 319                     {
 320                         /* url was redirected, check if we've hit the max depth */
 321                         if($this->maxredirs > $this->_redirectdepth)
 322                         {
 323                             if(!preg_match("|^".$URI_PARTS["scheme"]."://|", $this->_redirectaddr))
 324                                 $this->_redirectaddr = $this->_expandlinks($this->_redirectaddr,$URI_PARTS["scheme"]."://".$URI_PARTS["host"]);
 325
 326                             // only follow redirect if it's on this site, or offsiteok is true
 327                             if(preg_match("|^http://".preg_quote($this->host)."|i",$this->_redirectaddr) || $this->offsiteok)
 328                             {
 329                                 /* follow the redirect */
 330                                 $this->_redirectdepth++;
 331                                 $this->lastredirectaddr=$this->_redirectaddr;
 332                                 if( strpos( $this->_redirectaddr, "?" ) > 0 )
 333                                     $this->fetch($this->_redirectaddr); // the redirect has changed the request method from post to get
 334                                 else
 335                                     $this->submit($this->_redirectaddr,$formvars, $formfiles);
 336                             }
 337                         }
 338                     }
 339
 340                     if($this->_framedepth < $this->maxframes && count($this->_frameurls) > 0)
 341                     {
 342                         $frameurls = $this->_frameurls;
 343                         $this->_frameurls = array();
 344
 345                         while(list(,$frameurl) = each($frameurls))
 346                         {
 347                             if($this->_framedepth < $this->maxframes)
 348                             {
 349                                 $this->fetch($frameurl);
 350                                 $this->_framedepth++;
 351                             }
 352                             else
 353                                 break;
 354                         }
 355                     }
 356
 357                 }
 358                 else
 359                 {
 360                     return false;
 361                 }
 362                 return true;
 363                 break;
 364             case "https":
 365                 if(!$this->curl_path)
 366                     return false;
 367                 if(function_exists("is_executable"))
 368                     if (!is_executable($this->curl_path))
 369                         return false;
 370                 $this->host = $URI_PARTS["host"];
 371                 if(!empty($URI_PARTS["port"]))
 372                     $this->port = $URI_PARTS["port"];
 373                 if($this->_isproxy)
 374                 {
 375                     // using proxy, send entire URI
 376                     $this->_httpsrequest($URI, $URI, $this->_submit_method, $this->_submit_type, $postdata);
 377                 }
 378                 else
 379                 {
 380                     $path = $URI_PARTS["path"].($URI_PARTS["query"] ? "?".$URI_PARTS["query"] : "");
 381                     // no proxy, send only the path
 382                     $this->_httpsrequest($path, $URI, $this->_submit_method, $this->_submit_type, $postdata);
 383                 }
 384
 385                 if($this->_redirectaddr)
 386                 {
 387                     /* url was redirected, check if we've hit the max depth */
 388                     if($this->maxredirs > $this->_redirectdepth)
 389                     {
 390                         if(!preg_match("|^".$URI_PARTS["scheme"]."://|", $this->_redirectaddr))
 391                             $this->_redirectaddr = $this->_expandlinks($this->_redirectaddr,$URI_PARTS["scheme"]."://".$URI_PARTS["host"]);
 392
 393                         // only follow redirect if it's on this site, or offsiteok is true
 394                         if(preg_match("|^http://".preg_quote($this->host)."|i",$this->_redirectaddr) || $this->offsiteok)
 395                         {
 396                             /* follow the redirect */
 397                             $this->_redirectdepth++;
 398                             $this->lastredirectaddr=$this->_redirectaddr;
 399                             if( strpos( $this->_redirectaddr, "?" ) > 0 )
 400                                 $this->fetch($this->_redirectaddr); // the redirect has changed the request method from post to get
 401                             else
 402                                 $this->submit($this->_redirectaddr,$formvars, $formfiles);
 403                         }
 404                     }
 405                 }
 406
 407                 if($this->_framedepth < $this->maxframes && count($this->_frameurls) > 0)
 408                 {
 409                     $frameurls = $this->_frameurls;
 410                     $this->_frameurls = array();
 411
 412                     while(list(,$frameurl) = each($frameurls))
 413                     {
 414                         if($this->_framedepth < $this->maxframes)
 415                         {
 416                             $this->fetch($frameurl);
 417                             $this->_framedepth++;
 418                         }
 419                         else
 420                             break;
 421                     }
 422                 }
 423                 return true;
 424                 break;
 425
 426             default:
 427                 // not a valid protocol
 428                 $this->error    =    'Invalid protocol "'.$URI_PARTS["scheme"]; //moodlefix
 429                 return false;
 430                 break;
 431         }
 432         return true;
 433     }
 434
 435 /*======================================================================*\
 436     Function:    fetchlinks
 437     Purpose:    fetch the links from a web page
 438     Input:        $URI    where you are fetching from
 439     Output:        $this->results    an array of the URLs
 440 \*======================================================================*/
 441
 442     function fetchlinks($URI)
 443     {
 444         if ($this->fetch($URI))
 445         {
 446             if($this->lastredirectaddr)
 447                 $URI = $this->lastredirectaddr;
 448             if(is_array($this->results))
 449             {
 450                 for($x=0;$x<count($this->results);$x++)
 451                     $this->results[$x] = $this->_striplinks($this->results[$x]);
 452             }
 453             else
 454                 $this->results = $this->_striplinks($this->results);
 455
 456             if($this->expandlinks)
 457                 $this->results = $this->_expandlinks($this->results, $URI);
 458             return true;
 459         }
 460         else
 461             return false;
 462     }
 463
 464 /*======================================================================*\
 465     Function:    fetchform
 466     Purpose:    fetch the form elements from a web page
 467     Input:        $URI    where you are fetching from
 468     Output:        $this->results    the resulting html form
 469 \*======================================================================*/
 470
 471     function fetchform($URI)
 472     {
 473
 474         if ($this->fetch($URI))
 475         {
 476
 477             if(is_array($this->results))
 478             {
 479                 for($x=0;$x<count($this->results);$x++)
 480                     $this->results[$x] = $this->_stripform($this->results[$x]);
 481             }
 482             else
 483                 $this->results = $this->_stripform($this->results);
 484
 485             return true;
 486         }
 487         else
 488             return false;
 489     }
 490
 491
 492 /*======================================================================*\
 493     Function:    fetchtext
 494     Purpose:    fetch the text from a web page, stripping the links
 495     Input:        $URI    where you are fetching from
 496     Output:        $this->results    the text from the web page
 497 \*======================================================================*/
 498
 499     function fetchtext($URI)
 500     {
 501         if($this->fetch($URI))
 502         {
 503             if(is_array($this->results))
 504             {
 505                 for($x=0;$x<count($this->results);$x++)
 506                     $this->results[$x] = $this->_striptext($this->results[$x]);
 507             }
 508             else
 509                 $this->results = $this->_striptext($this->results);
 510             return true;
 511         }
 512         else
 513             return false;
 514     }
 515
 516 /*======================================================================*\
 517     Function:    submitlinks
 518     Purpose:    grab links from a form submission
 519     Input:        $URI    where you are submitting from
 520     Output:        $this->results    an array of the links from the post
 521 \*======================================================================*/
 522
 523     function submitlinks($URI, $formvars="", $formfiles="")
 524     {
 525         if($this->submit($URI,$formvars, $formfiles))
 526         {
 527             if($this->lastredirectaddr)
 528                 $URI = $this->lastredirectaddr;
 529             if(is_array($this->results))
 530             {
 531                 for($x=0;$x<count($this->results);$x++)
 532                 {
 533                     $this->results[$x] = $this->_striplinks($this->results[$x]);
 534                     if($this->expandlinks)
 535                         $this->results[$x] = $this->_expandlinks($this->results[$x],$URI);
 536                 }
 537             }
 538             else
 539             {
 540                 $this->results = $this->_striplinks($this->results);
 541                 if($this->expandlinks)
 542                     $this->results = $this->_expandlinks($this->results,$URI);
 543             }
 544             return true;
 545         }
 546         else
 547             return false;
 548     }
 549
 550 /*======================================================================*\
 551     Function:    submittext
 552     Purpose:    grab text from a form submission
 553     Input:        $URI    where you are submitting from
 554     Output:        $this->results    the text from the web page
 555 \*======================================================================*/
 556
 557     function submittext($URI, $formvars = "", $formfiles = "")
 558     {
 559         if($this->submit($URI,$formvars, $formfiles))
 560         {
 561             if($this->lastredirectaddr)
 562                 $URI = $this->lastredirectaddr;
 563             if(is_array($this->results))
 564             {
 565                 for($x=0;$x<count($this->results);$x++)
 566                 {
 567                     $this->results[$x] = $this->_striptext($this->results[$x]);
 568                     if($this->expandlinks)
 569                         $this->results[$x] = $this->_expandlinks($this->results[$x],$URI);
 570                 }
 571             }
 572             else
 573             {
 574                 $this->results = $this->_striptext($this->results);
 575                 if($this->expandlinks)
 576                     $this->results = $this->_expandlinks($this->results,$URI);
 577             }
 578             return true;
 579         }
 580         else
 581             return false;
 582     }
 583
 584
 585
 586 /*======================================================================*\
 587     Function:    set_submit_multipart
 588     Purpose:    Set the form submission content type to
 589                 multipart/form-data
 590 \*======================================================================*/
 591     function set_submit_multipart()
 592     {
 593         $this->_submit_type = "multipart/form-data";
 594     }
 595
 596
 597 /*======================================================================*\
 598     Function:    set_submit_normal
 599     Purpose:    Set the form submission content type to
 600                 application/x-www-form-urlencoded
 601 \*======================================================================*/
 602     function set_submit_normal()
 603     {
 604         $this->_submit_type = "application/x-www-form-urlencoded";
 605     }
 606
 607
 608
 609
 610 /*======================================================================*\
 611     Private functions
 612 \*======================================================================*/
 613
 614
 615 /*======================================================================*\
 616     Function:    _striplinks
 617     Purpose:    strip the hyperlinks from an html document
 618     Input:        $document    document to strip.
 619     Output:        $match        an array of the links
 620 \*======================================================================*/
 621
 622     function _striplinks($document)
 623     {
 624         preg_match_all("'<\s*a\s.*?href\s*=\s*            # find <a href=
 625                         ([\"\'])?                    # find single or double quote
 626                         (?(1) (.*?)\\1 | ([^\s\>]+))        # if quote found, match up to next matching
 627                                                     # quote, otherwise match up to next space
 628                         'isx",$document,$links);
 629
 630
 631         // catenate the non-empty matches from the conditional subpattern
 632
 633         while(list($key,$val) = each($links[2]))
 634         {
 635             if(!empty($val))
 636                 $match[] = $val;
 637         }
 638
 639         while(list($key,$val) = each($links[3]))
 640         {
 641             if(!empty($val))
 642                 $match[] = $val;
 643         }
 644
 645         // return the links
 646         return $match;
 647     }
 648
 649 /*======================================================================*\
 650     Function:    _stripform
 651     Purpose:    strip the form elements from an html document
 652     Input:        $document    document to strip.
 653     Output:        $match        an array of the links
 654 \*======================================================================*/
 655
 656     function _stripform($document)
 657     {
 658         preg_match_all("'<\/?(FORM|INPUT|SELECT|TEXTAREA|(OPTION))[^<>]*>(?(2)(.*(?=<\/?(option|select)[^<>]*>[\r\n]*)|(?=[\r\n]*))|(?=[\r\n]*))'Usi",$document,$elements);
 659
 660         // catenate the matches
 661         $match = implode("\r\n",$elements[0]);
 662
 663         // return the links
 664         return $match;
 665     }
 666
 667
 668
 669 /*======================================================================*\
 670     Function:    _striptext
 671     Purpose:    strip the text from an html document
 672     Input:        $document    document to strip.
 673     Output:        $text        the resulting text
 674 \*======================================================================*/
 675
 676     function _striptext($document)
 677     {
 678
 679         // I didn't use preg eval (//e) since that is only available in PHP 4.0.
 680         // so, list your entities one by one here. I included some of the
 681         // more common ones.
 682
 683         $search = array("'<script[^>]*?>.*?</script>'si",    // strip out javascript
 684                         "'<[\/\!]*?[^<>]*?>'si",            // strip out html tags
 685                         "'([\r\n])[\s]+'",                    // strip out white space
 686                         "'&(quot|#34|#034|#x22);'i",        // replace html entities
 687                         "'&(amp|#38|#038|#x26);'i",            // added hexadecimal values
 688                         "'&(lt|#60|#060|#x3c);'i",
 689                         "'&(gt|#62|#062|#x3e);'i",
 690                         "'&(nbsp|#160|#xa0);'i",
 691                         "'&(iexcl|#161);'i",
 692                         "'&(cent|#162);'i",
 693                         "'&(pound|#163);'i",
 694                         "'&(copy|#169);'i",
 695                         "'&(reg|#174);'i",
 696                         "'&(deg|#176);'i",
 697                         "'&(#39|#039|#x27);'",
 698                         "'&(euro|#8364);'i",                // europe
 699                         "'&a(uml|UML);'",                    // german
 700                         "'&o(uml|UML);'",
 701                         "'&u(uml|UML);'",
 702                         "'&A(uml|UML);'",
 703                         "'&O(uml|UML);'",
 704                         "'&U(uml|UML);'",
 705                         "'&szlig;'i",
 706                         );
 707         $replace = array(    "",
 708                             "",
 709                             "\\1",
 710                             "\"",
 711                             "&",
 712                             "<",
 713                             ">",
 714                             " ",
 715                             chr(161),
 716                             chr(162),
 717                             chr(163),
 718                             chr(169),
 719                             chr(174),
 720                             chr(176),
 721                             chr(39),
 722                             chr(128),
 723                             "�",
 724                             "�",
 725                             "�",
 726                             "�",
 727                             "�",
 728                             "�",
 729                             "�",
 730                         );
 731
 732         $text = preg_replace($search,$replace,$document);
 733
 734         return $text;
 735     }
 736
 737 /*======================================================================*\
 738     Function:    _expandlinks
 739     Purpose:    expand each link into a fully qualified URL
 740     Input:        $links            the links to qualify
 741                 $URI            the full URI to get the base from
 742     Output:        $expandedLinks    the expanded links
 743 \*======================================================================*/
 744
 745     function _expandlinks($links,$URI)
 746     {
 747
 748         preg_match("/^[^\?]+/",$URI,$match);
 749
 750         $match = preg_replace("|/[^\/\.]+\.[^\/\.]+$|","",$match[0]);
 751         $match = preg_replace("|/$|","",$match);
 752         $match_part = parse_url($match);
 753         $match_root =
 754         $match_part["scheme"]."://".$match_part["host"];
 755
 756         $search = array(     "|^http://".preg_quote($this->host)."|i",
 757                             "|^(\/)|i",
 758                             "|^(?!http://)(?!mailto:)|i",
 759                             "|/\./|",
 760                             "|/[^\/]+/\.\./|"
 761                         );
 762
 763         $replace = array(    "",
 764                             $match_root."/",
 765                             $match."/",
 766                             "/",
 767                             "/"
 768                         );
 769
 770         $expandedLinks = preg_replace($search,$replace,$links);
 771
 772         return $expandedLinks;
 773     }
 774
 775 /*======================================================================*\
 776     Function:    _httprequest
 777     Purpose:    go get the http data from the server
 778     Input:        $url        the url to fetch
 779                 $fp            the current open file pointer
 780                 $URI        the full URI
 781                 $body        body contents to send if any (POST)
 782     Output:
 783 \*======================================================================*/
 784
 785     function _httprequest($url,$fp,$URI,$http_method,$content_type="",$body="")
 786     {
 787         $cookie_headers = '';
 788         if($this->passcookies && $this->_redirectaddr)
 789             $this->setcookies();
 790
 791         $URI_PARTS = parse_url($URI);
 792         if(empty($url))
 793             $url = "/";
 794         $headers = $http_method." ".$url." ".$this->_httpversion."\r\n";
 795         if(!empty($this->agent))
 796             $headers .= "User-Agent: ".$this->agent."\r\n";
 797         if(!empty($this->host) && !isset($this->rawheaders['Host'])) {
 798             $headers .= "Host: ".$this->host;
 799             if(!empty($this->port))
 800                 $headers .= ":".$this->port;
 801             $headers .= "\r\n";
 802         }
 803         if(!empty($this->accept))
 804             $headers .= "Accept: ".$this->accept."\r\n";
 805         if(!empty($this->referer))
 806             $headers .= "Referer: ".$this->referer."\r\n";
 807         if(!empty($this->cookies))
 808         {
 809             if(!is_array($this->cookies))
 810                 $this->cookies = (array)$this->cookies;
 811
 812             reset($this->cookies);
 813             if ( count($this->cookies) > 0 ) {
 814                 $cookie_headers .= 'Cookie: ';
 815                 foreach ( $this->cookies as $cookieKey => $cookieVal ) {
 816                 $cookie_headers .= $cookieKey."=".urlencode($cookieVal)."; ";
 817                 }
 818                 $headers .= substr($cookie_headers,0,-2) . "\r\n";
 819             }
 820         }
 821         if(!empty($this->rawheaders))
 822         {
 823             if(!is_array($this->rawheaders))
 824                 $this->rawheaders = (array)$this->rawheaders;
 825             while(list($headerKey,$headerVal) = each($this->rawheaders))
 826                 $headers .= $headerKey.": ".$headerVal."\r\n";
 827         }
 828         if(!empty($content_type)) {
 829             $headers .= "Content-type: $content_type";
 830             if ($content_type == "multipart/form-data")
 831                 $headers .= "; boundary=".$this->_mime_boundary;
 832             $headers .= "\r\n";
 833         }
 834         if(!empty($body))
 835             $headers .= "Content-length: ".strlen($body)."\r\n";
 836         if(!empty($this->user) || !empty($this->pass))
 837             $headers .= "Authorization: Basic ".base64_encode($this->user.":".$this->pass)."\r\n";
 838
 839         //add proxy auth headers
 840         if(!empty($this->proxy_user))
 841             $headers .= 'Proxy-Authorization: ' . 'Basic ' . base64_encode($this->proxy_user . ':' . $this->proxy_pass)."\r\n";
 842
 843
 844         $headers .= "\r\n";
 845
 846         // set the read timeout if needed
 847         if ($this->read_timeout > 0)
 848             socket_set_timeout($fp, $this->read_timeout);
 849         $this->timed_out = false;
 850
 851         fwrite($fp,$headers.$body,strlen($headers.$body));
 852
 853         $this->_redirectaddr = false;
 854         unset($this->headers);
 855
 856         while($currentHeader = fgets($fp,$this->_maxlinelen))
 857         {
 858             if ($this->read_timeout > 0 && $this->_check_timeout($fp))
 859             {
 860                 $this->status=-100;
 861                 return false;
 862             }
 863
 864             if($currentHeader == "\r\n")
 865                 break;
 866
 867             // if a header begins with Location: or URI:, set the redirect
 868             if(preg_match("/^(Location:|URI:)/i",$currentHeader))
 869             {
 870                 // get URL portion of the redirect
 871                 preg_match("/^(Location:|URI:)[ ]+(.*)/i",chop($currentHeader),$matches);
 872                 // look for :// in the Location header to see if hostname is included
 873                 if(!preg_match("|\:\/\/|",$matches[2]))
 874                 {
 875                     // no host in the path, so prepend
 876                     $this->_redirectaddr = $URI_PARTS["scheme"]."://".$this->host.":".$this->port;
 877                     // eliminate double slash
 878                     if(!preg_match("|^/|",$matches[2]))
 879                             $this->_redirectaddr .= "/".$matches[2];
 880                     else
 881                             $this->_redirectaddr .= $matches[2];
 882                 }
 883                 else
 884                     $this->_redirectaddr = $matches[2];
 885             }
 886
 887             if(preg_match("|^HTTP/|",$currentHeader))
 888             {
 889                 if(preg_match("|^HTTP/[^\s]*\s(.*?)\s|",$currentHeader, $status))
 890                 {
 891                     $this->status= $status[1];
 892                 }
 893                 $this->response_code = $currentHeader;
 894             }
 895
 896             $this->headers[] = $currentHeader;
 897         }
 898
 899         $results = '';
 900         do {
 901             $_data = fread($fp, $this->maxlength);
 902             if (strlen($_data) == 0) {
 903                 break;
 904             }
 905             $results .= $_data;
 906         } while(true);
 907
 908         if ($this->read_timeout > 0 && $this->_check_timeout($fp))
 909         {
 910             $this->status=-100;
 911             return false;
 912         }
 913
 914         // check if there is a a redirect meta tag
 915
 916         if(preg_match("'<meta[\s]*http-equiv[^>]*?content[\s]*=[\s]*[\"\']?\d+;[\s]*URL[\s]*=[\s]*([^\"\']*?)[\"\']?>'i",$results,$match))
 917
 918         {
 919             $this->_redirectaddr = $this->_expandlinks($match[1],$URI);
 920         }
 921
 922         // have we hit our frame depth and is there frame src to fetch?
 923         if(($this->_framedepth < $this->maxframes) && preg_match_all("'<frame\s+.*src[\s]*=[\'\"]?([^\'\"\>]+)'i",$results,$match))
 924         {
 925             $this->results[] = $results;
 926             for($x=0; $x<count($match[1]); $x++)
 927                 $this->_frameurls[] = $this->_expandlinks($match[1][$x],$URI_PARTS["scheme"]."://".$this->host);
 928         }
 929         // have we already fetched framed content?
 930         elseif(is_array($this->results))
 931             $this->results[] = $results;
 932         // no framed content
 933         else
 934             $this->results = $results;
 935
 936         return true;
 937     }
 938
 939 /*======================================================================*\
 940     Function:    _httpsrequest
 941     Purpose:    go get the https data from the server using curl
 942     Input:        $url        the url to fetch
 943                 $URI        the full URI
 944                 $body        body contents to send if any (POST)
 945     Output:
 946 \*======================================================================*/
 947
 948     function _httpsrequest($url,$URI,$http_method,$content_type="",$body="")
 949     {
 950         if($this->passcookies && $this->_redirectaddr)
 951             $this->setcookies();
 952
 953         $headers = array();
 954
 955         $URI_PARTS = parse_url($URI);
 956         if(empty($url))
 957             $url = "/";
 958         // GET ... header not needed for curl
 959         //$headers[] = $http_method." ".$url." ".$this->_httpversion;
 960         if(!empty($this->agent))
 961             $headers[] = "User-Agent: ".$this->agent;
 962         if(!empty($this->host))
 963             if(!empty($this->port))
 964                 $headers[] = "Host: ".$this->host.":".$this->port;
 965             else
 966                 $headers[] = "Host: ".$this->host;
 967         if(!empty($this->accept))
 968             $headers[] = "Accept: ".$this->accept;
 969         if(!empty($this->referer))
 970             $headers[] = "Referer: ".$this->referer;
 971         if(!empty($this->cookies))
 972         {
 973             if(!is_array($this->cookies))
 974                 $this->cookies = (array)$this->cookies;
 975
 976             reset($this->cookies);
 977             if ( count($this->cookies) > 0 ) {
 978                 $cookie_str = 'Cookie: ';
 979                 foreach ( $this->cookies as $cookieKey => $cookieVal ) {
 980                 $cookie_str .= $cookieKey."=".urlencode($cookieVal)."; ";
 981                 }
 982                 $headers[] = substr($cookie_str,0,-2);
 983             }
 984         }
 985         if(!empty($this->rawheaders))
 986         {
 987             if(!is_array($this->rawheaders))
 988                 $this->rawheaders = (array)$this->rawheaders;
 989             while(list($headerKey,$headerVal) = each($this->rawheaders))
 990                 $headers[] = $headerKey.": ".$headerVal;
 991         }
 992         if(!empty($content_type)) {
 993             if ($content_type == "multipart/form-data")
 994                 $headers[] = "Content-type: $content_type; boundary=".$this->_mime_boundary;
 995             else
 996                 $headers[] = "Content-type: $content_type";
 997         }
 998         if(!empty($body))
 999             $headers[] = "Content-length: ".strlen($body);
1000         if(!empty($this->user) || !empty($this->pass))
1001             $headers[] = "Authorization: BASIC ".base64_encode($this->user.":".$this->pass);
1002
1003         for($curr_header = 0; $curr_header < count($headers); $curr_header++) {
1004             $safer_header = strtr( $headers[$curr_header], "\"", " " );
1005             $cmdline_params .= " -H \"".$safer_header."\"";
1006         }
1007
1008         if(!empty($body))
1009             $cmdline_params .= " -d \"$body\"";
1010
1011         if($this->read_timeout > 0)
1012             $cmdline_params .= " -m ".$this->read_timeout;
1013
1014         $headerfile = tempnam($temp_dir, "sno");
1015
1016         $safer_URI = strtr( $URI, "\"", " " ); // strip quotes from the URI to avoid shell access
1017         exec($this->curl_path." -D \"$headerfile\"".$cmdline_params." \"".$safer_URI."\"",$results,$return);
1018
1019         if($return)
1020         {
1021             $this->error = "Error: cURL could not retrieve the document, error $return.";
1022             return false;
1023         }
1024
1025
1026         $results = implode("\r\n",$results);
1027
1028         $result_headers = file("$headerfile");
1029
1030         $this->_redirectaddr = false;
1031         unset($this->headers);
1032
1033         for($currentHeader = 0; $currentHeader < count($result_headers); $currentHeader++)
1034         {
1035
1036             // if a header begins with Location: or URI:, set the redirect
1037             if(preg_match("/^(Location: |URI: )/i",$result_headers[$currentHeader]))
1038             {
1039                 // get URL portion of the redirect
1040                 preg_match("/^(Location: |URI:)\s+(.*)/",chop($result_headers[$currentHeader]),$matches);
1041                 // look for :// in the Location header to see if hostname is included
1042                 if(!preg_match("|\:\/\/|",$matches[2]))
1043                 {
1044                     // no host in the path, so prepend
1045                     $this->_redirectaddr = $URI_PARTS["scheme"]."://".$this->host.":".$this->port;
1046                     // eliminate double slash
1047                     if(!preg_match("|^/|",$matches[2]))
1048                             $this->_redirectaddr .= "/".$matches[2];
1049                     else
1050                             $this->_redirectaddr .= $matches[2];
1051                 }
1052                 else
1053                     $this->_redirectaddr = $matches[2];
1054             }
1055
1056             if(preg_match("|^HTTP/|",$result_headers[$currentHeader]))
1057                 $this->response_code = $result_headers[$currentHeader];
1058
1059             $this->headers[] = $result_headers[$currentHeader];
1060         }
1061
1062         // check if there is a a redirect meta tag
1063
1064         if(preg_match("'<meta[\s]*http-equiv[^>]*?content[\s]*=[\s]*[\"\']?\d+;[\s]*URL[\s]*=[\s]*([^\"\']*?)[\"\']?>'i",$results,$match))
1065         {
1066             $this->_redirectaddr = $this->_expandlinks($match[1],$URI);
1067         }
1068
1069         // have we hit our frame depth and is there frame src to fetch?
1070         if(($this->_framedepth < $this->maxframes) && preg_match_all("'<frame\s+.*src[\s]*=[\'\"]?([^\'\"\>]+)'i",$results,$match))
1071         {
1072             $this->results[] = $results;
1073             for($x=0; $x<count($match[1]); $x++)
1074                 $this->_frameurls[] = $this->_expandlinks($match[1][$x],$URI_PARTS["scheme"]."://".$this->host);
1075         }
1076         // have we already fetched framed content?
1077         elseif(is_array($this->results))
1078             $this->results[] = $results;
1079         // no framed content
1080         else
1081             $this->results = $results;
1082
1083         unlink("$headerfile");
1084
1085         return true;
1086     }
1087
1088 /*======================================================================*\
1089     Function:    setcookies()
1090     Purpose:    set cookies for a redirection
1091 \*======================================================================*/
1092
1093     function setcookies()
1094     {
1095         for($x=0; $x<count($this->headers); $x++)
1096         {
1097         if(preg_match('/^set-cookie:[\s]+([^=]+)=([^;]+)/i', $this->headers[$x],$match))
1098             $this->cookies[$match[1]] = urldecode($match[2]);
1099         }
1100     }
1101
1102
1103 /*======================================================================*\
1104     Function:    _check_timeout
1105     Purpose:    checks whether timeout has occurred
1106     Input:        $fp    file pointer
1107 \*======================================================================*/
1108
1109     function _check_timeout($fp)
1110     {
1111         if ($this->read_timeout > 0) {
1112             $fp_status = socket_get_status($fp);
1113             if ($fp_status["timed_out"]) {
1114                 $this->timed_out = true;
1115                 return true;
1116             }
1117         }
1118         return false;
1119     }
1120
1121 /*======================================================================*\
1122     Function:    _connect
1123     Purpose:    make a socket connection
1124     Input:        $fp    file pointer
1125 \*======================================================================*/
1126
1127     function _connect(&$fp)
1128     {
1129         if(!empty($this->proxy_host) && !empty($this->proxy_port))
1130             {
1131                 $this->_isproxy = true;
1132
1133                 $host = $this->proxy_host;
1134                 $port = $this->proxy_port;
1135             }
1136         else
1137         {
1138             $host = $this->host;
1139             $port = $this->port;
1140         }
1141
1142         $this->status = 0;
1143
1144         if($fp = fsockopen(
1145                     $host,
1146                     $port,
1147                     $errno,
1148                     $errstr,
1149                     $this->_fp_timeout
1150                     ))
1151         {
1152             // socket connection succeeded
1153
1154             return true;
1155         }
1156         else
1157         {
1158             // socket connection failed
1159             $this->status = $errno;
1160             switch($errno)
1161             {
1162                 case -3:
1163                     $this->error="socket creation failed (-3)";
1164                 case -4:
1165                     $this->error="dns lookup failure (-4)";
1166                 case -5:
1167                     $this->error="connection refused or timed out (-5)";
1168                 default:
1169                     $this->error="connection failed (".$errno.")";
1170             }
1171             return false;
1172         }
1173     }
1174 /*======================================================================*\
1175     Function:    _disconnect
1176     Purpose:    disconnect a socket connection
1177     Input:        $fp    file pointer
1178 \*======================================================================*/
1179
1180     function _disconnect($fp)
1181     {
1182         return(fclose($fp));
1183     }
1184
1185
1186 /*======================================================================*\
1187     Function:    _prepare_post_body
1188     Purpose:    Prepare post body according to encoding type
1189     Input:        $formvars  - form variables
1190                 $formfiles - form upload files
1191     Output:        post body
1192 \*======================================================================*/
1193
1194     function _prepare_post_body($formvars, $formfiles)
1195     {
1196         settype($formvars, "array");
1197         settype($formfiles, "array");
1198         $postdata = '';
1199
1200         if (count($formvars) == 0 && count($formfiles) == 0)
1201             return;
1202
1203         switch ($this->_submit_type) {
1204             case "application/x-www-form-urlencoded":
1205                 reset($formvars);
1206                 while(list($key,$val) = each($formvars)) {
1207                     if (is_array($val) || is_object($val)) {
1208                         while (list($cur_key, $cur_val) = each($val)) {
1209                             $postdata .= urlencode($key)."[]=".urlencode($cur_val)."&";
1210                         }
1211                     } else
1212                         $postdata .= urlencode($key)."=".urlencode($val)."&";
1213                 }
1214                 break;
1215
1216             case "multipart/form-data":
1217                 $this->_mime_boundary = "Snoopy".md5(uniqid(microtime()));
1218
1219                 reset($formvars);
1220                 while(list($key,$val) = each($formvars)) {
1221                     if (is_array($val) || is_object($val)) {
1222                         while (list($cur_key, $cur_val) = each($val)) {
1223                             $postdata .= "--".$this->_mime_boundary."\r\n";
1224                             $postdata .= "Content-Disposition: form-data; name=\"$key\[\]\"\r\n\r\n";
1225                             $postdata .= "$cur_val\r\n";
1226                         }
1227                     } else {
1228                         $postdata .= "--".$this->_mime_boundary."\r\n";
1229                         $postdata .= "Content-Disposition: form-data; name=\"$key\"\r\n\r\n";
1230                         $postdata .= "$val\r\n";
1231                     }
1232                 }
1233
1234                 reset($formfiles);
1235                 while (list($field_name, $file_names) = each($formfiles)) {
1236                     settype($file_names, "array");
1237                     while (list(, $file_name) = each($file_names)) {
1238                         if (!is_readable($file_name)) continue;
1239
1240                         $fp = fopen($file_name, "r");
1241                         $file_content = fread($fp, filesize($file_name));
1242                         fclose($fp);
1243                         $base_name = basename($file_name);
1244
1245                         $postdata .= "--".$this->_mime_boundary."\r\n";
1246                         $postdata .= "Content-Disposition: form-data; name=\"$field_name\"; filename=\"$base_name\"\r\n\r\n";
1247                         $postdata .= "$file_content\r\n";
1248                     }
1249                 }
1250                 $postdata .= "--".$this->_mime_boundary."--\r\n";
1251                 break;
1252         }
1253
1254         return $postdata;
1255     }
1256 }
1257
1258 ?>