Save all modification
[mozilla-1.9/m8.git] / tools / page-loader / loader.pl
blob736c1881641af6d582543b4c39694dc01a2a58df
1 #!/usr/bin/perl
3 # ***** BEGIN LICENSE BLOCK *****
4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
6 # The contents of this file are subject to the Mozilla Public License Version
7 # 1.1 (the "License"); you may not use this file except in compliance with
8 # the License. You may obtain a copy of the License at
9 # http://www.mozilla.org/MPL/
11 # Software distributed under the License is distributed on an "AS IS" basis,
12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
13 # for the specific language governing rights and limitations under the
14 # License.
16 # The Original Code is Mozilla page-loader test, released Aug 5, 2001.
18 # The Initial Developer of the Original Code is
19 # Netscape Communications Corporation.
20 # Portions created by the Initial Developer are Copyright (C) 2001
21 # the Initial Developer. All Rights Reserved.
23 # Contributor(s):
24 # John Morrison <jrgm@netscape.com>, original author
25 # Heikki Toivonen <heikki@netscape.com>
27 # Alternatively, the contents of this file may be used under the terms of
28 # either the GNU General Public License Version 2 or later (the "GPL"), or
29 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
30 # in which case the provisions of the GPL or the LGPL are applicable instead
31 # of those above. If you wish to allow use of your version of this file only
32 # under the terms of either the GPL or the LGPL, and not to allow others to
33 # use your version of this file under the terms of the MPL, indicate your
34 # decision by deleting the provisions above and replace them with the notice
35 # and other provisions required by the GPL or the LGPL. If you do not delete
36 # the provisions above, a recipient may use your version of this file under
37 # the terms of any one of the MPL, the GPL or the LGPL.
39 # ***** END LICENSE BLOCK *****
41 use strict;
42 use CGI::Request;
43 use CGI::Carp qw(fatalsToBrowser);
44 use Time::HiRes qw(gettimeofday tv_interval);
45 use POSIX qw(strftime);
46 use DBI;
48 # list of test pages, JS to insert, httpbase, filebase, etc.
49 use PageData;
51 use vars qw(%params $req $cgi $dbh $pagedata
52 $gStartNow $gStartNowStr
53 $gResponseNow $gLogging);
55 $gStartNow = [gettimeofday]; # checkpoint the time
56 $gStartNowStr = strftime "%Y%m%d%H%M%S", localtime;
57 $gLogging = 1;
59 $req = new CGI::Request; # get the HTTP/CGI request
60 $cgi = $req->cgi;
62 $pagedata = PageData->new;
64 setDefaultParams();
66 #XXXdebugcrap
67 #warn $params{index}, " ", $params{maxidx};
69 if (!defined($req->param('delay'))) {
70 # give the user a form to pick options (but note that going
71 # to "loader.pl?delay=1000" immediately starts the test run
72 outputForm();
74 elsif (!$req->param('id')) {
75 initialize(); # do redirect to start the cycle
77 elsif ($params{index} > $params{maxidx}) {
78 redirectToReport(); # the test is over; spit out a summary
79 markTestAsComplete(); # close the meta table entry
81 elsif (!isRequestStale()) {
82 outputPage(); # otherwise, keep dishing out pages
83 updateDataBase(); # client has the response; now write out stats to db
86 # cleanup
87 $req = undef;
88 $dbh->disconnect() if $dbh; # not strictly required (ignored in some cases anyways)
90 #logMessage(sprintf("Page load server responded in %3d msec, total time %3d msec, pid: %d",
91 # 1000*tv_interval($gStartNow, $gResponseNow), 1000*tv_interval($gStartNow), $$))
92 # if $gResponseNow; # log only when a test page has been dished out
94 exit 0;
96 #######################################################################
98 sub logMessage {
99 print STDERR strftime("[%a %b %d %H:%M:%S %Y] ", localtime), @_, "\n"
100 if $gLogging;
104 sub isRequestStale {
105 my $limit = 30*60; # 30 minutes, although if we never stalled on mac I'd make it 3 minutes
106 my $ts = decodeHiResTime($params{s_ts});
107 my $delta = tv_interval($ts, $gStartNow);
108 return undef if $delta < $limit;
109 # otherwise, punt this request
110 print "Content-type: text/html\n\n";
111 print <<"ENDOFHTML";
112 <html><head><title>Page Loading Times Test</title></head><body>
113 <p><b>The timestamp on the request is too old to continue:<br>
114 s_ts=$params{s_ts} was $delta seconds ago. Limit is $limit seconds.</b></p>
115 </body></html>
116 ENDOFHTML
117 return 1; # it's stale
121 sub initialize {
122 updateMetaTable();
123 createDataSetTable();
125 # start the test by bouncing off of an echo page
126 my $script = $cgi->var("SCRIPT_NAME");
127 my $server = $cgi->var("SERVER_NAME");
128 my $proto = $ENV{SERVER_PORT} == 443 ? 'https://' : 'http://';
129 my $me = $proto . $server . $script;
130 $script =~ /^(.*\/).*$/;
131 my $loc = "Location: ". $proto . $server . $1 . "echo.pl?";
132 for (qw(id index maxcyc delay replace nocache timeout)) {
133 $loc .= "$_=$params{$_}\&";
135 $loc .= "url=" . $me;
136 print $loc, "\n\n";
140 sub redirectToReport {
141 # n.b., can also add '&sort=1' to get a time sorted list
142 my $proto = $ENV{SERVER_PORT} == 443 ? 'https://' : 'http://';
143 my $loc = "Location: " . $proto . $cgi->var("SERVER_NAME");
144 $cgi->var("SCRIPT_NAME") =~ /^(.*\/).*$/;
145 $loc .= $1 . "report.pl?id=" . $params{id};
146 # To use for a tinderbox, comment out the line above and uncomment this:
147 # $loc .= $1 . "dump.pl?id=" . $params{id} . "&purge=1";
148 print $loc, "\n\n";
152 sub generateTestId {
153 # use the epoch time, in hex, plus a two-character random.
154 return sprintf "%8X%02X", time(), int(256*rand());
158 sub setDefaultParams {
159 $params{id} = $req->param('id') || generateTestId(); # "unique" id for this run
160 $params{index} = $req->param('index') || 0; # request index for the test
161 $params{maxcyc} = defined($req->param('maxcyc')) ?
162 $req->param('maxcyc') : 3; # max visits (zero-based count)
163 $params{delay} = $req->param('delay') || 1000; # setTimeout on the next request (msec)
164 $params{replace} = $req->param('replace') || 0; # use Location.replace (1) or Location.href (0)
165 $params{nocache} = $req->param('nocache') || 0; # serve content via uncacheable path
166 $params{c_part} = $req->param('c_part') || 0; # client time elapsed; page head to onload (msec)
167 $params{c_intvl} = $req->param('c_intvl') || 0; # client time elapsed; onload to onload event (msec)
168 $params{c_ts} = $req->param('c_ts') || 0; # client timestamp (.getTime()) (msec)
169 $params{content} = $req->param('content') || "UNKNOWN"; # name of content page for this data
170 $params{s_ts} = $req->param('s_ts') || undef; # server timestamp; no default
171 $params{timeout} = $req->param('timeout') || 30000; # msec; timer will cancel stalled page loading
172 $params{maxidx} = ($params{maxcyc}+1) * $pagedata->length; # total pages loads to be done
173 $params{curidx} = $params{index} % $pagedata->length; # current request index into page list
174 $params{curcyc} = int(($params{index}-1) / $pagedata->length); # current "cycle" (visit)
178 sub outputPage {
179 my $relpath = $pagedata->url($params{curidx});
180 my $file = $pagedata->filebase . $relpath;
181 open (HTML, "<$file") ||
182 die "Can't open file: $file, $!";
184 my $hook = "<script xmlns='http://www.w3.org/1999/xhtml'>\n";
185 $hook .= "var g_moztest_Start = (new Date()).getTime();\n";
186 $hook .= "var g_moztest_ServerTime='" . encodeHiResTime($gStartNow) . "';\n";
187 $hook .= "var g_moztest_Content='" . $pagedata->name($params{curidx}) . "';\n";
188 $hook .= $pagedata->clientJS; # ... and the main body
189 $hook .= "var g_moztest_safetyTimer = ";
190 $hook .= "window.setTimeout(moztest_safetyValve, " . $params{timeout} . ");";
191 $hook .= "</script>\n";
193 my $basepath = $pagedata->httpbase;
194 $basepath =~ s/^http:/https:/i
195 if $ENV{SERVER_PORT} == 443;
196 #warn "basepath: $basepath";
197 $basepath =~ s#^(.*?)(/base/)$#$1/nocache$2# if ($params{nocache});
198 $hook .= "<base href='". $basepath . $relpath .
199 "' xmlns='http://www.w3.org/1999/xhtml' />";
201 my $magic = $pagedata->magicString;
202 my $content = "";
203 while (<HTML>) {
204 s/$magic/$hook/;
205 $content .= $_;
208 my $contentTypeHeader;
209 my $mimetype = $pagedata->mimetype($params{curidx});
210 my $charset = $pagedata->charset($params{curidx});
211 if ($charset) {
212 $contentTypeHeader = qq{Content-type: $mimetype; charset="$charset"\n\n};
213 } else {
214 $contentTypeHeader = qq{Content-type: $mimetype\n\n};
216 #warn $contentTypeHeader; #XXXjrgm testing...
218 # N.B., these two cookie headers are obsolete, since I pass server info in
219 # JS now, to work around a bug in winEmbed with document.cookie. But
220 # since I _was_ sending two cookies as part of the test, I have to keep
221 # sending two cookies (at least for now, and it's not a bad thing to test)
222 #XXX other headers to test/use?
224 $gResponseNow = [gettimeofday]; # for logging
225 { # turn on output autoflush, locally in this block
226 print "Set-Cookie: moztest_SomeRandomCookie1=somerandomstring\n";
227 print "Set-Cookie: moztest_SomeRandomCookie2=somerandomstring\n";
228 print $contentTypeHeader;
229 local $| = 1;
230 print $content;
233 return;
237 sub encodeHiResTime {
238 my $timeref = shift;
239 return unless ref($timeref);
240 return $$timeref[0] . "-" . $$timeref[1];
244 sub decodeHiResTime {
245 my $timestr = shift;
246 return [ split('-', $timestr) ];
250 sub elapsedMilliSeconds {
251 my ($r_time, $timestr) = @_;
252 return "NaN" unless $timestr;
253 my $delta = tv_interval( [ split('-', $timestr) ], $r_time );
254 my $delta = int(($delta*1000) - $params{delay}); # adjust for delay (in msec)
255 return $delta;
259 sub updateDataBase {
260 connectToDataBase(); # (may already be cached)
261 updateMetaTable();
262 updateDataSetTable() unless $params{c_part} == -1; # the initial request
266 sub connectToDataBase {
267 # don't reconnect if already connected. (Other drivers provide this
268 # for free I think, but not this one).
269 if (!ref($dbh)) {
270 $dbh = DBI->connect("DBI:CSV:f_dir=./db", {RaiseError => 1, AutoCommit => 1})
271 || die "Cannot connect: " . $DBI::errstr;
277 # Holds the individual page load data for this id.
279 # (Of course, this should really be a single table for all datasets, but
280 # that was becoming punitively slow with DBD::CSV. I could have moved to
281 # a "real" database, but I didn't want to make that a requirement for
282 # installing this on another server and using this test (e.g., install a
283 # few modules and you can run this; no sql installation/maintenance required).
284 # At some point though, I may switch to some sql db, but hopefully still allow
285 # this to be used with a simple flat file db. (Hmm, maybe I should try a *dbm
286 # as a compromise (disk based but indexed)).
288 sub createDataSetTable {
289 my $table = "t" . $params{id};
290 return if -f "db/$table"; # don't create it if it exists
291 logMessage("createDataSetTable:\tdb/$table");
292 connectToDataBase(); # cached
294 my ($sth, $sql);
295 $sql = qq{
296 CREATE TABLE $table
297 (DATETIME CHAR(14),
298 ID CHAR(10),
299 INDEX INTEGER,
300 CUR_IDX INTEGER,
301 CUR_CYC INTEGER,
302 C_PART INTEGER,
303 S_INTVL INTEGER,
304 C_INTVL INTEGER,
305 CONTENT CHAR(128)
308 $sth = $dbh->prepare($sql);
309 $sth->execute();
310 $sth->finish();
311 return 1;
316 # holds the information about all test runs
318 sub createMetaTable {
319 my $table = shift;
320 return if -f "db/$table"; # don't create it if it exists
321 logMessage("createMetaTable:\tdb/$table");
323 my ($sth, $sql);
325 $sql = qq{
326 CREATE TABLE $table
327 (DATETIME CHAR(14),
328 LASTPING CHAR(14),
329 ID CHAR(8),
330 INDEX INTEGER,
331 CUR_IDX INTEGER,
332 CUR_CYC INTEGER,
333 CUR_CONTENT CHAR(128),
334 STATE INTEGER,
335 BLESSED INTEGER,
336 MAXCYC INTEGER,
337 MAXIDX INTEGER,
338 REPLACE INTEGER,
339 NOCACHE INTEGER,
340 DELAY INTEGER,
341 REMOTE_USER CHAR(16),
342 HTTP_USER_AGENT CHAR(128),
343 REMOTE_ADDR CHAR(15),
344 USER_EMAIL CHAR(32),
345 USER_COMMENT CHAR(256)
348 $sth = $dbh->prepare($sql);
349 $sth->execute();
350 $sth->finish();
351 warn 'created meta table';
352 return 1;
356 sub updateMetaTable {
358 connectToDataBase(); # if not already connected
360 my $table = "tMetaTable";
361 createMetaTable($table); # just returns if already created
363 my ($sth, $sql);
365 $sql = qq{
366 SELECT INDEX, MAXCYC, MAXIDX, REPLACE, NOCACHE,
367 DELAY, REMOTE_USER, HTTP_USER_AGENT, REMOTE_ADDR
368 FROM $table
369 WHERE ID = '$params{id}'
371 $sth = $dbh->prepare($sql);
372 $sth->execute();
374 my @dataset = ();
375 while (my @data = $sth->fetchrow_array()) {
376 push @dataset, {index => shift @data,
377 maxcyc => shift @data,
378 maxidx => shift @data,
379 replace => shift @data,
380 nocache => shift @data,
381 delay => shift @data,
382 remote_user => shift @data,
383 http_user_agent => shift @data,
384 remote_addr => shift @data
387 $sth->finish();
388 warn "More than one ID: $params{id} ??" if scalar(@dataset) > 1;
390 if (scalar(@dataset) == 0) {
391 # this is a new dataset and id
392 initMetaTableRecord($table);
393 return;
396 #XXX need to check that values are sane, and not update if they don't
397 # match certain params. This should not happen in a normal test run.
398 # However, if a test url was bookmarked or in history, I might get bogus
399 # data collected after the fact. But I have a stale date set on the URL,
400 # so that is good enough for now.
401 # my $ref = shift @dataset; # check some $ref->{foo}
403 $sql = qq{
404 UPDATE $table
405 SET LASTPING = ?,
406 INDEX = ?,
407 CUR_IDX = ?,
408 CUR_CYC = ?,
409 CUR_CONTENT = ?,
410 STATE = ?
411 WHERE ID = '$params{id}'
413 $sth = $dbh->prepare($sql);
414 $sth->execute($gStartNowStr,
415 $params{index}-1, # (index-1) is complete; (index) in progress
416 ($params{curidx}-1) % $pagedata->length,
417 $params{curcyc},
418 $params{content},
419 'OPEN'
421 $sth->finish();
426 sub markTestAsComplete {
427 connectToDataBase(); # if not already connected
428 my $table = "tMetaTable";
429 createMetaTable($table); # just returns if already created
430 my ($sth, $sql);
431 #XXX should probably check if this ID exists first
432 $sql = qq{
433 UPDATE $table
434 SET STATE = "COMPLETE"
435 WHERE ID = '$params{id}'
437 $sth = $dbh->prepare($sql);
438 $sth->execute();
439 $sth->finish();
443 sub initMetaTableRecord {
444 # we know this record doesn't exist, so put in the initial values
445 my $table = shift;
446 my ($sth, $sql);
447 $sql = qq{
448 INSERT INTO $table
449 (DATETIME,
450 LASTPING,
452 INDEX,
453 CUR_IDX,
454 CUR_CYC,
455 CUR_CONTENT,
456 STATE,
457 BLESSED,
458 MAXCYC,
459 MAXIDX,
460 REPLACE,
461 NOCACHE,
462 DELAY,
463 REMOTE_USER,
464 HTTP_USER_AGENT,
465 REMOTE_ADDR,
466 USER_EMAIL,
467 USER_COMMENT
469 VALUES (?,?,?,?,
470 ?,?,?,?,
471 ?,?,?,?,
472 ?,?,?,?,
473 ?,?,?)
475 $sth = $dbh->prepare($sql);
476 $sth->execute($gStartNowStr,
477 $gStartNowStr,
478 $params{id},
479 $params{index}-1,
480 ($params{curidx}-1) % $pagedata->length,
481 $params{curcyc},
482 $params{content},
483 "INIT",
485 $params{maxcyc},
486 $params{maxidx},
487 $params{replace},
488 $params{nocache},
489 $params{delay},
490 $cgi->var("REMOTE_USER"),
491 $cgi->var("HTTP_USER_AGENT"),
492 $cgi->var("REMOTE_ADDR"),
496 $sth->finish();
500 sub updateDataSetTable {
501 my $table = shift;
502 my $table = "t" . $params{id};
504 my ($sth, $sql);
505 $sql = qq{
506 INSERT INTO $table
507 (DATETIME,
509 INDEX,
510 CUR_IDX,
511 CUR_CYC,
512 C_PART,
513 S_INTVL,
514 C_INTVL,
515 CONTENT
517 VALUES (?,?,?,?,
518 ?,?,?,?,?)
521 my $s_intvl = elapsedMilliSeconds( $gStartNow, $params{s_ts} );
523 $sth = $dbh->prepare($sql);
524 $sth->execute($gStartNowStr,
525 $params{id},
526 $params{index}-1,
527 ($params{curidx}-1) % $pagedata->length,
528 $params{curcyc},
529 $params{c_part},
530 $s_intvl,
531 $params{c_intvl},
532 $req->param('content'),
534 $sth->finish();
539 sub outputForm {
540 my @prog = split('/', $0); my $prog = $prog[$#prog];
541 print "Content-type: text/html\n\n";
542 my $bgcolor = $ENV{SERVER_PORT} == 443 ? '#eebb66' : '#ffffff';
543 print <<"ENDOFHTML";
544 <html>
545 <head>
546 <title>Page Loading Times Test</title>
547 </head>
548 <body bgcolor="$bgcolor">
549 <h3>Page Loading Times Test</h3>
551 <p>Questions: <a href="mailto:jrgm\@netscape.com">John Morrison</a>
553 ENDOFHTML
554 print "&nbsp;&nbsp;-&nbsp;&nbsp;";
555 my $script = $cgi->var("SCRIPT_NAME");
556 my $server = $cgi->var("SERVER_NAME");
557 # pick the "other" protocol (i.e., test is inverted)
558 my $proto = $ENV{SERVER_PORT} == 443 ? 'http://' : 'https://';
559 my $other = $proto . $server . $script;
560 if ($ENV{SERVER_PORT} == 443) {
561 print "[&nbsp;<a href='$other'>With no SSL</a>&nbsp;|&nbsp;<b>With SSL</b>&nbsp;]<br>";
562 } else {
563 print "[&nbsp;<b>With no SSL</b>&nbsp;|&nbsp;<a href='$other'>With SSL</a>&nbsp;]<br>";
565 print <<"ENDOFHTML";
567 <form method="get" action="$prog" >
568 <table border="1" cellpadding="5" cellspacing="2">
569 <tr>
570 <td valign="top">
571 Page-load to Page-load Delay (msec):<br>
572 (Use 1000. Be nice.)
573 </td>
574 <td valign="top">
575 <select name="delay">
576 <option value="0">0
577 <option value="500">500
578 <option selected value="1000">1000
579 <option value="2000">2000
580 <option value="3000">3000
581 <option value="4000">4000
582 <option value="5000">5000
583 </select>
584 </td>
585 </tr>
586 <tr>
587 <td valign="top">
588 Number of test cycles to run:<br>
589 <br>
590 </td>
591 <td valign="top">
592 <select name="maxcyc">
593 <option value="0">1
594 <option value="1">2
595 <option value="2">3
596 <option value="3">4
597 <option value="4" selected>5
598 <option value="5">6
599 <option value="6">7
600 </select>
601 </td>
602 </tr>
603 <tr>
604 <td valign="top">
605 How long to wait before cancelling (msec):<br>
606 (Don't change this unless on a very slow link, or very slow machine.)
607 </td>
608 <td valign="top">
609 <select name="timeout">
610 <option value="15000">15000
611 <option selected value="30000">30000
612 <option value="45000">45000
613 <option value="60000">60000
614 <option value="90000">90000
615 </select>
616 </td>
617 </tr>
618 <tr>
619 <td valign="top">
620 <input type="reset" value="reset">
621 </td>
622 <td valign="top">
623 <input type="submit" value="submit">
624 </td>
625 </tr>
626 </table>
628 <hr>
630 You can visit the content that will be loaded, minus the embedded
631 javascript, by clicking on any of the links below.
632 </p>
634 <table border="1" cellpadding="5" cellspacing="2">
635 ENDOFHTML
637 my $i;
638 print "<tr>\n";
639 my $base = $pagedata->httpbase;
640 $base =~ s/^http:/https:/i
641 if $ENV{SERVER_PORT} == 443;
642 for ($i=0; $i<$pagedata->length; $i++) {
643 print "<td nowrap><a href='", $base, $pagedata->url($i), "'>";
644 print $pagedata->name($i);
645 print "</a>\n";
646 print "</tr><tr>\n" if (($i+1)%4 == 0);
648 print "</tr>" if (($i+1)%4 != 0);
649 print "</table></form></body></html>\n";
650 return;