testing/tests/memtest.py

   1 #!/usr/bin/python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is standalone Firefox memory test.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # Mozilla Corporation.
  20 # Portions created by the Initial Developer are Copyright (C) 2006
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #   Graydon Hoare <graydon@mozilla.com> (original author)
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40
  41 import subprocess
  42 import tempfile
  43 import os
  44 import re
  45 import getpass
  46 import datetime
  47 import logging
  48 import time
  49 import sys
  50 import signal
  51 import shutil
  52 import glob
  53
  54 def sighandler(signal, frame):
  55         print "signal %d, throwing" % signal
  56         raise Exception
  57
  58 signal.signal(signal.SIGHUP, sighandler)
  59
  60
  61 ########################################################################
  62
  63 def start_xvnc(disp, tmpdir):
  64     xvnc_stdout = open(os.path.join(tmpdir, "xvnc.stdout"), mode="w")
  65     xvnc_stderr = open(os.path.join(tmpdir, "xvnc.stderr"), mode="w")
  66     xvnc_proc = subprocess.Popen (["Xvnc",
  67                    "-fp", "/usr/share/X11/fonts/misc",
  68                                    "-localhost",
  69                                    "-SecurityTypes", "None",
  70                    "-IdleTimeout", "604800",
  71                                    "-depth",         "24",
  72                                    "-ac",
  73                                    (":%d" % disp)],
  74                                   cwd=tmpdir,
  75                                   shell=False,
  76                                   stdout=xvnc_stdout,
  77                                   stderr=xvnc_stderr)
  78     time.sleep(3)
  79     assert xvnc_proc.poll() == None
  80     logging.info("started Xvnc server (pid %d) on display :%d"
  81                  % (xvnc_proc.pid, disp))
  82     return xvnc_proc
  83
  84 ########################################################################
  85
  86 def large_variant_filename(variant):
  87     return variant + "-large.swf"
  88
  89 def large_variant_html_filename(variant):
  90     return variant + "-large.html"
  91
  92 def small_variant_filename(variant):
  93     return variant + ".swf"
  94
  95 def start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir):
  96     rec_stdout = open(os.path.join(tmpdir, "vnc2swf.stdout"), mode="w")
  97     rec_stderr = open(os.path.join(tmpdir, "vnc2swf.stderr"), mode="w")
  98     rec_proc = subprocess.Popen ([os.path.join(vnc2swfpath, "vnc2swf.py"),
  99                   "-n",
 100                   "-o", large_variant_filename(variant),
 101                   "-C", "800x600+0+100",
 102                   "-r", "2",
 103                   "-t", "video",
 104                   "localhost:%d" % disp],
 105                                   cwd=tmpdir,
 106                                   shell=False,
 107                                   stdout=rec_stdout,
 108                                   stderr=rec_stderr)
 109     time.sleep(3)
 110     assert rec_proc.poll() == None
 111     logging.info("started vnc2swf recorder (pid %d) on display :%d"
 112                  % (rec_proc.pid, disp))
 113     return rec_proc
 114
 115 def complete_xvnc_recording(vnc2swfpath, proc, variant, tmpdir):
 116     edit_stdout = open(os.path.join(tmpdir, "edit.stdout"), mode="w")
 117     edit_stderr = open(os.path.join(tmpdir, "edit.stderr"), mode="w")
 118     logging.info("killing vnc2swf recorder (pid %d)" % proc.pid)
 119     os.kill(proc.pid, signal.SIGINT)
 120     proc.wait()
 121     logging.info("down-sampling video")
 122     subprocess.Popen([os.path.join(vnc2swfpath, "edit.py"),
 123             "-c",
 124             "-o", small_variant_filename(variant),
 125             "-t", "video",
 126             "-s", "0.5",
 127             "-r", "32",
 128             large_variant_filename(variant)],
 129                         cwd=tmpdir,
 130                         shell=False,
 131                         stdout=edit_stdout,
 132                         stderr=edit_stderr).wait()
 133     #os.unlink(large_variant_html_filename(variant))
 134     #os.unlink(large_variant_filename(variant))
 135     logging.info("output video is in " + small_variant_filename(variant))
 136
 137
 138 ########################################################################
 139
 140 def write_vgopts(tmpdir, vgopts):
 141     f = open(os.path.join(tmpdir, ".valgrindrc"), "w")
 142     for i in vgopts:
 143         f.write(i + "\n")
 144     f.close()
 145
 146 ########################################################################
 147
 148 class Firefox_runner:
 149     def __init__(this, batchprefix, homedir, ffdir, timestr, tmpdir, disp):
 150         this.tmpdir = tmpdir
 151         this.homedir = homedir
 152         this.ffdir = ffdir
 153         this.profile = batchprefix + timestr
 154         this.profile_dir = os.path.join(this.tmpdir, this.profile)
 155         this.bin = os.path.join(this.ffdir, "firefox-bin")
 156         this.environ = {
 157             "PATH"            : os.getenv("PATH"),
 158             "HOME"            : this.homedir,
 159             "LD_LIBRARY_PATH" : this.ffdir,
 160             "MOZ_NO_REMOTE"   : "1",
 161 #            "DISPLAY"         : ":0",
 162             "DISPLAY"         : (":%d" % disp),
 163             }
 164
 165         this.kill_initial_profiles_ini()
 166         this.create_tmp_profile()
 167         this.write_proxy_pac_file()
 168         this.write_user_prefs()
 169         this.perform_initial_registration()
 170
 171     def kill_initial_profiles_ini(this):
 172         prof = os.path.join(this.homedir, ".mozilla")
 173         shutil.rmtree(prof, True)
 174         os.mkdir(prof)
 175
 176
 177     def clear_cache(this):
 178         shutil.rmtree(os.path.join(this.profile_dir, "Cache"), True)
 179
 180     def create_tmp_profile(this):
 181         subprocess.Popen ([this.bin, "-CreateProfile", (this.profile + " " + this.profile_dir)],
 182                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 183
 184     def write_proxy_pac_file(this):
 185         this.proxypac = os.path.join(this.tmpdir, "proxy.pac")
 186         p = open(this.proxypac, "w")
 187         p.write("function FindProxyForURL(url, host) {\n"
 188                 "  if (host == \"localhost\")\n"
 189                 "    return \"DIRECT\";\n"
 190                 "  else\n"
 191                 "    return \"PROXY localhost\";\n"
 192                 "}\n")
 193         p.close()
 194
 195     def write_user_prefs(this):
 196         userprefs = open(os.path.join(this.profile_dir, "user.js"), "w")
 197         userprefs.write("user_pref(\"network.proxy.autoconfig_url\", \"file://%s\");\n" % this.proxypac)
 198         userprefs.write("user_pref(\"network.proxy.type\", 2);\n")
 199         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 200         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 201         userprefs.write("user_pref(\"dom.allow_scripts_to_close_windows\", true);\n")
 202         userprefs.close()
 203
 204     def perform_initial_registration(this):
 205         dummy_file = os.path.join(this.tmpdir, "dummy.html")
 206         f = open(dummy_file, "w")
 207         f.write("<html><body onload=\"window.close()\" /></html>\n")
 208         f.close()
 209         logging.info("running dummy firefox under profile, for initial component-registration")
 210         subprocess.Popen ([this.bin, "-P", this.profile, "file://" + dummy_file],
 211                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 212
 213     def run_normal(this, url):
 214         ff_proc = subprocess.Popen ([this.bin, "-P", this.profile, url],
 215                                     cwd=this.tmpdir, shell=False, env=this.environ)
 216         logging.info("started 'firefox-bin ... %s' process (pid %d)"
 217                      % (url, ff_proc.pid))
 218         return ff_proc
 219
 220
 221     def run_valgrind(this, vg_tool, url):
 222         vg_proc = subprocess.Popen (["valgrind",
 223                                      "--tool=" + vg_tool,
 224                                      "--log-file=valgrind-" + vg_tool + "-log",
 225                                      this.bin, "-P", this.profile, url],
 226                                      cwd=this.tmpdir, shell=False, env=this.environ)
 227         logging.info("started 'valgrind --tool=%s firefox-bin ... %s' process (pid %d)"
 228                      % (vg_tool, url, vg_proc.pid))
 229
 230         return vg_proc
 231
 232
 233 ########################################################################
 234 # homebrew memory monitor until valgrind works properly
 235 ########################################################################
 236
 237
 238 class sampler:
 239
 240     def __init__(self, name):
 241         self.name = name
 242         self.max = 0
 243         self.final = 0
 244         self.sum = 0
 245         self.samples = 0
 246
 247     def sample(self):
 248         s = self.get_sample()
 249         self.samples += 1
 250         self.sum += s
 251         self.max = max(self.max, s)
 252         self.final = s
 253
 254     def report(self):
 255         self.samples = max(self.samples, 1)
 256         logging.info("__COUNT_%s_MAX!%d" % (self.name.upper(), self.max))
 257         logging.info("__COUNT_%s_AVG!%d" % (self.name.upper(), (self.sum / self.samples)))
 258         logging.info("__COUNT_%s_FINAL!%d" % (self.name.upper(), self.final))
 259
 260
 261 class proc_maps_heap_sampler(sampler):
 262
 263     def __init__(self, procpath):
 264         sampler.__init__(self, "heap")
 265         self.procpath = procpath
 266
 267     def get_sample(self):
 268         map_entry_rx = re.compile("^([0-9a-f]+)-([0-9a-f]+)\s+[-r][-w][-x][-p]\s+(?:\S+\s+){3}\[heap\]$")
 269         maps_path = os.path.join(self.procpath, "maps")
 270         maps_file = open(maps_path)
 271         for line in maps_file.xreadlines():
 272             m = map_entry_rx.match(line)
 273             if m:
 274                 (lo,hi) = m.group(1,2)
 275                 lo = int(lo, 16)
 276                 hi = int(hi, 16)
 277                 sz = hi - lo
 278                 maps_file.close()
 279                 return sz
 280         maps_file.close()
 281         return 0
 282
 283
 284 class proc_status_sampler(sampler):
 285
 286     def __init__(self, procpath, entry):
 287         sampler.__init__(self, entry)
 288         self.status_entry_rx = re.compile("^Vm%s:\s+(\d+) kB" % entry)
 289         self.procpath = procpath
 290
 291     def get_sample(self):
 292         status_path = os.path.join(self.procpath, "status")
 293         status_file = open(status_path)
 294         for line in status_file.xreadlines():
 295             m = self.status_entry_rx.match(line)
 296             if m:
 297                 status_file.close()
 298                 return int(m.group(1)) * 1024
 299
 300         status_file.close()
 301         return 0
 302
 303
 304 def wait_collecting_memory_stats(process):
 305
 306
 307     procdir = os.path.join("/proc", str(process.pid))
 308     samplers = [    proc_status_sampler(procdir, "RSS"),
 309             proc_status_sampler(procdir, "Size"),
 310             proc_maps_heap_sampler(procdir) ]
 311
 312     process.poll()
 313     while process.returncode == None:
 314
 315         for s in samplers:
 316             s.sample()
 317
 318         time.sleep(1)
 319         process.poll()
 320
 321
 322     for s in samplers:
 323         s.report()
 324
 325
 326 ########################################################################
 327 # config variables
 328 ########################################################################
 329
 330
 331 disp        = 25
 332 batchprefix = "batch-"
 333 homedir     = "/home/mozvalgrind"
 334 vnc2swfpath = "%s/pyvnc2swf-0.8.2" % homedir
 335 url         = "file://%s/workload.xml" % homedir
 336 probe_point = "nsWindow::SetTitle(*"
 337 num_frames  = 10
 338 vgopts   = [ "--memcheck:leak-check=yes",
 339              "--memcheck:error-limit=no",
 340              ("--memcheck:num-callers=%d" % num_frames),
 341 #             "--memcheck:leak-resolution=high",
 342 #             "--memcheck:show-reachable=yes",
 343              "--massif:format=html",
 344              ("--massif:depth=%d" % num_frames),
 345              "--massif:instrs=yes",
 346              "--callgrind:simulate-cache=yes",
 347              "--callgrind:simulate-hwpref=yes",
 348 #             ("--callgrind:dump-before=%s" % probe_point),
 349              "--callgrind:I1=65536,2,64",
 350              "--callgrind:D1=65536,2,64",
 351              "--callgrind:L2=524288,8,64",
 352          "--verbose"
 353              ]
 354
 355
 356 ######################################################
 357 # logging results
 358 ######################################################
 359
 360 def archive_dir(dir, sums):
 361     res = "current"
 362     tar = res + ".tar.gz"
 363
 364     if os.path.exists(res):
 365         shutil.rmtree(res)
 366
 367     if os.path.exists(tar):
 368         os.unlink(tar)
 369
 370     os.mkdir(res)
 371     ix = open(os.path.join(res, "index.html"), "w")
 372     ix.write("<html>\n<body>\n")
 373     ix.write("<h1>run: %s</h1>\n" % dir)
 374
 375     # summary info
 376     ix.write("<h2>Summary info</h2>\n")
 377     ix.write("<table>\n")
 378     for x in sums:
 379         ix.write("<tr><td>%s</td><td>%s</td></tr>\n" % (x, sums[x]))
 380     ix.write("</table>\n")
 381
 382
 383     # primary logs
 384     ix.write("<h2>Primary logs</h2>\n")
 385     for log in glob.glob(os.path.join(dir, "valgrind-*-log*")):
 386         (dirname, basename) = os.path.split(log)
 387         shutil.copy(log, os.path.join(res, basename))
 388         ix.write("<a href=\"%s\">%s</a><br />\n" % (basename, basename))
 389
 390
 391     # massif graphs
 392     ix.write("<h2>Massif results</h2>\n")
 393     ix.write("<h3>Click graph to see details</h3>\n")
 394     for mp in glob.glob(os.path.join(dir, "massif.*.ps")):
 395         (dirname,basename) = os.path.split(mp)
 396         (base,ext) = os.path.splitext(basename)
 397         png = base + ".png"
 398         html = base + ".html"
 399         subprocess.call(["convert", "-rotate", "270", mp, os.path.join(res, png)])
 400         shutil.copy(os.path.join(dir, html), os.path.join(res, html))
 401         ix.write("<a href=\"%s\"><img src=\"%s\" /></a><br />\n" % (html, png))
 402
 403     # run movies
 404     ix.write("<h2>Movies</h2>\n")
 405     for movie in ["memcheck", "massif", "callgrind"]:
 406         for ext in [".html", ".swf"]:
 407             base = movie + ext
 408             if os.path.exists(os.path.join(dir, base)):
 409                 shutil.copy(os.path.join(dir, base), os.path.join(res, base))
 410         if os.path.exists(os.path.join(res, movie + ".html")):
 411             ix.write("<a href=\"%s\">%s movie</a><br />\n" % (movie + ".html", movie))
 412
 413     # callgrind profile
 414     ix.write("<h2>Callgrind profiles</h2>\n")
 415     for cg in glob.glob(os.path.join(dir, "callgrind.out.*")):
 416         (dir, base) = os.path.split(cg)
 417         shutil.copy(cg, os.path.join(res, base))
 418         ix.write("<a href=\"%s\">%s</a><br />\n" % (base, base))
 419
 420     ix.write("</body>\n</html>\n")
 421     ix.close()
 422
 423     for i in glob.glob(os.path.join(res, "*")):
 424         os.chmod(i, 0644)
 425     os.chmod(res, 0755)
 426
 427     subprocess.call(["tar", "czvf", tar, res])
 428     os.chmod(tar, 0644)
 429
 430
 431 def log_result_summaries(tmpdir):
 432
 433     pats = {
 434         "IR"      :  re.compile("==\d+==\s+I\s+refs:\s+([0-9,]+)"),
 435         "ALLOC"      :  re.compile("==\d+==\s+malloc/free:\s+[0-9,]+\s+allocs,\s+[0-9,]+\s+frees,\s+([0-9,]+)\s+bytes allocated."),
 436         "LEAKED"  :  re.compile("==\d+==\s+(?:definitely|indirectly)\s+lost:\s+([0-9,]+)\s+bytes"),
 437         "DUBIOUS" :  re.compile("==\d+==\s+possibly\s+lost:\s+([0-9,]+)\s+bytes"),
 438         "LIVE"    :  re.compile("==\d+==\s+still\s+reachable:\s+([0-9,]+)\s+bytes"),
 439         }
 440
 441     sums = {}
 442
 443     for fname in glob.glob("%s/valgrind-*-log*" % tmpdir):
 444         f = open(fname)
 445
 446         for line in f.xreadlines():
 447             for pat in pats:
 448                 rx = pats[pat]
 449                 match = rx.search(line)
 450                 if match:
 451                     val = int(match.group(1).replace(",", ""))
 452                     if pat in sums:
 453                         val = val + sums[pat]
 454                     sums[pat] = val
 455         f.close()
 456
 457     for pat in sums:
 458         logging.info("__COUNT_%s!%d" % (pat, sums[pat]))
 459
 460     archive_dir(tmpdir, sums)
 461
 462
 463
 464 ########################################################################
 465 # main
 466 ########################################################################
 467
 468 if len(sys.argv) != 2:
 469     print("usage: %s <firefox-bin build dir>" % sys.argv[0])
 470     sys.exit(1)
 471
 472 logging.basicConfig(level=logging.INFO,
 473                     format='%(asctime)s %(levelname)s %(message)s')
 474
 475 logging.info("running as %s" % getpass.getuser())
 476
 477 logging.info("killing any residual processes in this group")
 478 subprocess.call(["killall", "-q", "-9", "memcheck", "callgrind", "massif", "valgrind", "firefox-bin"])
 479 time.sleep(3)
 480
 481 dirs=glob.glob(os.path.join(homedir, batchprefix + "*"))
 482 dirs.sort()
 483 if len(dirs) > 5:
 484     for olddir in dirs[:-5]:
 485         logging.info("cleaning up old directory %s" % olddir)
 486         shutil.rmtree(olddir)
 487
 488
 489 timestr = datetime.datetime.now().isoformat().replace(":", "-")
 490 tmpdir = tempfile.mkdtemp(prefix=(batchprefix + timestr + "-"), dir=homedir)
 491
 492 logging.info("started batch run in dir " + tmpdir)
 493
 494 os.chdir(tmpdir)
 495 ffdir = sys.argv[1]
 496
 497 write_vgopts(tmpdir, vgopts)
 498
 499 xvnc_proc = None
 500 runner = None
 501 recorder = None
 502
 503 ######################################################
 504 # note: runit is supervising a single Xvnc on disp 25
 505 # there is no need to run one here as well
 506 ######################################################
 507 # xvnc_proc = start_xvnc(disp, tmpdir)
 508 ######################################################
 509
 510 try:
 511
 512     runner = Firefox_runner(batchprefix, homedir, ffdir, timestr, tmpdir, disp)
 513
 514     wait_collecting_memory_stats(runner.run_normal(url))
 515     runner.clear_cache()
 516
 517 #    for variant in ["memcheck", "massif", "callgrind"]:
 518 #        recorder = start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir)
 519 #        runner.run_valgrind(variant, url).wait()
 520 #        runner.clear_cache()
 521 #        complete_xvnc_recording(vnc2swfpath, recorder, variant, tmpdir)
 522
 523     log_result_summaries(tmpdir)
 524     logging.info("valgrind-firefox processes complete")
 525
 526 finally:
 527     if recorder and recorder.poll() == None:
 528         logging.info("killing recorder process %d" % recorder.pid)
 529     os.kill(recorder.pid, signal.SIGKILL)
 530     if xvnc_proc and xvnc_proc.poll() == None:
 531         logging.info("killing Xvnc process %d" % xvnc_proc.pid)
 532         os.kill(xvnc_proc.pid, signal.SIGKILL)
 533
 534 logging.info("batch run in dir %s complete" % tmpdir)