testing/tests/memtest.py

   1 #!/usr/bin/python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is standalone Firefox memory test.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # Mozilla Corporation.
  20 # Portions created by the Initial Developer are Copyright (C) 2006
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #   Graydon Hoare <graydon@mozilla.com> (original author)
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40
  41 import subprocess
  42 import tempfile
  43 import os
  44 import re
  45 import getpass
  46 import datetime
  47 import logging
  48 import time
  49 import sys
  50 import signal
  51 import shutil
  52 import glob
  53
  54 def sighandler(signal, frame):
  55         print "signal %d, throwing" % signal
  56         raise Exception
  57
  58 signal.signal(signal.SIGHUP, sighandler)
  59
  60
  61 ########################################################################
  62
  63 def start_xvnc(disp, tmpdir):
  64     xvnc_stdout = open(os.path.join(tmpdir, "xvnc.stdout"), mode="w")
  65     xvnc_stderr = open(os.path.join(tmpdir, "xvnc.stderr"), mode="w")
  66     xvnc_proc = subprocess.Popen (["Xvnc",
  67                    "-fp", "/usr/share/X11/fonts/misc",
  68                                    "-localhost",
  69                                    "-SecurityTypes", "None",
  70                    "-IdleTimeout", "604800",
  71                                    "-depth",         "24",
  72                                    "-ac",
  73                                    (":%d" % disp)],
  74                                   cwd=tmpdir,
  75                                   shell=False,
  76                                   stdout=xvnc_stdout,
  77                                   stderr=xvnc_stderr)
  78     time.sleep(3)
  79     assert xvnc_proc.poll() == None
  80     logging.info("started Xvnc server (pid %d) on display :%d"
  81                  % (xvnc_proc.pid, disp))
  82     return xvnc_proc
  83
  84 ########################################################################
  85
  86 def large_variant_filename(variant):
  87     return variant + "-large.swf"
  88
  89 def large_variant_html_filename(variant):
  90     return variant + "-large.html"
  91
  92 def small_variant_filename(variant):
  93     return variant + ".swf"
  94
  95 def start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir):
  96     rec_stdout = open(os.path.join(tmpdir, "vnc2swf.stdout"), mode="w")
  97     rec_stderr = open(os.path.join(tmpdir, "vnc2swf.stderr"), mode="w")
  98     rec_proc = subprocess.Popen ([os.path.join(vnc2swfpath, "vnc2swf.py"),
  99                   "-n",
 100                   "-o", large_variant_filename(variant),
 101                   "-C", "800x600+0+100",
 102                   "-r", "2",
 103                   "-t", "video",
 104                   "localhost:%d" % disp],
 105                                   cwd=tmpdir,
 106                                   shell=False,
 107                                   stdout=rec_stdout,
 108                                   stderr=rec_stderr)
 109     time.sleep(3)
 110     assert rec_proc.poll() == None
 111     logging.info("started vnc2swf recorder (pid %d) on display :%d"
 112                  % (rec_proc.pid, disp))
 113     return rec_proc
 114
 115 def complete_xvnc_recording(vnc2swfpath, proc, variant, tmpdir):
 116     edit_stdout = open(os.path.join(tmpdir, "edit.stdout"), mode="w")
 117     edit_stderr = open(os.path.join(tmpdir, "edit.stderr"), mode="w")
 118     logging.info("killing vnc2swf recorder (pid %d)" % proc.pid)
 119     os.kill(proc.pid, signal.SIGINT)
 120     proc.wait()
 121     logging.info("down-sampling video")
 122     subprocess.Popen([os.path.join(vnc2swfpath, "edit.py"),
 123             "-c",
 124             "-o", small_variant_filename(variant),
 125             "-t", "video",
 126             "-s", "0.5",
 127             "-r", "32",
 128             large_variant_filename(variant)],
 129                         cwd=tmpdir,
 130                         shell=False,
 131                         stdout=edit_stdout,
 132                         stderr=edit_stderr).wait()
 133     #os.unlink(large_variant_html_filename(variant))
 134     #os.unlink(large_variant_filename(variant))
 135     logging.info("output video is in " + small_variant_filename(variant))
 136
 137
 138 ########################################################################
 139
 140 def write_vgopts(tmpdir, vgopts):
 141     f = open(os.path.join(tmpdir, ".valgrindrc"), "w")
 142     for i in vgopts:
 143         f.write(i + "\n")
 144     f.close()
 145
 146 ########################################################################
 147
 148 class Firefox_runner:
 149     def __init__(this, batchprefix, homedir, ffdir, timestr, tmpdir, disp):
 150         this.tmpdir = tmpdir
 151         this.homedir = homedir
 152         this.ffdir = ffdir
 153         this.profile = batchprefix + timestr
 154         this.profile_dir = os.path.join(this.tmpdir, this.profile)
 155         this.bin = os.path.join(this.ffdir, "firefox-bin")
 156         this.environ = {
 157             "PATH"            : os.getenv("PATH"),
 158             "HOME"            : this.homedir,
 159             "LD_LIBRARY_PATH" : this.ffdir,
 160             "MOZ_NO_REMOTE"   : "1",
 161 #            "DISPLAY"         : ":0",
 162             "DISPLAY"         : (":%d" % disp),
 163             }
 164
 165         this.kill_initial_profiles_ini()
 166         this.create_tmp_profile()
 167         this.write_proxy_pac_file()
 168         this.write_user_prefs()
 169         this.perform_initial_registration()
 170
 171     def kill_initial_profiles_ini(this):
 172         prof = os.path.join(this.homedir, ".mozilla")
 173         shutil.rmtree(prof, True)
 174         os.mkdir(prof)
 175
 176
 177     def clear_cache(this):
 178         shutil.rmtree(os.path.join(this.profile_dir, "Cache"), True)
 179
 180     def create_tmp_profile(this):
 181         subprocess.Popen ([this.bin, "-CreateProfile", (this.profile + " " + this.profile_dir)],
 182                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 183
 184     def write_proxy_pac_file(this):
 185         this.proxypac = os.path.join(this.tmpdir, "proxy.pac")
 186         p = open(this.proxypac, "w")
 187         p.write("function FindProxyForURL(url, host) {\n"
 188                 "  if (host == \"localhost\")\n"
 189                 "    return \"DIRECT\";\n"
 190                 "  else\n"
 191                 "    return \"PROXY localhost\";\n"
 192                 "}\n")
 193         p.close()
 194
 195     def write_user_prefs(this):
 196         userprefs = open(os.path.join(this.profile_dir, "user.js"), "w")
 197         userprefs.write("user_pref(\"network.proxy.autoconfig_url\", \"file://%s\");\n" % this.proxypac)
 198         userprefs.write("user_pref(\"network.proxy.type\", 2);\n")
 199         userprefs.write("user_pref(\"privacy.popups.firstTime\", false);\n")
 200         userprefs.write("user_pref(\"plugin.default_plugin_disabled\", false);\n")
 201         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 202         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 203         userprefs.write("user_pref(\"dom.allow_scripts_to_close_windows\", true);\n")
 204         userprefs.close()
 205
 206     def perform_initial_registration(this):
 207         dummy_file = os.path.join(this.tmpdir, "dummy.html")
 208         f = open(dummy_file, "w")
 209         f.write("<html><body onload=\"window.close()\" /></html>\n")
 210         f.close()
 211         logging.info("running dummy firefox under profile, for initial component-registration")
 212         subprocess.Popen ([this.bin, "-P", this.profile, "file://" + dummy_file],
 213                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 214
 215     def run_normal(this, url):
 216         ff_proc = subprocess.Popen ([this.bin, "-P", this.profile, url],
 217                                     cwd=this.tmpdir, shell=False, env=this.environ)
 218         logging.info("started 'firefox-bin ... %s' process (pid %d)"
 219                      % (url, ff_proc.pid))
 220         return ff_proc
 221
 222
 223     def run_valgrind(this, vg_tool, url):
 224         vg_proc = subprocess.Popen (["valgrind",
 225                                      "--tool=" + vg_tool,
 226                                      "--log-file=valgrind-" + vg_tool + "-log",
 227                                      this.bin, "-P", this.profile, url],
 228                                      cwd=this.tmpdir, shell=False, env=this.environ)
 229         logging.info("started 'valgrind --tool=%s firefox-bin ... %s' process (pid %d)"
 230                      % (vg_tool, url, vg_proc.pid))
 231
 232         return vg_proc
 233
 234
 235 ########################################################################
 236 # homebrew memory monitor until valgrind works properly
 237 ########################################################################
 238
 239
 240 class sampler:
 241
 242     def __init__(self, name):
 243         self.name = name
 244         self.max = 0
 245         self.final = 0
 246         self.sum = 0
 247         self.samples = 0
 248
 249     def sample(self):
 250         s = self.get_sample()
 251         self.samples += 1
 252         self.sum += s
 253         self.max = max(self.max, s)
 254         self.final = s
 255
 256     def report(self):
 257         self.samples = max(self.samples, 1)
 258         logging.info("__COUNT_%s_MAX!%d" % (self.name.upper(), self.max))
 259         logging.info("__COUNT_%s_AVG!%d" % (self.name.upper(), (self.sum / self.samples)))
 260         logging.info("__COUNT_%s_FINAL!%d" % (self.name.upper(), self.final))
 261
 262
 263 class proc_maps_heap_sampler(sampler):
 264
 265     def __init__(self, procpath):
 266         sampler.__init__(self, "heap")
 267         self.procpath = procpath
 268
 269     def get_sample(self):
 270         map_entry_rx = re.compile("^([0-9a-f]+)-([0-9a-f]+)\s+[-r][-w][-x][-p]\s+(?:\S+\s+){3}\[heap\]$")
 271         maps_path = os.path.join(self.procpath, "maps")
 272         maps_file = open(maps_path)
 273         for line in maps_file.xreadlines():
 274             m = map_entry_rx.match(line)
 275             if m:
 276                 (lo,hi) = m.group(1,2)
 277                 lo = int(lo, 16)
 278                 hi = int(hi, 16)
 279                 sz = hi - lo
 280                 maps_file.close()
 281                 return sz
 282         maps_file.close()
 283         return 0
 284
 285
 286 class proc_status_sampler(sampler):
 287
 288     def __init__(self, procpath, entry):
 289         sampler.__init__(self, entry)
 290         self.status_entry_rx = re.compile("^Vm%s:\s+(\d+) kB" % entry)
 291         self.procpath = procpath
 292
 293     def get_sample(self):
 294         status_path = os.path.join(self.procpath, "status")
 295         status_file = open(status_path)
 296         for line in status_file.xreadlines():
 297             m = self.status_entry_rx.match(line)
 298             if m:
 299                 status_file.close()
 300                 return int(m.group(1)) * 1024
 301
 302         status_file.close()
 303         return 0
 304
 305
 306 def wait_collecting_memory_stats(process):
 307
 308
 309     procdir = os.path.join("/proc", str(process.pid))
 310     samplers = [    proc_status_sampler(procdir, "RSS"),
 311             proc_status_sampler(procdir, "Size"),
 312             proc_maps_heap_sampler(procdir) ]
 313
 314     process.poll()
 315     while process.returncode == None:
 316
 317         for s in samplers:
 318             s.sample()
 319
 320         time.sleep(1)
 321         process.poll()
 322
 323
 324     for s in samplers:
 325         s.report()
 326
 327
 328 ########################################################################
 329 # config variables
 330 ########################################################################
 331
 332
 333 disp        = 25
 334 batchprefix = "batch-"
 335 homedir     = "/home/mozvalgrind"
 336 vnc2swfpath = "%s/pyvnc2swf-0.8.2" % homedir
 337 url         = "file://%s/workload.xml" % homedir
 338 probe_point = "nsWindow::SetTitle(*"
 339 num_frames  = 10
 340 vgopts   = [ "--memcheck:leak-check=yes",
 341              "--memcheck:error-limit=no",
 342              ("--memcheck:num-callers=%d" % num_frames),
 343 #             "--memcheck:leak-resolution=high",
 344 #             "--memcheck:show-reachable=yes",
 345              "--massif:format=html",
 346              ("--massif:depth=%d" % num_frames),
 347              "--massif:instrs=yes",
 348              "--callgrind:simulate-cache=yes",
 349              "--callgrind:simulate-hwpref=yes",
 350 #             ("--callgrind:dump-before=%s" % probe_point),
 351              "--callgrind:I1=65536,2,64",
 352              "--callgrind:D1=65536,2,64",
 353              "--callgrind:L2=524288,8,64",
 354          "--verbose"
 355              ]
 356
 357
 358 ######################################################
 359 # logging results
 360 ######################################################
 361
 362 def archive_dir(dir, sums):
 363     res = "current"
 364     tar = res + ".tar.gz"
 365
 366     if os.path.exists(res):
 367         shutil.rmtree(res)
 368
 369     if os.path.exists(tar):
 370         os.unlink(tar)
 371
 372     os.mkdir(res)
 373     ix = open(os.path.join(res, "index.html"), "w")
 374     ix.write("<html>\n<body>\n")
 375     ix.write("<h1>run: %s</h1>\n" % dir)
 376
 377     # summary info
 378     ix.write("<h2>Summary info</h2>\n")
 379     ix.write("<table>\n")
 380     for x in sums:
 381         ix.write("<tr><td>%s</td><td>%s</td></tr>\n" % (x, sums[x]))
 382     ix.write("</table>\n")
 383
 384
 385     # primary logs
 386     ix.write("<h2>Primary logs</h2>\n")
 387     for log in glob.glob(os.path.join(dir, "valgrind-*-log*")):
 388         (dirname, basename) = os.path.split(log)
 389         shutil.copy(log, os.path.join(res, basename))
 390         ix.write("<a href=\"%s\">%s</a><br />\n" % (basename, basename))
 391
 392
 393     # massif graphs
 394     ix.write("<h2>Massif results</h2>\n")
 395     ix.write("<h3>Click graph to see details</h3>\n")
 396     for mp in glob.glob(os.path.join(dir, "massif.*.ps")):
 397         (dirname,basename) = os.path.split(mp)
 398         (base,ext) = os.path.splitext(basename)
 399         png = base + ".png"
 400         html = base + ".html"
 401         subprocess.call(["convert", "-rotate", "270", mp, os.path.join(res, png)])
 402         shutil.copy(os.path.join(dir, html), os.path.join(res, html))
 403         ix.write("<a href=\"%s\"><img src=\"%s\" /></a><br />\n" % (html, png))
 404
 405     # run movies
 406     ix.write("<h2>Movies</h2>\n")
 407     for movie in ["memcheck", "massif", "callgrind"]:
 408         for ext in [".html", ".swf"]:
 409             base = movie + ext
 410             if os.path.exists(os.path.join(dir, base)):
 411                 shutil.copy(os.path.join(dir, base), os.path.join(res, base))
 412         if os.path.exists(os.path.join(res, movie + ".html")):
 413             ix.write("<a href=\"%s\">%s movie</a><br />\n" % (movie + ".html", movie))
 414
 415     # callgrind profile
 416     ix.write("<h2>Callgrind profiles</h2>\n")
 417     for cg in glob.glob(os.path.join(dir, "callgrind.out.*")):
 418         (dir, base) = os.path.split(cg)
 419         shutil.copy(cg, os.path.join(res, base))
 420         ix.write("<a href=\"%s\">%s</a><br />\n" % (base, base))
 421
 422     ix.write("</body>\n</html>\n")
 423     ix.close()
 424
 425     for i in glob.glob(os.path.join(res, "*")):
 426         os.chmod(i, 0644)
 427     os.chmod(res, 0755)
 428
 429     subprocess.call(["tar", "czvf", tar, res])
 430     os.chmod(tar, 0644)
 431
 432
 433 def log_result_summaries(tmpdir):
 434
 435     pats = {
 436         "IR"      :  re.compile("==\d+==\s+I\s+refs:\s+([0-9,]+)"),
 437         "ALLOC"      :  re.compile("==\d+==\s+malloc/free:\s+[0-9,]+\s+allocs,\s+[0-9,]+\s+frees,\s+([0-9,]+)\s+bytes allocated."),
 438         "LEAKED"  :  re.compile("==\d+==\s+(?:definitely|indirectly)\s+lost:\s+([0-9,]+)\s+bytes"),
 439         "DUBIOUS" :  re.compile("==\d+==\s+possibly\s+lost:\s+([0-9,]+)\s+bytes"),
 440         "LIVE"    :  re.compile("==\d+==\s+still\s+reachable:\s+([0-9,]+)\s+bytes"),
 441         }
 442
 443     sums = {}
 444
 445     for fname in glob.glob("%s/valgrind-*-log*" % tmpdir):
 446         f = open(fname)
 447
 448         for line in f.xreadlines():
 449             for pat in pats:
 450                 rx = pats[pat]
 451                 match = rx.search(line)
 452                 if match:
 453                     val = int(match.group(1).replace(",", ""))
 454                     if pat in sums:
 455                         val = val + sums[pat]
 456                     sums[pat] = val
 457         f.close()
 458
 459     for pat in sums:
 460         logging.info("__COUNT_%s!%d" % (pat, sums[pat]))
 461
 462     archive_dir(tmpdir, sums)
 463
 464
 465
 466 ########################################################################
 467 # main
 468 ########################################################################
 469
 470 if len(sys.argv) != 2:
 471     print("usage: %s <firefox-bin build dir>" % sys.argv[0])
 472     sys.exit(1)
 473
 474 logging.basicConfig(level=logging.INFO,
 475                     format='%(asctime)s %(levelname)s %(message)s')
 476
 477 logging.info("running as %s" % getpass.getuser())
 478
 479 logging.info("killing any residual processes in this group")
 480 subprocess.call(["killall", "-q", "-9", "memcheck", "callgrind", "massif", "valgrind", "firefox-bin"])
 481 time.sleep(3)
 482
 483 dirs=glob.glob(os.path.join(homedir, batchprefix + "*"))
 484 dirs.sort()
 485 if len(dirs) > 5:
 486     for olddir in dirs[:-5]:
 487         logging.info("cleaning up old directory %s" % olddir)
 488         shutil.rmtree(olddir)
 489
 490
 491 timestr = datetime.datetime.now().isoformat().replace(":", "-")
 492 tmpdir = tempfile.mkdtemp(prefix=(batchprefix + timestr + "-"), dir=homedir)
 493
 494 logging.info("started batch run in dir " + tmpdir)
 495
 496 os.chdir(tmpdir)
 497 ffdir = sys.argv[1]
 498
 499 write_vgopts(tmpdir, vgopts)
 500
 501 xvnc_proc = None
 502 runner = None
 503 recorder = None
 504
 505 ######################################################
 506 # note: runit is supervising a single Xvnc on disp 25
 507 # there is no need to run one here as well
 508 ######################################################
 509 # xvnc_proc = start_xvnc(disp, tmpdir)
 510 ######################################################
 511
 512 try:
 513
 514     runner = Firefox_runner(batchprefix, homedir, ffdir, timestr, tmpdir, disp)
 515
 516     wait_collecting_memory_stats(runner.run_normal(url))
 517     runner.clear_cache()
 518
 519 #    for variant in ["memcheck", "massif", "callgrind"]:
 520 #        recorder = start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir)
 521 #        runner.run_valgrind(variant, url).wait()
 522 #        runner.clear_cache()
 523 #        complete_xvnc_recording(vnc2swfpath, recorder, variant, tmpdir)
 524
 525     log_result_summaries(tmpdir)
 526     logging.info("valgrind-firefox processes complete")
 527
 528 finally:
 529     if recorder and recorder.poll() == None:
 530         logging.info("killing recorder process %d" % recorder.pid)
 531     os.kill(recorder.pid, signal.SIGKILL)
 532     if xvnc_proc and xvnc_proc.poll() == None:
 533         logging.info("killing Xvnc process %d" % xvnc_proc.pid)
 534         os.kill(xvnc_proc.pid, signal.SIGKILL)
 535
 536 logging.info("batch run in dir %s complete" % tmpdir)