testing/tests/memtest.py

   1 #!/usr/bin/python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is standalone Firefox memory test.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # Mozilla Corporation.
  20 # Portions created by the Initial Developer are Copyright (C) 2006
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #   Graydon Hoare <graydon@mozilla.com> (original author)
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40
  41 import subprocess
  42 import tempfile
  43 import os
  44 import re
  45 import getpass
  46 import datetime
  47 import logging
  48 import time
  49 import sys
  50 import signal
  51 import shutil
  52 import glob
  53
  54 def sighandler(signal, frame):
  55         print "signal %d, throwing" % signal
  56         raise Exception
  57
  58 signal.signal(signal.SIGHUP, sighandler)
  59
  60
  61 ########################################################################
  62
  63 def start_xvnc(disp, tmpdir):
  64     xvnc_stdout = open(os.path.join(tmpdir, "xvnc.stdout"), mode="w")
  65     xvnc_stderr = open(os.path.join(tmpdir, "xvnc.stderr"), mode="w")
  66     xvnc_proc = subprocess.Popen (["Xvnc",
  67                    "-fp", "/usr/share/X11/fonts/misc",
  68                                    "-localhost",
  69                                    "-SecurityTypes", "None",
  70                    "-IdleTimeout", "604800",
  71                                    "-depth",         "24",
  72                                    "-ac",
  73                                    (":%d" % disp)],
  74                                   cwd=tmpdir,
  75                                   shell=False,
  76                                   stdout=xvnc_stdout,
  77                                   stderr=xvnc_stderr)
  78     time.sleep(3)
  79     assert xvnc_proc.poll() == None
  80     logging.info("started Xvnc server (pid %d) on display :%d"
  81                  % (xvnc_proc.pid, disp))
  82     return xvnc_proc
  83
  84 ########################################################################
  85
  86 def large_variant_filename(variant):
  87     return variant + "-large.swf"
  88
  89 def large_variant_html_filename(variant):
  90     return variant + "-large.html"
  91
  92 def small_variant_filename(variant):
  93     return variant + ".swf"
  94
  95 def start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir):
  96     rec_stdout = open(os.path.join(tmpdir, "vnc2swf.stdout"), mode="w")
  97     rec_stderr = open(os.path.join(tmpdir, "vnc2swf.stderr"), mode="w")
  98     rec_proc = subprocess.Popen ([os.path.join(vnc2swfpath, "vnc2swf.py"),
  99                   "-n",
 100                   "-o", large_variant_filename(variant),
 101                   "-C", "800x600+0+100",
 102                   "-r", "2",
 103                   "-t", "video",
 104                   "localhost:%d" % disp],
 105                                   cwd=tmpdir,
 106                                   shell=False,
 107                                   stdout=rec_stdout,
 108                                   stderr=rec_stderr)
 109     time.sleep(3)
 110     assert rec_proc.poll() == None
 111     logging.info("started vnc2swf recorder (pid %d) on display :%d"
 112                  % (rec_proc.pid, disp))
 113     return rec_proc
 114
 115 def complete_xvnc_recording(vnc2swfpath, proc, variant, tmpdir):
 116     edit_stdout = open(os.path.join(tmpdir, "edit.stdout"), mode="w")
 117     edit_stderr = open(os.path.join(tmpdir, "edit.stderr"), mode="w")
 118     logging.info("killing vnc2swf recorder (pid %d)" % proc.pid)
 119     os.kill(proc.pid, signal.SIGINT)
 120     proc.wait()
 121     logging.info("down-sampling video")
 122     subprocess.Popen([os.path.join(vnc2swfpath, "edit.py"),
 123             "-c",
 124             "-o", small_variant_filename(variant),
 125             "-t", "video",
 126             "-s", "0.5",
 127             "-r", "32",
 128             large_variant_filename(variant)],
 129                         cwd=tmpdir,
 130                         shell=False,
 131                         stdout=edit_stdout,
 132                         stderr=edit_stderr).wait()
 133     #os.unlink(large_variant_html_filename(variant))
 134     #os.unlink(large_variant_filename(variant))
 135     logging.info("output video is in " + small_variant_filename(variant))
 136
 137
 138 ########################################################################
 139
 140 def write_vgopts(tmpdir, vgopts):
 141     f = open(os.path.join(tmpdir, ".valgrindrc"), "w")
 142     for i in vgopts:
 143         f.write(i + "\n")
 144     f.close()
 145
 146 ########################################################################
 147
 148 class Firefox_runner:
 149     def __init__(this, batchprefix, homedir, ffdir, timestr, tmpdir, disp):
 150         this.tmpdir = tmpdir
 151         this.homedir = homedir
 152         this.ffdir = ffdir
 153         this.profile = batchprefix + timestr
 154         this.profile_dir = os.path.join(this.tmpdir, this.profile)
 155         this.bin = os.path.join(this.ffdir, "firefox-bin")
 156         this.environ = {
 157             "PATH"            : os.getenv("PATH"),
 158             "HOME"            : this.homedir,
 159             "LD_LIBRARY_PATH" : this.ffdir,
 160             "MOZ_NO_REMOTE"   : "1",
 161 #            "DISPLAY"         : ":0",
 162             "DISPLAY"         : (":%d" % disp),
 163             }
 164
 165         this.kill_initial_profiles_ini()
 166         this.create_tmp_profile()
 167         this.write_proxy_pac_file()
 168         this.write_user_prefs()
 169         this.perform_initial_registration()
 170
 171     def kill_initial_profiles_ini(this):
 172         prof = os.path.join(this.homedir, ".mozilla")
 173         shutil.rmtree(prof, True)
 174         os.mkdir(prof)
 175
 176
 177     def clear_cache(this):
 178         shutil.rmtree(os.path.join(this.profile_dir, "Cache"), True)
 179
 180     def create_tmp_profile(this):
 181         subprocess.Popen ([this.bin, "-CreateProfile", (this.profile + " " + this.profile_dir)],
 182                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 183
 184     def write_proxy_pac_file(this):
 185         this.proxypac = os.path.join(this.tmpdir, "proxy.pac")
 186         p = open(this.proxypac, "w")
 187         p.write("function FindProxyForURL(url, host) {\n"
 188                 "  if (host == \"localhost\")\n"
 189                 "    return \"DIRECT\";\n"
 190                 "  else\n"
 191                 "    return \"PROXY localhost\";\n"
 192                 "}\n")
 193         p.close()
 194
 195     def write_user_prefs(this):
 196         userprefs = open(os.path.join(this.profile_dir, "user.js"), "w")
 197         userprefs.write("user_pref(\"network.proxy.autoconfig_url\", \"file://%s\");\n" % this.proxypac)
 198         userprefs.write("user_pref(\"network.proxy.type\", 2);\n")
 199         userprefs.write("user_pref(\"privacy.popups.firstTime\", false);\n")
 200         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 201         userprefs.write("user_pref(\"dom.max_script_run_time\", 0);\n")
 202         userprefs.write("user_pref(\"dom.allow_scripts_to_close_windows\", true);\n")
 203         userprefs.close()
 204
 205     def perform_initial_registration(this):
 206         dummy_file = os.path.join(this.tmpdir, "dummy.html")
 207         f = open(dummy_file, "w")
 208         f.write("<html><body onload=\"window.close()\" /></html>\n")
 209         f.close()
 210         logging.info("running dummy firefox under profile, for initial component-registration")
 211         subprocess.Popen ([this.bin, "-P", this.profile, "file://" + dummy_file],
 212                           cwd=this.tmpdir, shell=False, env=this.environ).wait()
 213
 214     def run_normal(this, url):
 215         ff_proc = subprocess.Popen ([this.bin, "-P", this.profile, url],
 216                                     cwd=this.tmpdir, shell=False, env=this.environ)
 217         logging.info("started 'firefox-bin ... %s' process (pid %d)"
 218                      % (url, ff_proc.pid))
 219         return ff_proc
 220
 221
 222     def run_valgrind(this, vg_tool, url):
 223         vg_proc = subprocess.Popen (["valgrind",
 224                                      "--tool=" + vg_tool,
 225                                      "--log-file=valgrind-" + vg_tool + "-log",
 226                                      this.bin, "-P", this.profile, url],
 227                                      cwd=this.tmpdir, shell=False, env=this.environ)
 228         logging.info("started 'valgrind --tool=%s firefox-bin ... %s' process (pid %d)"
 229                      % (vg_tool, url, vg_proc.pid))
 230
 231         return vg_proc
 232
 233
 234 ########################################################################
 235 # homebrew memory monitor until valgrind works properly
 236 ########################################################################
 237
 238
 239 class sampler:
 240
 241     def __init__(self, name):
 242         self.name = name
 243         self.max = 0
 244         self.final = 0
 245         self.sum = 0
 246         self.samples = 0
 247
 248     def sample(self):
 249         s = self.get_sample()
 250         self.samples += 1
 251         self.sum += s
 252         self.max = max(self.max, s)
 253         self.final = s
 254
 255     def report(self):
 256         self.samples = max(self.samples, 1)
 257         logging.info("__COUNT_%s_MAX!%d" % (self.name.upper(), self.max))
 258         logging.info("__COUNT_%s_AVG!%d" % (self.name.upper(), (self.sum / self.samples)))
 259         logging.info("__COUNT_%s_FINAL!%d" % (self.name.upper(), self.final))
 260
 261
 262 class proc_maps_heap_sampler(sampler):
 263
 264     def __init__(self, procpath):
 265         sampler.__init__(self, "heap")
 266         self.procpath = procpath
 267
 268     def get_sample(self):
 269         map_entry_rx = re.compile("^([0-9a-f]+)-([0-9a-f]+)\s+[-r][-w][-x][-p]\s+(?:\S+\s+){3}\[heap\]$")
 270         maps_path = os.path.join(self.procpath, "maps")
 271         maps_file = open(maps_path)
 272         for line in maps_file.xreadlines():
 273             m = map_entry_rx.match(line)
 274             if m:
 275                 (lo,hi) = m.group(1,2)
 276                 lo = int(lo, 16)
 277                 hi = int(hi, 16)
 278                 sz = hi - lo
 279                 maps_file.close()
 280                 return sz
 281         maps_file.close()
 282         return 0
 283
 284
 285 class proc_status_sampler(sampler):
 286
 287     def __init__(self, procpath, entry):
 288         sampler.__init__(self, entry)
 289         self.status_entry_rx = re.compile("^Vm%s:\s+(\d+) kB" % entry)
 290         self.procpath = procpath
 291
 292     def get_sample(self):
 293         status_path = os.path.join(self.procpath, "status")
 294         status_file = open(status_path)
 295         for line in status_file.xreadlines():
 296             m = self.status_entry_rx.match(line)
 297             if m:
 298                 status_file.close()
 299                 return int(m.group(1)) * 1024
 300
 301         status_file.close()
 302         return 0
 303
 304
 305 def wait_collecting_memory_stats(process):
 306
 307
 308     procdir = os.path.join("/proc", str(process.pid))
 309     samplers = [    proc_status_sampler(procdir, "RSS"),
 310             proc_status_sampler(procdir, "Size"),
 311             proc_maps_heap_sampler(procdir) ]
 312
 313     process.poll()
 314     while process.returncode == None:
 315
 316         for s in samplers:
 317             s.sample()
 318
 319         time.sleep(1)
 320         process.poll()
 321
 322
 323     for s in samplers:
 324         s.report()
 325
 326
 327 ########################################################################
 328 # config variables
 329 ########################################################################
 330
 331
 332 disp        = 25
 333 batchprefix = "batch-"
 334 homedir     = "/home/mozvalgrind"
 335 vnc2swfpath = "%s/pyvnc2swf-0.8.2" % homedir
 336 url         = "file://%s/workload.xml" % homedir
 337 probe_point = "nsWindow::SetTitle(*"
 338 num_frames  = 10
 339 vgopts   = [ "--memcheck:leak-check=yes",
 340              "--memcheck:error-limit=no",
 341              ("--memcheck:num-callers=%d" % num_frames),
 342 #             "--memcheck:leak-resolution=high",
 343 #             "--memcheck:show-reachable=yes",
 344              "--massif:format=html",
 345              ("--massif:depth=%d" % num_frames),
 346              "--massif:instrs=yes",
 347              "--callgrind:simulate-cache=yes",
 348              "--callgrind:simulate-hwpref=yes",
 349 #             ("--callgrind:dump-before=%s" % probe_point),
 350              "--callgrind:I1=65536,2,64",
 351              "--callgrind:D1=65536,2,64",
 352              "--callgrind:L2=524288,8,64",
 353          "--verbose"
 354              ]
 355
 356
 357 ######################################################
 358 # logging results
 359 ######################################################
 360
 361 def archive_dir(dir, sums):
 362     res = "current"
 363     tar = res + ".tar.gz"
 364
 365     if os.path.exists(res):
 366         shutil.rmtree(res)
 367
 368     if os.path.exists(tar):
 369         os.unlink(tar)
 370
 371     os.mkdir(res)
 372     ix = open(os.path.join(res, "index.html"), "w")
 373     ix.write("<html>\n<body>\n")
 374     ix.write("<h1>run: %s</h1>\n" % dir)
 375
 376     # summary info
 377     ix.write("<h2>Summary info</h2>\n")
 378     ix.write("<table>\n")
 379     for x in sums:
 380         ix.write("<tr><td>%s</td><td>%s</td></tr>\n" % (x, sums[x]))
 381     ix.write("</table>\n")
 382
 383
 384     # primary logs
 385     ix.write("<h2>Primary logs</h2>\n")
 386     for log in glob.glob(os.path.join(dir, "valgrind-*-log*")):
 387         (dirname, basename) = os.path.split(log)
 388         shutil.copy(log, os.path.join(res, basename))
 389         ix.write("<a href=\"%s\">%s</a><br />\n" % (basename, basename))
 390
 391
 392     # massif graphs
 393     ix.write("<h2>Massif results</h2>\n")
 394     ix.write("<h3>Click graph to see details</h3>\n")
 395     for mp in glob.glob(os.path.join(dir, "massif.*.ps")):
 396         (dirname,basename) = os.path.split(mp)
 397         (base,ext) = os.path.splitext(basename)
 398         png = base + ".png"
 399         html = base + ".html"
 400         subprocess.call(["convert", "-rotate", "270", mp, os.path.join(res, png)])
 401         shutil.copy(os.path.join(dir, html), os.path.join(res, html))
 402         ix.write("<a href=\"%s\"><img src=\"%s\" /></a><br />\n" % (html, png))
 403
 404     # run movies
 405     ix.write("<h2>Movies</h2>\n")
 406     for movie in ["memcheck", "massif", "callgrind"]:
 407         for ext in [".html", ".swf"]:
 408             base = movie + ext
 409             if os.path.exists(os.path.join(dir, base)):
 410                 shutil.copy(os.path.join(dir, base), os.path.join(res, base))
 411         if os.path.exists(os.path.join(res, movie + ".html")):
 412             ix.write("<a href=\"%s\">%s movie</a><br />\n" % (movie + ".html", movie))
 413
 414     # callgrind profile
 415     ix.write("<h2>Callgrind profiles</h2>\n")
 416     for cg in glob.glob(os.path.join(dir, "callgrind.out.*")):
 417         (dir, base) = os.path.split(cg)
 418         shutil.copy(cg, os.path.join(res, base))
 419         ix.write("<a href=\"%s\">%s</a><br />\n" % (base, base))
 420
 421     ix.write("</body>\n</html>\n")
 422     ix.close()
 423
 424     for i in glob.glob(os.path.join(res, "*")):
 425         os.chmod(i, 0644)
 426     os.chmod(res, 0755)
 427
 428     subprocess.call(["tar", "czvf", tar, res])
 429     os.chmod(tar, 0644)
 430
 431
 432 def log_result_summaries(tmpdir):
 433
 434     pats = {
 435         "IR"      :  re.compile("==\d+==\s+I\s+refs:\s+([0-9,]+)"),
 436         "ALLOC"      :  re.compile("==\d+==\s+malloc/free:\s+[0-9,]+\s+allocs,\s+[0-9,]+\s+frees,\s+([0-9,]+)\s+bytes allocated."),
 437         "LEAKED"  :  re.compile("==\d+==\s+(?:definitely|indirectly)\s+lost:\s+([0-9,]+)\s+bytes"),
 438         "DUBIOUS" :  re.compile("==\d+==\s+possibly\s+lost:\s+([0-9,]+)\s+bytes"),
 439         "LIVE"    :  re.compile("==\d+==\s+still\s+reachable:\s+([0-9,]+)\s+bytes"),
 440         }
 441
 442     sums = {}
 443
 444     for fname in glob.glob("%s/valgrind-*-log*" % tmpdir):
 445         f = open(fname)
 446
 447         for line in f.xreadlines():
 448             for pat in pats:
 449                 rx = pats[pat]
 450                 match = rx.search(line)
 451                 if match:
 452                     val = int(match.group(1).replace(",", ""))
 453                     if pat in sums:
 454                         val = val + sums[pat]
 455                     sums[pat] = val
 456         f.close()
 457
 458     for pat in sums:
 459         logging.info("__COUNT_%s!%d" % (pat, sums[pat]))
 460
 461     archive_dir(tmpdir, sums)
 462
 463
 464
 465 ########################################################################
 466 # main
 467 ########################################################################
 468
 469 if len(sys.argv) != 2:
 470     print("usage: %s <firefox-bin build dir>" % sys.argv[0])
 471     sys.exit(1)
 472
 473 logging.basicConfig(level=logging.INFO,
 474                     format='%(asctime)s %(levelname)s %(message)s')
 475
 476 logging.info("running as %s" % getpass.getuser())
 477
 478 logging.info("killing any residual processes in this group")
 479 subprocess.call(["killall", "-q", "-9", "memcheck", "callgrind", "massif", "valgrind", "firefox-bin"])
 480 time.sleep(3)
 481
 482 dirs=glob.glob(os.path.join(homedir, batchprefix + "*"))
 483 dirs.sort()
 484 if len(dirs) > 5:
 485     for olddir in dirs[:-5]:
 486         logging.info("cleaning up old directory %s" % olddir)
 487         shutil.rmtree(olddir)
 488
 489
 490 timestr = datetime.datetime.now().isoformat().replace(":", "-")
 491 tmpdir = tempfile.mkdtemp(prefix=(batchprefix + timestr + "-"), dir=homedir)
 492
 493 logging.info("started batch run in dir " + tmpdir)
 494
 495 os.chdir(tmpdir)
 496 ffdir = sys.argv[1]
 497
 498 write_vgopts(tmpdir, vgopts)
 499
 500 xvnc_proc = None
 501 runner = None
 502 recorder = None
 503
 504 ######################################################
 505 # note: runit is supervising a single Xvnc on disp 25
 506 # there is no need to run one here as well
 507 ######################################################
 508 # xvnc_proc = start_xvnc(disp, tmpdir)
 509 ######################################################
 510
 511 try:
 512
 513     runner = Firefox_runner(batchprefix, homedir, ffdir, timestr, tmpdir, disp)
 514
 515     wait_collecting_memory_stats(runner.run_normal(url))
 516     runner.clear_cache()
 517
 518 #    for variant in ["memcheck", "massif", "callgrind"]:
 519 #        recorder = start_xvnc_recorder(vnc2swfpath, disp, variant, tmpdir)
 520 #        runner.run_valgrind(variant, url).wait()
 521 #        runner.clear_cache()
 522 #        complete_xvnc_recording(vnc2swfpath, recorder, variant, tmpdir)
 523
 524     log_result_summaries(tmpdir)
 525     logging.info("valgrind-firefox processes complete")
 526
 527 finally:
 528     if recorder and recorder.poll() == None:
 529         logging.info("killing recorder process %d" % recorder.pid)
 530     os.kill(recorder.pid, signal.SIGKILL)
 531     if xvnc_proc and xvnc_proc.poll() == None:
 532         logging.info("killing Xvnc process %d" % xvnc_proc.pid)
 533         os.kill(xvnc_proc.pid, signal.SIGKILL)
 534
 535 logging.info("batch run in dir %s complete" % tmpdir)