technical/unit-tests.html

   1 <?xml version="1.0" encoding="UTF-8"?>
   2 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
   3     "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
   4 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
   5 <head>
   6 <meta http-equiv="Content-Type" content="application/xhtml+xml; charset=UTF-8" />
   7 <meta name="generator" content="AsciiDoc 10.2.0" />
   8 <title>Unit Testing</title>
   9 <style type="text/css">
  10 /* Shared CSS for AsciiDoc xhtml11 and html5 backends */
  11
  12 /* Default font. */
  13 body {
  14   font-family: Georgia,serif;
  15 }
  16
  17 /* Title font. */
  18 h1, h2, h3, h4, h5, h6,
  19 div.title, caption.title,
  20 thead, p.table.header,
  21 #toctitle,
  22 #author, #revnumber, #revdate, #revremark,
  23 #footer {
  24   font-family: Arial,Helvetica,sans-serif;
  25 }
  26
  27 body {
  28   margin: 1em 5% 1em 5%;
  29 }
  30
  31 a {
  32   color: blue;
  33   text-decoration: underline;
  34 }
  35 a:visited {
  36   color: fuchsia;
  37 }
  38
  39 em {
  40   font-style: italic;
  41   color: navy;
  42 }
  43
  44 strong {
  45   font-weight: bold;
  46   color: #083194;
  47 }
  48
  49 h1, h2, h3, h4, h5, h6 {
  50   color: #527bbd;
  51   margin-top: 1.2em;
  52   margin-bottom: 0.5em;
  53   line-height: 1.3;
  54 }
  55
  56 h1, h2, h3 {
  57   border-bottom: 2px solid silver;
  58 }
  59 h2 {
  60   padding-top: 0.5em;
  61 }
  62 h3 {
  63   float: left;
  64 }
  65 h3 + * {
  66   clear: left;
  67 }
  68 h5 {
  69   font-size: 1.0em;
  70 }
  71
  72 div.sectionbody {
  73   margin-left: 0;
  74 }
  75
  76 hr {
  77   border: 1px solid silver;
  78 }
  79
  80 p {
  81   margin-top: 0.5em;
  82   margin-bottom: 0.5em;
  83 }
  84
  85 ul, ol, li > p {
  86   margin-top: 0;
  87 }
  88 ul > li     { color: #aaa; }
  89 ul > li > * { color: black; }
  90
  91 .monospaced, code, pre {
  92   font-family: "Courier New", Courier, monospace;
  93   font-size: inherit;
  94   color: navy;
  95   padding: 0;
  96   margin: 0;
  97 }
  98 pre {
  99   white-space: pre-wrap;
 100 }
 101
 102 #author {
 103   color: #527bbd;
 104   font-weight: bold;
 105   font-size: 1.1em;
 106 }
 107 #email {
 108 }
 109 #revnumber, #revdate, #revremark {
 110 }
 111
 112 #footer {
 113   font-size: small;
 114   border-top: 2px solid silver;
 115   padding-top: 0.5em;
 116   margin-top: 4.0em;
 117 }
 118 #footer-text {
 119   float: left;
 120   padding-bottom: 0.5em;
 121 }
 122 #footer-badges {
 123   float: right;
 124   padding-bottom: 0.5em;
 125 }
 126
 127 #preamble {
 128   margin-top: 1.5em;
 129   margin-bottom: 1.5em;
 130 }
 131 div.imageblock, div.exampleblock, div.verseblock,
 132 div.quoteblock, div.literalblock, div.listingblock, div.sidebarblock,
 133 div.admonitionblock {
 134   margin-top: 1.0em;
 135   margin-bottom: 1.5em;
 136 }
 137 div.admonitionblock {
 138   margin-top: 2.0em;
 139   margin-bottom: 2.0em;
 140   margin-right: 10%;
 141   color: #606060;
 142 }
 143
 144 div.content { /* Block element content. */
 145   padding: 0;
 146 }
 147
 148 /* Block element titles. */
 149 div.title, caption.title {
 150   color: #527bbd;
 151   font-weight: bold;
 152   text-align: left;
 153   margin-top: 1.0em;
 154   margin-bottom: 0.5em;
 155 }
 156 div.title + * {
 157   margin-top: 0;
 158 }
 159
 160 td div.title:first-child {
 161   margin-top: 0.0em;
 162 }
 163 div.content div.title:first-child {
 164   margin-top: 0.0em;
 165 }
 166 div.content + div.title {
 167   margin-top: 0.0em;
 168 }
 169
 170 div.sidebarblock > div.content {
 171   background: #ffffee;
 172   border: 1px solid #dddddd;
 173   border-left: 4px solid #f0f0f0;
 174   padding: 0.5em;
 175 }
 176
 177 div.listingblock > div.content {
 178   border: 1px solid #dddddd;
 179   border-left: 5px solid #f0f0f0;
 180   background: #f8f8f8;
 181   padding: 0.5em;
 182 }
 183
 184 div.quoteblock, div.verseblock {
 185   padding-left: 1.0em;
 186   margin-left: 1.0em;
 187   margin-right: 10%;
 188   border-left: 5px solid #f0f0f0;
 189   color: #888;
 190 }
 191
 192 div.quoteblock > div.attribution {
 193   padding-top: 0.5em;
 194   text-align: right;
 195 }
 196
 197 div.verseblock > pre.content {
 198   font-family: inherit;
 199   font-size: inherit;
 200 }
 201 div.verseblock > div.attribution {
 202   padding-top: 0.75em;
 203   text-align: left;
 204 }
 205 /* DEPRECATED: Pre version 8.2.7 verse style literal block. */
 206 div.verseblock + div.attribution {
 207   text-align: left;
 208 }
 209
 210 div.admonitionblock .icon {
 211   vertical-align: top;
 212   font-size: 1.1em;
 213   font-weight: bold;
 214   text-decoration: underline;
 215   color: #527bbd;
 216   padding-right: 0.5em;
 217 }
 218 div.admonitionblock td.content {
 219   padding-left: 0.5em;
 220   border-left: 3px solid #dddddd;
 221 }
 222
 223 div.exampleblock > div.content {
 224   border-left: 3px solid #dddddd;
 225   padding-left: 0.5em;
 226 }
 227
 228 div.imageblock div.content { padding-left: 0; }
 229 span.image img { border-style: none; vertical-align: text-bottom; }
 230 a.image:visited { color: white; }
 231
 232 dl {
 233   margin-top: 0.8em;
 234   margin-bottom: 0.8em;
 235 }
 236 dt {
 237   margin-top: 0.5em;
 238   margin-bottom: 0;
 239   font-style: normal;
 240   color: navy;
 241 }
 242 dd > *:first-child {
 243   margin-top: 0.1em;
 244 }
 245
 246 ul, ol {
 247     list-style-position: outside;
 248 }
 249 ol.arabic {
 250   list-style-type: decimal;
 251 }
 252 ol.loweralpha {
 253   list-style-type: lower-alpha;
 254 }
 255 ol.upperalpha {
 256   list-style-type: upper-alpha;
 257 }
 258 ol.lowerroman {
 259   list-style-type: lower-roman;
 260 }
 261 ol.upperroman {
 262   list-style-type: upper-roman;
 263 }
 264
 265 div.compact ul, div.compact ol,
 266 div.compact p, div.compact p,
 267 div.compact div, div.compact div {
 268   margin-top: 0.1em;
 269   margin-bottom: 0.1em;
 270 }
 271
 272 tfoot {
 273   font-weight: bold;
 274 }
 275 td > div.verse {
 276   white-space: pre;
 277 }
 278
 279 div.hdlist {
 280   margin-top: 0.8em;
 281   margin-bottom: 0.8em;
 282 }
 283 div.hdlist tr {
 284   padding-bottom: 15px;
 285 }
 286 dt.hdlist1.strong, td.hdlist1.strong {
 287   font-weight: bold;
 288 }
 289 td.hdlist1 {
 290   vertical-align: top;
 291   font-style: normal;
 292   padding-right: 0.8em;
 293   color: navy;
 294 }
 295 td.hdlist2 {
 296   vertical-align: top;
 297 }
 298 div.hdlist.compact tr {
 299   margin: 0;
 300   padding-bottom: 0;
 301 }
 302
 303 .comment {
 304   background: yellow;
 305 }
 306
 307 .footnote, .footnoteref {
 308   font-size: 0.8em;
 309 }
 310
 311 span.footnote, span.footnoteref {
 312   vertical-align: super;
 313 }
 314
 315 #footnotes {
 316   margin: 20px 0 20px 0;
 317   padding: 7px 0 0 0;
 318 }
 319
 320 #footnotes div.footnote {
 321   margin: 0 0 5px 0;
 322 }
 323
 324 #footnotes hr {
 325   border: none;
 326   border-top: 1px solid silver;
 327   height: 1px;
 328   text-align: left;
 329   margin-left: 0;
 330   width: 20%;
 331   min-width: 100px;
 332 }
 333
 334 div.colist td {
 335   padding-right: 0.5em;
 336   padding-bottom: 0.3em;
 337   vertical-align: top;
 338 }
 339 div.colist td img {
 340   margin-top: 0.3em;
 341 }
 342
 343 @media print {
 344   #footer-badges { display: none; }
 345 }
 346
 347 #toc {
 348   margin-bottom: 2.5em;
 349 }
 350
 351 #toctitle {
 352   color: #527bbd;
 353   font-size: 1.1em;
 354   font-weight: bold;
 355   margin-top: 1.0em;
 356   margin-bottom: 0.1em;
 357 }
 358
 359 div.toclevel0, div.toclevel1, div.toclevel2, div.toclevel3, div.toclevel4 {
 360   margin-top: 0;
 361   margin-bottom: 0;
 362 }
 363 div.toclevel2 {
 364   margin-left: 2em;
 365   font-size: 0.9em;
 366 }
 367 div.toclevel3 {
 368   margin-left: 4em;
 369   font-size: 0.9em;
 370 }
 371 div.toclevel4 {
 372   margin-left: 6em;
 373   font-size: 0.9em;
 374 }
 375
 376 span.aqua { color: aqua; }
 377 span.black { color: black; }
 378 span.blue { color: blue; }
 379 span.fuchsia { color: fuchsia; }
 380 span.gray { color: gray; }
 381 span.green { color: green; }
 382 span.lime { color: lime; }
 383 span.maroon { color: maroon; }
 384 span.navy { color: navy; }
 385 span.olive { color: olive; }
 386 span.purple { color: purple; }
 387 span.red { color: red; }
 388 span.silver { color: silver; }
 389 span.teal { color: teal; }
 390 span.white { color: white; }
 391 span.yellow { color: yellow; }
 392
 393 span.aqua-background { background: aqua; }
 394 span.black-background { background: black; }
 395 span.blue-background { background: blue; }
 396 span.fuchsia-background { background: fuchsia; }
 397 span.gray-background { background: gray; }
 398 span.green-background { background: green; }
 399 span.lime-background { background: lime; }
 400 span.maroon-background { background: maroon; }
 401 span.navy-background { background: navy; }
 402 span.olive-background { background: olive; }
 403 span.purple-background { background: purple; }
 404 span.red-background { background: red; }
 405 span.silver-background { background: silver; }
 406 span.teal-background { background: teal; }
 407 span.white-background { background: white; }
 408 span.yellow-background { background: yellow; }
 409
 410 span.big { font-size: 2em; }
 411 span.small { font-size: 0.6em; }
 412
 413 span.underline { text-decoration: underline; }
 414 span.overline { text-decoration: overline; }
 415 span.line-through { text-decoration: line-through; }
 416
 417 div.unbreakable { page-break-inside: avoid; }
 418
 419
 420 /*
 421  * xhtml11 specific
 422  *
 423  * */
 424
 425 div.tableblock {
 426   margin-top: 1.0em;
 427   margin-bottom: 1.5em;
 428 }
 429 div.tableblock > table {
 430   border: 3px solid #527bbd;
 431 }
 432 thead, p.table.header {
 433   font-weight: bold;
 434   color: #527bbd;
 435 }
 436 p.table {
 437   margin-top: 0;
 438 }
 439 /* Because the table frame attribute is overridden by CSS in most browsers. */
 440 div.tableblock > table[frame="void"] {
 441   border-style: none;
 442 }
 443 div.tableblock > table[frame="hsides"] {
 444   border-left-style: none;
 445   border-right-style: none;
 446 }
 447 div.tableblock > table[frame="vsides"] {
 448   border-top-style: none;
 449   border-bottom-style: none;
 450 }
 451
 452
 453 /*
 454  * html5 specific
 455  *
 456  * */
 457
 458 table.tableblock {
 459   margin-top: 1.0em;
 460   margin-bottom: 1.5em;
 461 }
 462 thead, p.tableblock.header {
 463   font-weight: bold;
 464   color: #527bbd;
 465 }
 466 p.tableblock {
 467   margin-top: 0;
 468 }
 469 table.tableblock {
 470   border-width: 3px;
 471   border-spacing: 0px;
 472   border-style: solid;
 473   border-color: #527bbd;
 474   border-collapse: collapse;
 475 }
 476 th.tableblock, td.tableblock {
 477   border-width: 1px;
 478   padding: 4px;
 479   border-style: solid;
 480   border-color: #527bbd;
 481 }
 482
 483 table.tableblock.frame-topbot {
 484   border-left-style: hidden;
 485   border-right-style: hidden;
 486 }
 487 table.tableblock.frame-sides {
 488   border-top-style: hidden;
 489   border-bottom-style: hidden;
 490 }
 491 table.tableblock.frame-none {
 492   border-style: hidden;
 493 }
 494
 495 th.tableblock.halign-left, td.tableblock.halign-left {
 496   text-align: left;
 497 }
 498 th.tableblock.halign-center, td.tableblock.halign-center {
 499   text-align: center;
 500 }
 501 th.tableblock.halign-right, td.tableblock.halign-right {
 502   text-align: right;
 503 }
 504
 505 th.tableblock.valign-top, td.tableblock.valign-top {
 506   vertical-align: top;
 507 }
 508 th.tableblock.valign-middle, td.tableblock.valign-middle {
 509   vertical-align: middle;
 510 }
 511 th.tableblock.valign-bottom, td.tableblock.valign-bottom {
 512   vertical-align: bottom;
 513 }
 514
 515
 516 /*
 517  * manpage specific
 518  *
 519  * */
 520
 521 body.manpage h1 {
 522   padding-top: 0.5em;
 523   padding-bottom: 0.5em;
 524   border-top: 2px solid silver;
 525   border-bottom: 2px solid silver;
 526 }
 527 body.manpage h2 {
 528   border-style: none;
 529 }
 530 body.manpage div.sectionbody {
 531   margin-left: 3em;
 532 }
 533
 534 @media print {
 535   body.manpage div#toc { display: none; }
 536 }
 537
 538
 539 </style>
 540 <script type="text/javascript">
 541 /*<![CDATA[*/
 542 var asciidoc = {  // Namespace.
 543
 544 /////////////////////////////////////////////////////////////////////
 545 // Table Of Contents generator
 546 /////////////////////////////////////////////////////////////////////
 547
 548 /* Author: Mihai Bazon, September 2002
 549  * http://students.infoiasi.ro/~mishoo
 550  *
 551  * Table Of Content generator
 552  * Version: 0.4
 553  *
 554  * Feel free to use this script under the terms of the GNU General Public
 555  * License, as long as you do not remove or alter this notice.
 556  */
 557
 558  /* modified by Troy D. Hanson, September 2006. License: GPL */
 559  /* modified by Stuart Rackham, 2006, 2009. License: GPL */
 560
 561 // toclevels = 1..4.
 562 toc: function (toclevels) {
 563
 564   function getText(el) {
 565     var text = "";
 566     for (var i = el.firstChild; i != null; i = i.nextSibling) {
 567       if (i.nodeType == 3 /* Node.TEXT_NODE */) // IE doesn't speak constants.
 568         text += i.data;
 569       else if (i.firstChild != null)
 570         text += getText(i);
 571     }
 572     return text;
 573   }
 574
 575   function TocEntry(el, text, toclevel) {
 576     this.element = el;
 577     this.text = text;
 578     this.toclevel = toclevel;
 579   }
 580
 581   function tocEntries(el, toclevels) {
 582     var result = new Array;
 583     var re = new RegExp('[hH]([1-'+(toclevels+1)+'])');
 584     // Function that scans the DOM tree for header elements (the DOM2
 585     // nodeIterator API would be a better technique but not supported by all
 586     // browsers).
 587     var iterate = function (el) {
 588       for (var i = el.firstChild; i != null; i = i.nextSibling) {
 589         if (i.nodeType == 1 /* Node.ELEMENT_NODE */) {
 590           var mo = re.exec(i.tagName);
 591           if (mo && (i.getAttribute("class") || i.getAttribute("className")) != "float") {
 592             result[result.length] = new TocEntry(i, getText(i), mo[1]-1);
 593           }
 594           iterate(i);
 595         }
 596       }
 597     }
 598     iterate(el);
 599     return result;
 600   }
 601
 602   var toc = document.getElementById("toc");
 603   if (!toc) {
 604     return;
 605   }
 606
 607   // Delete existing TOC entries in case we're reloading the TOC.
 608   var tocEntriesToRemove = [];
 609   var i;
 610   for (i = 0; i < toc.childNodes.length; i++) {
 611     var entry = toc.childNodes[i];
 612     if (entry.nodeName.toLowerCase() == 'div'
 613      && entry.getAttribute("class")
 614      && entry.getAttribute("class").match(/^toclevel/))
 615       tocEntriesToRemove.push(entry);
 616   }
 617   for (i = 0; i < tocEntriesToRemove.length; i++) {
 618     toc.removeChild(tocEntriesToRemove[i]);
 619   }
 620
 621   // Rebuild TOC entries.
 622   var entries = tocEntries(document.getElementById("content"), toclevels);
 623   for (var i = 0; i < entries.length; ++i) {
 624     var entry = entries[i];
 625     if (entry.element.id == "")
 626       entry.element.id = "_toc_" + i;
 627     var a = document.createElement("a");
 628     a.href = "#" + entry.element.id;
 629     a.appendChild(document.createTextNode(entry.text));
 630     var div = document.createElement("div");
 631     div.appendChild(a);
 632     div.className = "toclevel" + entry.toclevel;
 633     toc.appendChild(div);
 634   }
 635   if (entries.length == 0)
 636     toc.parentNode.removeChild(toc);
 637 },
 638
 639
 640 /////////////////////////////////////////////////////////////////////
 641 // Footnotes generator
 642 /////////////////////////////////////////////////////////////////////
 643
 644 /* Based on footnote generation code from:
 645  * http://www.brandspankingnew.net/archive/2005/07/format_footnote.html
 646  */
 647
 648 footnotes: function () {
 649   // Delete existing footnote entries in case we're reloading the footnodes.
 650   var i;
 651   var noteholder = document.getElementById("footnotes");
 652   if (!noteholder) {
 653     return;
 654   }
 655   var entriesToRemove = [];
 656   for (i = 0; i < noteholder.childNodes.length; i++) {
 657     var entry = noteholder.childNodes[i];
 658     if (entry.nodeName.toLowerCase() == 'div' && entry.getAttribute("class") == "footnote")
 659       entriesToRemove.push(entry);
 660   }
 661   for (i = 0; i < entriesToRemove.length; i++) {
 662     noteholder.removeChild(entriesToRemove[i]);
 663   }
 664
 665   // Rebuild footnote entries.
 666   var cont = document.getElementById("content");
 667   var spans = cont.getElementsByTagName("span");
 668   var refs = {};
 669   var n = 0;
 670   for (i=0; i<spans.length; i++) {
 671     if (spans[i].className == "footnote") {
 672       n++;
 673       var note = spans[i].getAttribute("data-note");
 674       if (!note) {
 675         // Use [\s\S] in place of . so multi-line matches work.
 676         // Because JavaScript has no s (dotall) regex flag.
 677         note = spans[i].innerHTML.match(/\s*\[([\s\S]*)]\s*/)[1];
 678         spans[i].innerHTML =
 679           "[<a id='_footnoteref_" + n + "' href='#_footnote_" + n +
 680           "' title='View footnote' class='footnote'>" + n + "</a>]";
 681         spans[i].setAttribute("data-note", note);
 682       }
 683       noteholder.innerHTML +=
 684         "<div class='footnote' id='_footnote_" + n + "'>" +
 685         "<a href='#_footnoteref_" + n + "' title='Return to text'>" +
 686         n + "</a>. " + note + "</div>";
 687       var id =spans[i].getAttribute("id");
 688       if (id != null) refs["#"+id] = n;
 689     }
 690   }
 691   if (n == 0)
 692     noteholder.parentNode.removeChild(noteholder);
 693   else {
 694     // Process footnoterefs.
 695     for (i=0; i<spans.length; i++) {
 696       if (spans[i].className == "footnoteref") {
 697         var href = spans[i].getElementsByTagName("a")[0].getAttribute("href");
 698         href = href.match(/#.*/)[0];  // Because IE return full URL.
 699         n = refs[href];
 700         spans[i].innerHTML =
 701           "[<a href='#_footnote_" + n +
 702           "' title='View footnote' class='footnote'>" + n + "</a>]";
 703       }
 704     }
 705   }
 706 },
 707
 708 install: function(toclevels) {
 709   var timerId;
 710
 711   function reinstall() {
 712     asciidoc.footnotes();
 713     if (toclevels) {
 714       asciidoc.toc(toclevels);
 715     }
 716   }
 717
 718   function reinstallAndRemoveTimer() {
 719     clearInterval(timerId);
 720     reinstall();
 721   }
 722
 723   timerId = setInterval(reinstall, 500);
 724   if (document.addEventListener)
 725     document.addEventListener("DOMContentLoaded", reinstallAndRemoveTimer, false);
 726   else
 727     window.onload = reinstallAndRemoveTimer;
 728 }
 729
 730 }
 731 asciidoc.install();
 732 /*]]>*/
 733 </script>
 734 </head>
 735 <body class="article">
 736 <div id="header">
 737 <h1>Unit Testing</h1>
 738 <span id="revdate">2024-01-26</span>
 739 </div>
 740 <div id="content">
 741 <div id="preamble">
 742 <div class="sectionbody">
 743 <div class="paragraph"><p>In our current testing environment, we spend a significant amount of effort
 744 crafting end-to-end tests for error conditions that could easily be captured by
 745 unit tests (or we simply forgo some hard-to-setup and rare error conditions).
 746 Unit tests additionally provide stability to the codebase and can simplify
 747 debugging through isolation. Writing unit tests in pure C, rather than with our
 748 current shell/test-tool helper setup, simplifies test setup, simplifies passing
 749 data around (no shell-isms required), and reduces testing runtime by not
 750 spawning a separate process for every test invocation.</p></div>
 751 <div class="paragraph"><p>We believe that a large body of unit tests, living alongside the existing test
 752 suite, will improve code quality for the Git project.</p></div>
 753 </div>
 754 </div>
 755 <div class="sect1">
 756 <h2 id="_definitions">Definitions</h2>
 757 <div class="sectionbody">
 758 <div class="paragraph"><p>For the purposes of this document, we&#8217;ll use <strong>test framework</strong> to refer to
 759 projects that support writing test cases and running tests within the context
 760 of a single executable. <strong>Test harness</strong> will refer to projects that manage
 761 running multiple executables (each of which may contain multiple test cases) and
 762 aggregating their results.</p></div>
 763 <div class="paragraph"><p>In reality, these terms are not strictly defined, and many of the projects
 764 discussed below contain features from both categories.</p></div>
 765 <div class="paragraph"><p>For now, we will evaluate projects solely on their framework features. Since we
 766 are relying on having TAP output (see below), we can assume that any framework
 767 can be made to work with a harness that we can choose later.</p></div>
 768 </div>
 769 </div>
 770 <div class="sect1">
 771 <h2 id="_summary">Summary</h2>
 772 <div class="sectionbody">
 773 <div class="paragraph"><p>We believe the best way forward is to implement a custom TAP framework for the
 774 Git project. We use a version of the framework originally proposed in
 775 <a href="https://lore.kernel.org/git/c902a166-98ce-afba-93f2-ea6027557176@gmail.com/">1</a>.</p></div>
 776 <div class="paragraph"><p>See the <a href="#framework-selection">Framework Selection</a> section below for the
 777 rationale behind this decision.</p></div>
 778 </div>
 779 </div>
 780 <div class="sect1">
 781 <h2 id="_choosing_a_test_harness">Choosing a test harness</h2>
 782 <div class="sectionbody">
 783 <div class="paragraph"><p>During upstream discussion, it was occasionally noted that <code>prove</code> provides many
 784 convenient features, such as scheduling slower tests first, or re-running
 785 previously failed tests.</p></div>
 786 <div class="paragraph"><p>While we already support the use of <code>prove</code> as a test harness for the shell
 787 tests, it is not strictly required. The t/Makefile allows running shell tests
 788 directly (though with interleaved output if parallelism is enabled). Git
 789 developers who wish to use <code>prove</code> as a more advanced harness can do so by
 790 setting DEFAULT_TEST_TARGET=prove in their config.mak.</p></div>
 791 <div class="paragraph"><p>We will follow a similar approach for unit tests: by default the test
 792 executables will be run directly from the t/Makefile, but <code>prove</code> can be
 793 configured with DEFAULT_UNIT_TEST_TARGET=prove.</p></div>
 794 </div>
 795 </div>
 796 <div class="sect1">
 797 <h2 id="framework-selection">Framework selection</h2>
 798 <div class="sectionbody">
 799 <div class="paragraph"><p>There are a variety of features we can use to rank the candidate frameworks, and
 800 those features have different priorities:</p></div>
 801 <div class="ulist"><ul>
 802 <li>
 803 <p>
 804 Critical features: we probably won&#8217;t consider a framework without these
 805 </p>
 806 <div class="ulist"><ul>
 807 <li>
 808 <p>
 809 Can we legally / easily use the project?
 810 </p>
 811 <div class="ulist"><ul>
 812 <li>
 813 <p>
 814 <a href="#license">License</a>
 815 </p>
 816 </li>
 817 <li>
 818 <p>
 819 <a href="#vendorable-or-ubiquitous">Vendorable or ubiquitous</a>
 820 </p>
 821 </li>
 822 <li>
 823 <p>
 824 <a href="#maintainable-extensible">Maintainable / extensible</a>
 825 </p>
 826 </li>
 827 <li>
 828 <p>
 829 <a href="#major-platform-support">Major platform support</a>
 830 </p>
 831 </li>
 832 </ul></div>
 833 </li>
 834 <li>
 835 <p>
 836 Does the project support our bare-minimum needs?
 837 </p>
 838 <div class="ulist"><ul>
 839 <li>
 840 <p>
 841 <a href="#tap-support">TAP support</a>
 842 </p>
 843 </li>
 844 <li>
 845 <p>
 846 <a href="#diagnostic-output">Diagnostic output</a>
 847 </p>
 848 </li>
 849 <li>
 850 <p>
 851 <a href="#runtime-skippable-tests">Runtime-skippable tests</a>
 852 </p>
 853 </li>
 854 </ul></div>
 855 </li>
 856 </ul></div>
 857 </li>
 858 <li>
 859 <p>
 860 Nice-to-have features:
 861 </p>
 862 <div class="ulist"><ul>
 863 <li>
 864 <p>
 865 <a href="#parallel-execution">Parallel execution</a>
 866 </p>
 867 </li>
 868 <li>
 869 <p>
 870 <a href="#mock-support">Mock support</a>
 871 </p>
 872 </li>
 873 <li>
 874 <p>
 875 <a href="#signal-error-handling">Signal &amp; error-handling</a>
 876 </p>
 877 </li>
 878 </ul></div>
 879 </li>
 880 <li>
 881 <p>
 882 Tie-breaker stats
 883 </p>
 884 <div class="ulist"><ul>
 885 <li>
 886 <p>
 887 <a href="#project-kloc">Project KLOC</a>
 888 </p>
 889 </li>
 890 <li>
 891 <p>
 892 <a href="#adoption">Adoption</a>
 893 </p>
 894 </li>
 895 </ul></div>
 896 </li>
 897 </ul></div>
 898 <div class="sect2">
 899 <h3 id="license">License</h3>
 900 <div class="paragraph"><p>We must be able to legally use the framework in connection with Git. As Git is
 901 licensed only under GPLv2, we must eliminate any LGPLv3, GPLv3, or Apache 2.0
 902 projects.</p></div>
 903 </div>
 904 <div class="sect2">
 905 <h3 id="vendorable-or-ubiquitous">Vendorable or ubiquitous</h3>
 906 <div class="paragraph"><p>We want to avoid forcing Git developers to install new tools just to run unit
 907 tests. Any prospective frameworks and harnesses must either be vendorable
 908 (meaning, we can copy their source directly into Git&#8217;s repository), or so
 909 ubiquitous that it is reasonable to expect that most developers will have the
 910 tools installed already.</p></div>
 911 </div>
 912 <div class="sect2">
 913 <h3 id="maintainable-extensible">Maintainable / extensible</h3>
 914 <div class="paragraph"><p>It is unlikely that any pre-existing project perfectly fits our needs, so any
 915 project we select will need to be actively maintained and open to accepting
 916 changes. Alternatively, assuming we are vendoring the source into our repo, it
 917 must be simple enough that Git developers can feel comfortable making changes as
 918 needed to our version.</p></div>
 919 <div class="paragraph"><p>In the comparison table below, "True" means that the framework seems to have
 920 active developers, that it is simple enough that Git developers can make changes
 921 to it, and that the project seems open to accepting external contributions (or
 922 that it is vendorable). "Partial" means that at least one of the above
 923 conditions holds.</p></div>
 924 </div>
 925 <div class="sect2">
 926 <h3 id="major-platform-support">Major platform support</h3>
 927 <div class="paragraph"><p>At a bare minimum, unit-testing must work on Linux, MacOS, and Windows.</p></div>
 928 <div class="paragraph"><p>In the comparison table below, "True" means that it works on all three major
 929 platforms with no issues. "Partial" means that there may be annoyances on one or
 930 more platforms, but it is still usable in principle.</p></div>
 931 </div>
 932 <div class="sect2">
 933 <h3 id="tap-support">TAP support</h3>
 934 <div class="paragraph"><p>The <a href="https://testanything.org/">Test Anything Protocol</a> is a text-based interface
 935 that allows tests to communicate with a test harness. It is already used by
 936 Git&#8217;s integration test suite. Supporting TAP output is a mandatory feature for
 937 any prospective test framework.</p></div>
 938 <div class="paragraph"><p>In the comparison table below, "True" means this is natively supported.
 939 "Partial" means TAP output must be generated by post-processing the native
 940 output.</p></div>
 941 <div class="paragraph"><p>Frameworks that do not have at least Partial support will not be evaluated
 942 further.</p></div>
 943 </div>
 944 <div class="sect2">
 945 <h3 id="diagnostic-output">Diagnostic output</h3>
 946 <div class="paragraph"><p>When a test case fails, the framework must generate enough diagnostic output to
 947 help developers find the appropriate test case in source code in order to debug
 948 the failure.</p></div>
 949 </div>
 950 <div class="sect2">
 951 <h3 id="runtime-skippable-tests">Runtime-skippable tests</h3>
 952 <div class="paragraph"><p>Test authors may wish to skip certain test cases based on runtime circumstances,
 953 so the framework should support this.</p></div>
 954 </div>
 955 <div class="sect2">
 956 <h3 id="parallel-execution">Parallel execution</h3>
 957 <div class="paragraph"><p>Ideally, we will build up a significant collection of unit test cases, most
 958 likely split across multiple executables. It will be necessary to run these
 959 tests in parallel to enable fast develop-test-debug cycles.</p></div>
 960 <div class="paragraph"><p>In the comparison table below, "True" means that individual test cases within a
 961 single test executable can be run in parallel. We assume that executable-level
 962 parallelism can be handled by the test harness.</p></div>
 963 </div>
 964 <div class="sect2">
 965 <h3 id="mock-support">Mock support</h3>
 966 <div class="paragraph"><p>Unit test authors may wish to test code that interacts with objects that may be
 967 inconvenient to handle in a test (e.g. interacting with a network service).
 968 Mocking allows test authors to provide a fake implementation of these objects
 969 for more convenient tests.</p></div>
 970 </div>
 971 <div class="sect2">
 972 <h3 id="signal-error-handling">Signal &amp; error handling</h3>
 973 <div class="paragraph"><p>The test framework should fail gracefully when test cases are themselves buggy
 974 or when they are interrupted by signals during runtime.</p></div>
 975 </div>
 976 <div class="sect2">
 977 <h3 id="project-kloc">Project KLOC</h3>
 978 <div class="paragraph"><p>The size of the project, in thousands of lines of code as measured by
 979 <a href="https://dwheeler.com/sloccount/">sloccount</a> (rounded up to the next multiple of
 980 1,000). As a tie-breaker, we probably prefer a project with fewer LOC.</p></div>
 981 </div>
 982 <div class="sect2">
 983 <h3 id="adoption">Adoption</h3>
 984 <div class="paragraph"><p>As a tie-breaker, we prefer a more widely-used project. We use the number of
 985 GitHub / GitLab stars to estimate this.</p></div>
 986 </div>
 987 <div class="sect2">
 988 <h3 id="_comparison">Comparison</h3>
 989 <div class="tableblock">
 990 <table rules="all"
 991 width="33%"
 992 frame="border"
 993 cellspacing="0" cellpadding="4">
 994 <col width="4%" />
 995 <col width="4%" />
 996 <col width="4%" />
 997 <col width="4%" />
 998 <col width="4%" />
 999 <col width="4%" />
1000 <col width="4%" />
1001 <col width="4%" />
1002 <col width="4%" />
1003 <col width="4%" />
1004 <col width="4%" />
1005 <col width="4%" />
1006 <col width="4%" />
1007 <col width="4%" />
1008 <col width="4%" />
1009 <col width="4%" />
1010 <col width="4%" />
1011 <col width="4%" />
1012 <col width="4%" />
1013 <col width="4%" />
1014 <col width="4%" />
1015 <col width="4%" />
1016 <col width="4%" />
1017 <col width="4%" />
1018 <col width="4%" />
1019 <thead>
1020 <tr>
1021 <th align="left" valign="top">Framework</th>
1022 <th align="left" valign="top"><a href="#license">License</a></th>
1023 <th align="left" valign="top"><a href="#vendorable-or-ubiquitous">Vendorable or ubiquitous</a></th>
1024 <th align="left" valign="top"><a href="#maintainable-extensible">Maintainable / extensible</a></th>
1025 <th align="left" valign="top"><a href="#major-platform-support">Major platform support</a></th>
1026 <th align="left" valign="top"><a href="#tap-support">TAP support</a></th>
1027 <th align="left" valign="top"><a href="#diagnostic-output">Diagnostic output</a></th>
1028 <th align="left" valign="top"><a href="#runtime--skippable-tests">Runtime- skippable tests</a></th>
1029 <th align="left" valign="top"><a href="#parallel-execution">Parallel execution</a></th>
1030 <th align="left" valign="top"><a href="#mock-support">Mock support</a></th>
1031 <th align="left" valign="top"><a href="#signal-error-handling">Signal &amp; error handling</a></th>
1032 <th align="left" valign="top"><a href="#project-kloc">Project KLOC</a></th>
1033 <th align="left" valign="top"><a href="#adoption">Adoption</a></th>
1034 </tr>
1035 </thead>
1036 <tbody>
1037 <tr>
1038 <td align="left" valign="top"><p class="table"><a href="https://lore.kernel.org/git/c902a166-98ce-afba-93f2-ea6027557176@gmail.com/">Custom Git impl.</a></p></td>
1039 <td align="left" valign="top"><p class="table"><span class="lime-background">GPL v2</span></p></td>
1040 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1041 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1042 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1043 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1044 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1045 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1046 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1047 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1048 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1049 <td align="left" valign="top"><p class="table">1</p></td>
1050 <td align="left" valign="top"><p class="table">0</p></td>
1051 </tr>
1052 <tr>
1053 <td align="left" valign="top"><p class="table"><a href="https://github.com/silentbicycle/greatest">Greatest</a></p></td>
1054 <td align="left" valign="top"><p class="table"><span class="lime-background">ISC</span></p></td>
1055 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1056 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1057 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1058 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1059 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1060 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1061 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1062 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1063 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1064 <td align="left" valign="top"><p class="table">3</p></td>
1065 <td align="left" valign="top"><p class="table">1400</p></td>
1066 </tr>
1067 <tr>
1068 <td align="left" valign="top"><p class="table"><a href="https://github.com/Snaipe/Criterion">Criterion</a></p></td>
1069 <td align="left" valign="top"><p class="table"><span class="lime-background">MIT</span></p></td>
1070 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1071 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1072 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1073 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1074 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1075 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1076 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1077 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1078 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1079 <td align="left" valign="top"><p class="table">19</p></td>
1080 <td align="left" valign="top"><p class="table">1800</p></td>
1081 </tr>
1082 <tr>
1083 <td align="left" valign="top"><p class="table"><a href="https://github.com/rra/c-tap-harness/">C TAP</a></p></td>
1084 <td align="left" valign="top"><p class="table"><span class="lime-background">Expat</span></p></td>
1085 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1086 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1087 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1088 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1089 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1090 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1091 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1092 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1093 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1094 <td align="left" valign="top"><p class="table">4</p></td>
1095 <td align="left" valign="top"><p class="table">33</p></td>
1096 </tr>
1097 <tr>
1098 <td align="left" valign="top"><p class="table"><a href="https://libcheck.github.io/check/">Check</a></p></td>
1099 <td align="left" valign="top"><p class="table"><span class="lime-background">LGPL v2.1</span></p></td>
1100 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1101 <td align="left" valign="top"><p class="table"><span class="yellow-background">Partial</span></p></td>
1102 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1103 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1104 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1105 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1106 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1107 <td align="left" valign="top"><p class="table"><span class="red-background">False</span></p></td>
1108 <td align="left" valign="top"><p class="table"><span class="lime-background">True</span></p></td>
1109 <td align="left" valign="top"><p class="table">17</p></td>
1110 <td align="left" valign="top"><p class="table">973</p></td>
1111 </tr>
1112 </tbody>
1113 </table>
1114 </div>
1115 </div>
1116 <div class="sect2">
1117 <h3 id="_additional_framework_candidates">Additional framework candidates</h3>
1118 <div class="paragraph"><p>Several suggested frameworks have been eliminated from consideration:</p></div>
1119 <div class="ulist"><ul>
1120 <li>
1121 <p>
1122 Incompatible licenses:
1123 </p>
1124 <div class="ulist"><ul>
1125 <li>
1126 <p>
1127 <a href="https://github.com/zorgnax/libtap">libtap</a> (LGPL v3)
1128 </p>
1129 </li>
1130 <li>
1131 <p>
1132 <a href="https://cmocka.org/">cmocka</a> (Apache 2.0)
1133 </p>
1134 </li>
1135 </ul></div>
1136 </li>
1137 <li>
1138 <p>
1139 Missing source: <a href="https://www.kindahl.net/mytap/doc/index.html">MyTap</a>
1140 </p>
1141 </li>
1142 <li>
1143 <p>
1144 No TAP support:
1145 </p>
1146 <div class="ulist"><ul>
1147 <li>
1148 <p>
1149 <a href="https://nemequ.github.io/munit/">µnit</a>
1150 </p>
1151 </li>
1152 <li>
1153 <p>
1154 <a href="https://github.com/google/cmockery">cmockery</a>
1155 </p>
1156 </li>
1157 <li>
1158 <p>
1159 <a href="https://github.com/lpabon/cmockery2">cmockery2</a>
1160 </p>
1161 </li>
1162 <li>
1163 <p>
1164 <a href="https://github.com/ThrowTheSwitch/Unity">Unity</a>
1165 </p>
1166 </li>
1167 <li>
1168 <p>
1169 <a href="https://github.com/siu/minunit">minunit</a>
1170 </p>
1171 </li>
1172 <li>
1173 <p>
1174 <a href="https://cunit.sourceforge.net/">CUnit</a>
1175 </p>
1176 </li>
1177 </ul></div>
1178 </li>
1179 </ul></div>
1180 </div>
1181 </div>
1182 </div>
1183 <div class="sect1">
1184 <h2 id="_milestones">Milestones</h2>
1185 <div class="sectionbody">
1186 <div class="ulist"><ul>
1187 <li>
1188 <p>
1189 Add useful tests of library-like code
1190 </p>
1191 </li>
1192 <li>
1193 <p>
1194 Integrate with
1195   <a href="https://lore.kernel.org/git/20230502211454.1673000-1-calvinwan@google.com/">stdlib
1196   work</a>
1197 </p>
1198 </li>
1199 <li>
1200 <p>
1201 Run alongside regular <code>make test</code> target
1202 </p>
1203 </li>
1204 </ul></div>
1205 </div>
1206 </div>
1207 </div>
1208 <div id="footnotes"><hr /></div>
1209 <div id="footer">
1210 <div id="footer-text">
1211 Last updated
1212  2023-12-09 17:40:53 PST
1213 </div>
1214 </div>
1215 </body>
1216 </html>