python/mozbuild/mozbuild/test/test_preprocessor.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 import os
   6 import shutil
   7 import unittest
   8 from tempfile import mkdtemp
   9
  10 from mozunit import MockedOpen, main
  11 from six import StringIO
  12
  13 from mozbuild.preprocessor import Preprocessor
  14
  15
  16 class TestPreprocessor(unittest.TestCase):
  17     """
  18     Unit tests for the Context class
  19     """
  20
  21     def setUp(self):
  22         self.pp = Preprocessor()
  23         self.pp.out = StringIO()
  24
  25     def do_include_compare(self, content_lines, expected_lines):
  26         content = "%s" % "\n".join(content_lines)
  27         expected = "%s".rstrip() % "\n".join(expected_lines)
  28
  29         with MockedOpen({"dummy": content}):
  30             self.pp.do_include("dummy")
  31             self.assertEqual(self.pp.out.getvalue().rstrip("\n"), expected)
  32
  33     def do_include_pass(self, content_lines):
  34         self.do_include_compare(content_lines, ["PASS"])
  35
  36     def test_conditional_if_0(self):
  37         self.do_include_pass(
  38             [
  39                 "#if 0",
  40                 "FAIL",
  41                 "#else",
  42                 "PASS",
  43                 "#endif",
  44             ]
  45         )
  46
  47     def test_no_marker(self):
  48         lines = [
  49             "#if 0",
  50             "PASS",
  51             "#endif",
  52         ]
  53         self.pp.setMarker(None)
  54         self.do_include_compare(lines, lines)
  55
  56     def test_string_value(self):
  57         self.do_include_compare(
  58             [
  59                 "#define FOO STRING",
  60                 "#if FOO",
  61                 "string value is true",
  62                 "#else",
  63                 "string value is false",
  64                 "#endif",
  65             ],
  66             ["string value is false"],
  67         )
  68
  69     def test_number_value(self):
  70         self.do_include_compare(
  71             [
  72                 "#define FOO 1",
  73                 "#if FOO",
  74                 "number value is true",
  75                 "#else",
  76                 "number value is false",
  77                 "#endif",
  78             ],
  79             ["number value is true"],
  80         )
  81
  82     def test_conditional_if_0_elif_1(self):
  83         self.do_include_pass(
  84             [
  85                 "#if 0",
  86                 "#elif 1",
  87                 "PASS",
  88                 "#else",
  89                 "FAIL",
  90                 "#endif",
  91             ]
  92         )
  93
  94     def test_conditional_if_1(self):
  95         self.do_include_pass(
  96             [
  97                 "#if 1",
  98                 "PASS",
  99                 "#else",
 100                 "FAIL",
 101                 "#endif",
 102             ]
 103         )
 104
 105     def test_conditional_if_0_or_1(self):
 106         self.do_include_pass(
 107             [
 108                 "#if 0 || 1",
 109                 "PASS",
 110                 "#else",
 111                 "FAIL",
 112                 "#endif",
 113             ]
 114         )
 115
 116     def test_conditional_if_1_elif_1_else(self):
 117         self.do_include_pass(
 118             [
 119                 "#if 1",
 120                 "PASS",
 121                 "#elif 1",
 122                 "FAIL",
 123                 "#else",
 124                 "FAIL",
 125                 "#endif",
 126             ]
 127         )
 128
 129     def test_conditional_if_1_if_1(self):
 130         self.do_include_pass(
 131             [
 132                 "#if 1",
 133                 "#if 1",
 134                 "PASS",
 135                 "#else",
 136                 "FAIL",
 137                 "#endif",
 138                 "#else",
 139                 "FAIL",
 140                 "#endif",
 141             ]
 142         )
 143
 144     def test_conditional_not_0(self):
 145         self.do_include_pass(
 146             [
 147                 "#if !0",
 148                 "PASS",
 149                 "#else",
 150                 "FAIL",
 151                 "#endif",
 152             ]
 153         )
 154
 155     def test_conditional_not_0_and_1(self):
 156         self.do_include_pass(
 157             [
 158                 "#if !0 && !1",
 159                 "FAIL",
 160                 "#else",
 161                 "PASS",
 162                 "#endif",
 163             ]
 164         )
 165
 166     def test_conditional_not_1(self):
 167         self.do_include_pass(
 168             [
 169                 "#if !1",
 170                 "FAIL",
 171                 "#else",
 172                 "PASS",
 173                 "#endif",
 174             ]
 175         )
 176
 177     def test_conditional_not_emptyval(self):
 178         self.do_include_compare(
 179             [
 180                 "#define EMPTYVAL",
 181                 "#ifndef EMPTYVAL",
 182                 "FAIL",
 183                 "#else",
 184                 "PASS",
 185                 "#endif",
 186                 "#ifdef EMPTYVAL",
 187                 "PASS",
 188                 "#else",
 189                 "FAIL",
 190                 "#endif",
 191             ],
 192             ["PASS", "PASS"],
 193         )
 194
 195     def test_conditional_not_nullval(self):
 196         self.do_include_pass(
 197             [
 198                 "#define NULLVAL 0",
 199                 "#if !NULLVAL",
 200                 "PASS",
 201                 "#else",
 202                 "FAIL",
 203                 "#endif",
 204             ]
 205         )
 206
 207     def test_indentation(self):
 208         self.do_include_pass(
 209             [
 210                 "         #define NULLVAL 0",
 211                 " #if !NULLVAL",
 212                 "PASS",
 213                 "           #else",
 214                 "FAIL",
 215                 "     #endif",
 216             ]
 217         )
 218
 219     def test_expand(self):
 220         self.do_include_pass(
 221             [
 222                 "#define ASVAR AS",
 223                 "#expand P__ASVAR__S",
 224             ]
 225         )
 226
 227     def test_undef_defined(self):
 228         self.do_include_compare(
 229             [
 230                 "#define BAR",
 231                 "#undef BAR",
 232                 "BAR",
 233             ],
 234             ["BAR"],
 235         )
 236
 237     def test_undef_undefined(self):
 238         self.do_include_compare(
 239             [
 240                 "#undef BAR",
 241             ],
 242             [],
 243         )
 244
 245     def test_filter_attemptSubstitution(self):
 246         self.do_include_compare(
 247             [
 248                 "#filter attemptSubstitution",
 249                 "@PASS@",
 250                 "#unfilter attemptSubstitution",
 251             ],
 252             ["@PASS@"],
 253         )
 254
 255     def test_filter_emptyLines(self):
 256         self.do_include_compare(
 257             [
 258                 "lines with a",
 259                 "",
 260                 "blank line",
 261                 "#filter emptyLines",
 262                 "lines with",
 263                 "",
 264                 "no blank lines",
 265                 "#unfilter emptyLines",
 266                 "yet more lines with",
 267                 "",
 268                 "blank lines",
 269             ],
 270             [
 271                 "lines with a",
 272                 "",
 273                 "blank line",
 274                 "lines with",
 275                 "no blank lines",
 276                 "yet more lines with",
 277                 "",
 278                 "blank lines",
 279             ],
 280         )
 281
 282     def test_filter_dumbComments(self):
 283         self.do_include_compare(
 284             [
 285                 "#filter dumbComments",
 286                 "PASS//PASS  // PASS",
 287                 "  //FAIL",
 288                 "//     FAIL",
 289                 "PASS  //",
 290                 "PASS  //    FAIL",
 291                 "//",
 292                 "",
 293                 "#unfilter dumbComments",
 294                 "// PASS",
 295             ],
 296             [
 297                 "PASS//PASS  // PASS",
 298                 "",
 299                 "",
 300                 "PASS  //",
 301                 "PASS  //    FAIL",
 302                 "",
 303                 "",
 304                 "// PASS",
 305             ],
 306         )
 307
 308     def test_filter_dumbComments_and_emptyLines(self):
 309         self.do_include_compare(
 310             [
 311                 "#filter dumbComments emptyLines",
 312                 "PASS//PASS  // PASS",
 313                 "  //FAIL",
 314                 "//     FAIL",
 315                 "PASS  //",
 316                 "PASS  //    FAIL",
 317                 "//",
 318                 "",
 319                 "#unfilter dumbComments emptyLines",
 320                 "",
 321                 "// PASS",
 322             ],
 323             [
 324                 "PASS//PASS  // PASS",
 325                 "PASS  //",
 326                 "PASS  //    FAIL",
 327                 "",
 328                 "// PASS",
 329             ],
 330         )
 331
 332     def test_filter_substitution(self):
 333         self.do_include_pass(
 334             [
 335                 "#define VAR ASS",
 336                 "#filter substitution",
 337                 "P@VAR@",
 338                 "#unfilter substitution",
 339             ]
 340         )
 341
 342     def test_error(self):
 343         with MockedOpen({"f": "#error spit this message out\n"}):
 344             with self.assertRaises(Preprocessor.Error) as e:
 345                 self.pp.do_include("f")
 346                 self.assertEqual(e.args[0][-1], "spit this message out")
 347
 348     def test_ambigous_command(self):
 349         comment = "# if I tell you a joke\n"
 350         with MockedOpen({"f": comment}):
 351             with self.assertRaises(Preprocessor.Error) as e:
 352                 self.pp.do_include("f")
 353             the_exception = e.exception
 354             self.assertEqual(the_exception.args[0][-1], comment)
 355
 356     def test_javascript_line(self):
 357         # The preprocessor is reading the filename from somewhere not caught
 358         # by MockedOpen.
 359         tmpdir = mkdtemp()
 360         try:
 361             full = os.path.join(tmpdir, "javascript_line.js.in")
 362             with open(full, "w") as fh:
 363                 fh.write(
 364                     "\n".join(
 365                         [
 366                             "// Line 1",
 367                             "#if 0",
 368                             "// line 3",
 369                             "#endif",
 370                             "// line 5",
 371                             "# comment",
 372                             "// line 7",
 373                             "// line 8",
 374                             "// line 9",
 375                             "# another comment",
 376                             "// line 11",
 377                             "#define LINE 1",
 378                             "// line 13, given line number overwritten with 2",
 379                             "",
 380                         ]
 381                     )
 382                 )
 383
 384             self.pp.do_include(full)
 385             out = "\n".join(
 386                 [
 387                     "// Line 1",
 388                     '//@line 5 "CWDjavascript_line.js.in"',
 389                     "// line 5",
 390                     '//@line 7 "CWDjavascript_line.js.in"',
 391                     "// line 7",
 392                     "// line 8",
 393                     "// line 9",
 394                     '//@line 11 "CWDjavascript_line.js.in"',
 395                     "// line 11",
 396                     '//@line 2 "CWDjavascript_line.js.in"',
 397                     "// line 13, given line number overwritten with 2",
 398                     "",
 399                 ]
 400             )
 401             out = out.replace("CWD", tmpdir + os.path.sep)
 402             self.assertEqual(self.pp.out.getvalue(), out)
 403         finally:
 404             shutil.rmtree(tmpdir)
 405
 406     def test_literal(self):
 407         self.do_include_pass(
 408             [
 409                 "#literal PASS",
 410             ]
 411         )
 412
 413     def test_var_directory(self):
 414         self.do_include_pass(
 415             [
 416                 "#ifdef DIRECTORY",
 417                 "PASS",
 418                 "#else",
 419                 "FAIL",
 420                 "#endif",
 421             ]
 422         )
 423
 424     def test_var_file(self):
 425         self.do_include_pass(
 426             [
 427                 "#ifdef FILE",
 428                 "PASS",
 429                 "#else",
 430                 "FAIL",
 431                 "#endif",
 432             ]
 433         )
 434
 435     def test_var_if_0(self):
 436         self.do_include_pass(
 437             [
 438                 "#define VAR 0",
 439                 "#if VAR",
 440                 "FAIL",
 441                 "#else",
 442                 "PASS",
 443                 "#endif",
 444             ]
 445         )
 446
 447     def test_var_if_0_elifdef(self):
 448         self.do_include_pass(
 449             [
 450                 "#if 0",
 451                 "#elifdef FILE",
 452                 "PASS",
 453                 "#else",
 454                 "FAIL",
 455                 "#endif",
 456             ]
 457         )
 458
 459     def test_var_if_0_elifndef(self):
 460         self.do_include_pass(
 461             [
 462                 "#if 0",
 463                 "#elifndef VAR",
 464                 "PASS",
 465                 "#else",
 466                 "FAIL",
 467                 "#endif",
 468             ]
 469         )
 470
 471     def test_var_ifdef_0(self):
 472         self.do_include_pass(
 473             [
 474                 "#define VAR 0",
 475                 "#ifdef VAR",
 476                 "PASS",
 477                 "#else",
 478                 "FAIL",
 479                 "#endif",
 480             ]
 481         )
 482
 483     def test_var_ifdef_1_or_undef(self):
 484         self.do_include_pass(
 485             [
 486                 "#define FOO 1",
 487                 "#if defined(FOO) || defined(BAR)",
 488                 "PASS",
 489                 "#else",
 490                 "FAIL",
 491                 "#endif",
 492             ]
 493         )
 494
 495     def test_var_ifdef_undef(self):
 496         self.do_include_pass(
 497             [
 498                 "#define VAR 0",
 499                 "#undef VAR",
 500                 "#ifdef VAR",
 501                 "FAIL",
 502                 "#else",
 503                 "PASS",
 504                 "#endif",
 505             ]
 506         )
 507
 508     def test_var_ifndef_0(self):
 509         self.do_include_pass(
 510             [
 511                 "#define VAR 0",
 512                 "#ifndef VAR",
 513                 "FAIL",
 514                 "#else",
 515                 "PASS",
 516                 "#endif",
 517             ]
 518         )
 519
 520     def test_var_ifndef_0_and_undef(self):
 521         self.do_include_pass(
 522             [
 523                 "#define FOO 0",
 524                 "#if !defined(FOO) && !defined(BAR)",
 525                 "FAIL",
 526                 "#else",
 527                 "PASS",
 528                 "#endif",
 529             ]
 530         )
 531
 532     def test_var_ifndef_undef(self):
 533         self.do_include_pass(
 534             [
 535                 "#define VAR 0",
 536                 "#undef VAR",
 537                 "#ifndef VAR",
 538                 "PASS",
 539                 "#else",
 540                 "FAIL",
 541                 "#endif",
 542             ]
 543         )
 544
 545     def test_var_line(self):
 546         self.do_include_pass(
 547             [
 548                 "#ifdef LINE",
 549                 "PASS",
 550                 "#else",
 551                 "FAIL",
 552                 "#endif",
 553             ]
 554         )
 555
 556     def test_filterDefine(self):
 557         self.do_include_pass(
 558             [
 559                 "#filter substitution",
 560                 "#define VAR AS",
 561                 "#define VAR2 P@VAR@",
 562                 "@VAR2@S",
 563             ]
 564         )
 565
 566     def test_number_value_equals(self):
 567         self.do_include_pass(
 568             [
 569                 "#define FOO 1000",
 570                 "#if FOO == 1000",
 571                 "PASS",
 572                 "#else",
 573                 "FAIL",
 574                 "#endif",
 575             ]
 576         )
 577
 578     def test_default_defines(self):
 579         self.pp.handleCommandLine(["-DFOO"])
 580         self.do_include_pass(
 581             [
 582                 "#if FOO == 1",
 583                 "PASS",
 584                 "#else",
 585                 "FAIL",
 586             ]
 587         )
 588
 589     def test_number_value_equals_defines(self):
 590         self.pp.handleCommandLine(["-DFOO=1000"])
 591         self.do_include_pass(
 592             [
 593                 "#if FOO == 1000",
 594                 "PASS",
 595                 "#else",
 596                 "FAIL",
 597             ]
 598         )
 599
 600     def test_octal_value_equals(self):
 601         self.do_include_pass(
 602             [
 603                 "#define FOO 0100",
 604                 "#if FOO == 0100",
 605                 "PASS",
 606                 "#else",
 607                 "FAIL",
 608                 "#endif",
 609             ]
 610         )
 611
 612     def test_octal_value_equals_defines(self):
 613         self.pp.handleCommandLine(["-DFOO=0100"])
 614         self.do_include_pass(
 615             [
 616                 "#if FOO == 0100",
 617                 "PASS",
 618                 "#else",
 619                 "FAIL",
 620                 "#endif",
 621             ]
 622         )
 623
 624     def test_value_quoted_expansion(self):
 625         """
 626         Quoted values on the commandline don't currently have quotes stripped.
 627         Pike says this is for compat reasons.
 628         """
 629         self.pp.handleCommandLine(['-DFOO="ABCD"'])
 630         self.do_include_compare(
 631             [
 632                 "#filter substitution",
 633                 "@FOO@",
 634             ],
 635             ['"ABCD"'],
 636         )
 637
 638     def test_octal_value_quoted_expansion(self):
 639         self.pp.handleCommandLine(['-DFOO="0100"'])
 640         self.do_include_compare(
 641             [
 642                 "#filter substitution",
 643                 "@FOO@",
 644             ],
 645             ['"0100"'],
 646         )
 647
 648     def test_number_value_not_equals_quoted_defines(self):
 649         self.pp.handleCommandLine(['-DFOO="1000"'])
 650         self.do_include_pass(
 651             [
 652                 "#if FOO == 1000",
 653                 "FAIL",
 654                 "#else",
 655                 "PASS",
 656                 "#endif",
 657             ]
 658         )
 659
 660     def test_octal_value_not_equals_quoted_defines(self):
 661         self.pp.handleCommandLine(['-DFOO="0100"'])
 662         self.do_include_pass(
 663             [
 664                 "#if FOO == 0100",
 665                 "FAIL",
 666                 "#else",
 667                 "PASS",
 668                 "#endif",
 669             ]
 670         )
 671
 672     def test_undefined_variable(self):
 673         with MockedOpen({"f": "#filter substitution\n@foo@"}):
 674             with self.assertRaises(Preprocessor.Error) as e:
 675                 self.pp.do_include("f")
 676                 self.assertEqual(e.key, "UNDEFINED_VAR")
 677
 678     def test_include(self):
 679         files = {
 680             "foo/test": "\n".join(
 681                 [
 682                     "#define foo foobarbaz",
 683                     "#include @inc@",
 684                     "@bar@",
 685                     "",
 686                 ]
 687             ),
 688             "bar": "\n".join(
 689                 [
 690                     "#define bar barfoobaz",
 691                     "@foo@",
 692                     "",
 693                 ]
 694             ),
 695             "f": "\n".join(
 696                 [
 697                     "#filter substitution",
 698                     "#define inc ../bar",
 699                     "#include foo/test",
 700                     "",
 701                 ]
 702             ),
 703         }
 704
 705         with MockedOpen(files):
 706             self.pp.do_include("f")
 707             self.assertEqual(self.pp.out.getvalue(), "foobarbaz\nbarfoobaz\n")
 708
 709     def test_include_line(self):
 710         files = {
 711             "srcdir/test.js": "\n".join(
 712                 [
 713                     "#define foo foobarbaz",
 714                     "#include @inc@",
 715                     "@bar@",
 716                     "",
 717                 ]
 718             ),
 719             "srcdir/bar.js": "\n".join(
 720                 [
 721                     "#define bar barfoobaz",
 722                     "@foo@",
 723                     "",
 724                 ]
 725             ),
 726             "srcdir/foo.js": "\n".join(
 727                 [
 728                     "bazfoobar",
 729                     "#include bar.js",
 730                     "bazbarfoo",
 731                     "",
 732                 ]
 733             ),
 734             "objdir/baz.js": "baz\n",
 735             "srcdir/f.js": "\n".join(
 736                 [
 737                     "#include foo.js",
 738                     "#filter substitution",
 739                     "#define inc bar.js",
 740                     "#include test.js",
 741                     "#include ../objdir/baz.js",
 742                     "fin",
 743                     "",
 744                 ]
 745             ),
 746         }
 747
 748         preprocessed = (
 749             '//@line 1 "$SRCDIR/foo.js"\n'
 750             "bazfoobar\n"
 751             '//@line 2 "$SRCDIR/bar.js"\n'
 752             "@foo@\n"
 753             '//@line 3 "$SRCDIR/foo.js"\n'
 754             "bazbarfoo\n"
 755             '//@line 2 "$SRCDIR/bar.js"\n'
 756             "foobarbaz\n"
 757             '//@line 3 "$SRCDIR/test.js"\n'
 758             "barfoobaz\n"
 759             '//@line 1 "$OBJDIR/baz.js"\n'
 760             "baz\n"
 761             '//@line 6 "$SRCDIR/f.js"\n'
 762             "fin\n"
 763         )
 764
 765         # Try with separate srcdir/objdir
 766         with MockedOpen(files):
 767             self.pp.topsrcdir = os.path.abspath("srcdir")
 768             self.pp.topobjdir = os.path.abspath("objdir")
 769             self.pp.do_include("srcdir/f.js")
 770             self.assertEqual(self.pp.out.getvalue(), preprocessed)
 771
 772         # Try again with relative objdir
 773         self.setUp()
 774         files["srcdir/objdir/baz.js"] = files["objdir/baz.js"]
 775         del files["objdir/baz.js"]
 776         files["srcdir/f.js"] = files["srcdir/f.js"].replace("../", "")
 777         with MockedOpen(files):
 778             self.pp.topsrcdir = os.path.abspath("srcdir")
 779             self.pp.topobjdir = os.path.abspath("srcdir/objdir")
 780             self.pp.do_include("srcdir/f.js")
 781             self.assertEqual(self.pp.out.getvalue(), preprocessed)
 782
 783     def test_include_missing_file(self):
 784         with MockedOpen({"f": "#include foo\n"}):
 785             with self.assertRaises(Preprocessor.Error) as e:
 786                 self.pp.do_include("f")
 787             self.assertEqual(e.exception.key, "FILE_NOT_FOUND")
 788
 789     def test_include_undefined_variable(self):
 790         with MockedOpen({"f": "#filter substitution\n#include @foo@\n"}):
 791             with self.assertRaises(Preprocessor.Error) as e:
 792                 self.pp.do_include("f")
 793             self.assertEqual(e.exception.key, "UNDEFINED_VAR")
 794
 795     def test_include_literal_at(self):
 796         files = {
 797             "@foo@": "#define foo foobarbaz\n",
 798             "f": "#include @foo@\n#filter substitution\n@foo@\n",
 799         }
 800
 801         with MockedOpen(files):
 802             self.pp.do_include("f")
 803             self.assertEqual(self.pp.out.getvalue(), "foobarbaz\n")
 804
 805     def test_command_line_literal_at(self):
 806         with MockedOpen({"@foo@.in": "@foo@\n"}):
 807             self.pp.handleCommandLine(["-Fsubstitution", "-Dfoo=foobarbaz", "@foo@.in"])
 808             self.assertEqual(self.pp.out.getvalue(), "foobarbaz\n")
 809
 810     def test_invalid_ifdef(self):
 811         with MockedOpen({"dummy": "#ifdef FOO == BAR\nPASS\n#endif"}):
 812             with self.assertRaises(Preprocessor.Error) as e:
 813                 self.pp.do_include("dummy")
 814             self.assertEqual(e.exception.key, "INVALID_VAR")
 815
 816         with MockedOpen({"dummy": "#ifndef FOO == BAR\nPASS\n#endif"}):
 817             with self.assertRaises(Preprocessor.Error) as e:
 818                 self.pp.do_include("dummy")
 819             self.assertEqual(e.exception.key, "INVALID_VAR")
 820
 821         # Trailing whitespaces, while not nice, shouldn't be an error.
 822         self.do_include_pass(
 823             [
 824                 "#ifndef  FOO ",
 825                 "PASS",
 826                 "#endif",
 827             ]
 828         )
 829
 830
 831 if __name__ == "__main__":
 832     main()