t/t4216-log-bloom.sh

   1 #!/bin/sh
   2
   3 test_description='git log for a path with Bloom filters'
   4 . ./test-lib.sh
   5
   6 GIT_TEST_COMMIT_GRAPH=0
   7 GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=0
   8
   9 test_expect_success 'setup test - repo, commits, commit graph, log outputs' '
  10         git init &&
  11         mkdir A A/B A/B/C &&
  12         test_commit c1 A/file1 &&
  13         test_commit c2 A/B/file2 &&
  14         test_commit c3 A/B/C/file3 &&
  15         test_commit c4 A/file1 &&
  16         test_commit c5 A/B/file2 &&
  17         test_commit c6 A/B/C/file3 &&
  18         test_commit c7 A/file1 &&
  19         test_commit c8 A/B/file2 &&
  20         test_commit c9 A/B/C/file3 &&
  21         test_commit c10 file_to_be_deleted &&
  22         git checkout -b side HEAD~4 &&
  23         test_commit side-1 file4 &&
  24         git checkout master &&
  25         git merge side &&
  26         test_commit c11 file5 &&
  27         mv file5 file5_renamed &&
  28         git add file5_renamed &&
  29         git commit -m "rename" &&
  30         rm file_to_be_deleted &&
  31         git add . &&
  32         git commit -m "file removed" &&
  33         git commit --allow-empty -m "empty" &&
  34         git commit-graph write --reachable --changed-paths &&
  35
  36         test_oid_cache <<-EOF
  37         oid_version sha1:1
  38         oid_version sha256:2
  39         EOF
  40 '
  41
  42 graph_read_expect () {
  43         NUM_CHUNKS=5
  44         cat >expect <<- EOF
  45         header: 43475048 1 $(test_oid oid_version) $NUM_CHUNKS 0
  46         num_commits: $1
  47         chunks: oid_fanout oid_lookup commit_metadata bloom_indexes bloom_data
  48         EOF
  49         test-tool read-graph >actual &&
  50         test_cmp expect actual
  51 }
  52
  53 test_expect_success 'commit-graph write wrote out the bloom chunks' '
  54         graph_read_expect 16
  55 '
  56
  57 # Turn off any inherited trace2 settings for this test.
  58 sane_unset GIT_TRACE2 GIT_TRACE2_PERF GIT_TRACE2_EVENT
  59 sane_unset GIT_TRACE2_PERF_BRIEF
  60 sane_unset GIT_TRACE2_CONFIG_PARAMS
  61
  62 setup () {
  63         rm -f "$TRASH_DIRECTORY/trace.perf" &&
  64         git -c core.commitGraph=false log --pretty="format:%s" $1 >log_wo_bloom &&
  65         GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.perf" git -c core.commitGraph=true log --pretty="format:%s" $1 >log_w_bloom
  66 }
  67
  68 test_bloom_filters_used () {
  69         log_args=$1
  70         bloom_trace_prefix="statistics:{\"filter_not_present\":${2:-0},\"maybe\""
  71         setup "$log_args" &&
  72         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
  73         test_cmp log_wo_bloom log_w_bloom &&
  74     test_path_is_file "$TRASH_DIRECTORY/trace.perf"
  75 }
  76
  77 test_bloom_filters_not_used () {
  78         log_args=$1
  79         setup "$log_args" &&
  80         ! grep -q "statistics:{\"filter_not_present\":" "$TRASH_DIRECTORY/trace.perf" &&
  81         test_cmp log_wo_bloom log_w_bloom
  82 }
  83
  84 for path in A A/B A/B/C A/file1 A/B/file2 A/B/C/file3 file4 file5 file5_renamed file_to_be_deleted
  85 do
  86         for option in "" \
  87               "--all" \
  88                       "--full-history" \
  89                       "--full-history --simplify-merges" \
  90                       "--simplify-merges" \
  91                       "--simplify-by-decoration" \
  92                       "--follow" \
  93                       "--first-parent" \
  94                       "--topo-order" \
  95                       "--date-order" \
  96                       "--author-date-order" \
  97                       "--ancestry-path side..master"
  98         do
  99                 test_expect_success "git log option: $option for path: $path" '
 100                         test_bloom_filters_used "$option -- $path" &&
 101                         test_config commitgraph.readChangedPaths false &&
 102                         test_bloom_filters_not_used "$option -- $path"
 103                 '
 104         done
 105 done
 106
 107 test_expect_success 'git log -- folder works with and without the trailing slash' '
 108         test_bloom_filters_used "-- A" &&
 109         test_bloom_filters_used "-- A/"
 110 '
 111
 112 test_expect_success 'git log for path that does not exist. ' '
 113         test_bloom_filters_used "-- path_does_not_exist"
 114 '
 115
 116 test_expect_success 'git log with --walk-reflogs does not use Bloom filters' '
 117         test_bloom_filters_not_used "--walk-reflogs -- A"
 118 '
 119
 120 test_expect_success 'git log -- multiple path specs does not use Bloom filters' '
 121         test_bloom_filters_not_used "-- file4 A/file1"
 122 '
 123
 124 test_expect_success 'git log -- "." pathspec at root does not use Bloom filters' '
 125         test_bloom_filters_not_used "-- ."
 126 '
 127
 128 test_expect_success 'git log with wildcard that resolves to a single path uses Bloom filters' '
 129         test_bloom_filters_used "-- *4" &&
 130         test_bloom_filters_used "-- *renamed"
 131 '
 132
 133 test_expect_success 'git log with wildcard that resolves to a multiple paths does not uses Bloom filters' '
 134         test_bloom_filters_not_used "-- *" &&
 135         test_bloom_filters_not_used "-- file*"
 136 '
 137
 138 test_expect_success 'setup - add commit-graph to the chain without Bloom filters' '
 139         test_commit c14 A/anotherFile2 &&
 140         test_commit c15 A/B/anotherFile2 &&
 141         test_commit c16 A/B/C/anotherFile2 &&
 142         git commit-graph write --reachable --split --no-changed-paths &&
 143         test_line_count = 2 .git/objects/info/commit-graphs/commit-graph-chain
 144 '
 145
 146 test_expect_success 'use Bloom filters even if the latest graph does not have Bloom filters' '
 147         # Ensure that the number of empty filters is equal to the number of
 148         # filters in the latest graph layer to prove that they are loaded (and
 149         # ignored).
 150         test_bloom_filters_used "-- A/B" 3
 151 '
 152
 153 test_expect_success 'setup - add commit-graph to the chain with Bloom filters' '
 154         test_commit c17 A/anotherFile3 &&
 155         git commit-graph write --reachable --changed-paths --split &&
 156         test_line_count = 3 .git/objects/info/commit-graphs/commit-graph-chain
 157 '
 158
 159 test_bloom_filters_used_when_some_filters_are_missing () {
 160         log_args=$1
 161         bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":9"
 162         setup "$log_args" &&
 163         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
 164         test_cmp log_wo_bloom log_w_bloom
 165 }
 166
 167 test_expect_success 'Use Bloom filters if they exist in the latest but not all commit graphs in the chain.' '
 168         test_bloom_filters_used_when_some_filters_are_missing "-- A/B"
 169 '
 170
 171 test_expect_success 'persist filter settings' '
 172         test_when_finished rm -rf .git/objects/info/commit-graph* &&
 173         rm -rf .git/objects/info/commit-graph* &&
 174         GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
 175                 GIT_TRACE2_EVENT_NESTING=5 \
 176                 GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=9 \
 177                 GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY=15 \
 178                 git commit-graph write --reachable --changed-paths &&
 179         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2.txt &&
 180         GIT_TRACE2_EVENT="$(pwd)/trace2-auto.txt" \
 181                 GIT_TRACE2_EVENT_NESTING=5 \
 182                 git commit-graph write --reachable --changed-paths &&
 183         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2-auto.txt
 184 '
 185
 186 test_max_changed_paths () {
 187         grep "\"max_changed_paths\":$1" $2
 188 }
 189
 190 test_filter_not_computed () {
 191         grep "\"key\":\"filter-not-computed\",\"value\":\"$1\"" $2
 192 }
 193
 194 test_filter_computed () {
 195         grep "\"key\":\"filter-computed\",\"value\":\"$1\"" $2
 196 }
 197
 198 test_filter_trunc_empty () {
 199         grep "\"key\":\"filter-trunc-empty\",\"value\":\"$1\"" $2
 200 }
 201
 202 test_filter_trunc_large () {
 203         grep "\"key\":\"filter-trunc-large\",\"value\":\"$1\"" $2
 204 }
 205
 206 test_expect_success 'correctly report changes over limit' '
 207         git init limits &&
 208         (
 209                 cd limits &&
 210                 mkdir d &&
 211                 mkdir d/e &&
 212
 213                 for i in $(test_seq 1 2)
 214                 do
 215                         printf $i >d/file$i.txt &&
 216                         printf $i >d/e/file$i.txt || return 1
 217                 done &&
 218
 219                 mkdir mode &&
 220                 printf bash >mode/script.sh &&
 221
 222                 mkdir foo &&
 223                 touch foo/bar &&
 224                 touch foo.txt &&
 225
 226                 git add d foo foo.txt mode &&
 227                 git commit -m "files" &&
 228
 229                 # Commit has 7 file and 4 directory adds
 230                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
 231                         GIT_TRACE2_EVENT="$(pwd)/trace" \
 232                         git commit-graph write --reachable --changed-paths &&
 233                 test_max_changed_paths 10 trace &&
 234                 test_filter_computed 1 trace &&
 235                 test_filter_trunc_large 1 trace &&
 236
 237                 for path in $(git ls-tree -r --name-only HEAD)
 238                 do
 239                         git -c commitGraph.readChangedPaths=false log \
 240                                 -- $path >expect &&
 241                         git log -- $path >actual &&
 242                         test_cmp expect actual || return 1
 243                 done &&
 244
 245                 # Make a variety of path changes
 246                 printf new1 >d/e/file1.txt &&
 247                 printf new2 >d/file2.txt &&
 248                 rm d/e/file2.txt &&
 249                 rm -r foo &&
 250                 printf text >foo &&
 251                 mkdir f &&
 252                 printf new1 >f/file1.txt &&
 253
 254                 # including a mode-only change (counts as modified)
 255                 git update-index --chmod=+x mode/script.sh &&
 256
 257                 git add foo d f &&
 258                 git commit -m "complicated" &&
 259
 260                 # start from scratch and rebuild
 261                 rm -f .git/objects/info/commit-graph &&
 262                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
 263                         GIT_TRACE2_EVENT="$(pwd)/trace-edit" \
 264                         git commit-graph write --reachable --changed-paths &&
 265                 test_max_changed_paths 10 trace-edit &&
 266                 test_filter_computed 2 trace-edit &&
 267                 test_filter_trunc_large 2 trace-edit &&
 268
 269                 for path in $(git ls-tree -r --name-only HEAD)
 270                 do
 271                         git -c commitGraph.readChangedPaths=false log \
 272                                 -- $path >expect &&
 273                         git log -- $path >actual &&
 274                         test_cmp expect actual || return 1
 275                 done &&
 276
 277                 # start from scratch and rebuild
 278                 rm -f .git/objects/info/commit-graph &&
 279                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=11 \
 280                         GIT_TRACE2_EVENT="$(pwd)/trace-update" \
 281                         git commit-graph write --reachable --changed-paths &&
 282                 test_max_changed_paths 11 trace-update &&
 283                 test_filter_computed 2 trace-update &&
 284                 test_filter_trunc_large 0 trace-update &&
 285
 286                 for path in $(git ls-tree -r --name-only HEAD)
 287                 do
 288                         git -c commitGraph.readChangedPaths=false log \
 289                                 -- $path >expect &&
 290                         git log -- $path >actual &&
 291                         test_cmp expect actual || return 1
 292                 done
 293         )
 294 '
 295
 296 test_expect_success 'correctly report commits with no changed paths' '
 297         git init empty &&
 298         test_when_finished "rm -fr empty" &&
 299         (
 300                 cd empty &&
 301
 302                 git commit --allow-empty -m "initial commit" &&
 303
 304                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 305                         git commit-graph write --reachable --changed-paths &&
 306                 test_filter_computed 1 trace.event &&
 307                 test_filter_not_computed 0 trace.event &&
 308                 test_filter_trunc_empty 1 trace.event &&
 309                 test_filter_trunc_large 0 trace.event
 310         )
 311 '
 312
 313 test_expect_success 'Bloom generation is limited by --max-new-filters' '
 314         (
 315                 cd limits &&
 316                 test_commit c2 filter &&
 317                 test_commit c3 filter &&
 318                 test_commit c4 no-filter &&
 319
 320                 rm -f trace.event &&
 321                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 322                         git commit-graph write --reachable --split=replace \
 323                                 --changed-paths --max-new-filters=2 &&
 324
 325                 test_filter_computed 2 trace.event &&
 326                 test_filter_not_computed 3 trace.event &&
 327                 test_filter_trunc_empty 0 trace.event &&
 328                 test_filter_trunc_large 0 trace.event
 329         )
 330 '
 331
 332 test_expect_success 'Bloom generation backfills previously-skipped filters' '
 333         # Check specifying commitGraph.maxNewFilters over "git config" works.
 334         test_config -C limits commitGraph.maxNewFilters 1 &&
 335         (
 336                 cd limits &&
 337
 338                 rm -f trace.event &&
 339                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 340                         git commit-graph write --reachable --changed-paths \
 341                                 --split=replace &&
 342                 test_filter_computed 1 trace.event &&
 343                 test_filter_not_computed 4 trace.event &&
 344                 test_filter_trunc_empty 0 trace.event &&
 345                 test_filter_trunc_large 0 trace.event
 346         )
 347 '
 348
 349 test_expect_success '--max-new-filters overrides configuration' '
 350         git init override &&
 351         test_when_finished "rm -fr override" &&
 352         test_config -C override commitGraph.maxNewFilters 2 &&
 353         (
 354                 cd override &&
 355                 test_commit one &&
 356                 test_commit two &&
 357
 358                 rm -f trace.event &&
 359                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 360                         git commit-graph write --reachable --changed-paths \
 361                                 --max-new-filters=1 &&
 362                 test_filter_computed 1 trace.event &&
 363                 test_filter_not_computed 1 trace.event &&
 364                 test_filter_trunc_empty 0 trace.event &&
 365                 test_filter_trunc_large 0 trace.event
 366         )
 367 '
 368
 369 test_expect_success 'Bloom generation backfills empty commits' '
 370         git init empty &&
 371         test_when_finished "rm -fr empty" &&
 372         (
 373                 cd empty &&
 374                 for i in $(test_seq 1 6)
 375                 do
 376                         git commit --allow-empty -m "$i"
 377                 done &&
 378
 379                 # Generate Bloom filters for empty commits 1-6, two at a time.
 380                 for i in $(test_seq 1 3)
 381                 do
 382                         rm -f trace.event &&
 383                         GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 384                                 git commit-graph write --reachable \
 385                                         --changed-paths --max-new-filters=2 &&
 386                         test_filter_computed 2 trace.event &&
 387                         test_filter_not_computed 4 trace.event &&
 388                         test_filter_trunc_empty 2 trace.event &&
 389                         test_filter_trunc_large 0 trace.event
 390                 done &&
 391
 392                 # Finally, make sure that once all commits have filters, that
 393                 # none are subsequently recomputed.
 394                 rm -f trace.event &&
 395                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 396                         git commit-graph write --reachable \
 397                                 --changed-paths --max-new-filters=2 &&
 398                 test_filter_computed 0 trace.event &&
 399                 test_filter_not_computed 6 trace.event &&
 400                 test_filter_trunc_empty 0 trace.event &&
 401                 test_filter_trunc_large 0 trace.event
 402         )
 403 '
 404
 405 test_done