tests/test-data-format.sh

   1 #!/usr/bin/env bash
   2 # nbdkit
   3 # Copyright (C) 2018-2021 Red Hat Inc.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 # * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #
  12 # * Redistributions in binary form must reproduce the above copyright
  13 # notice, this list of conditions and the following disclaimer in the
  14 # documentation and/or other materials provided with the distribution.
  15 #
  16 # * Neither the name of Red Hat nor the names of its contributors may be
  17 # used to endorse or promote products derived from this software without
  18 # specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
  21 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  22 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  23 # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
  24 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  27 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  28 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  29 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  30 # OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  31 # SUCH DAMAGE.
  32
  33 # Test the data plugin data parameter.
  34
  35 source ./functions.sh
  36 set -e
  37 set -x
  38
  39 requires_run
  40 requires_nbdsh_uri
  41
  42 # Which allocators can we test?
  43 allocators="sparse malloc"
  44
  45 if nbdkit data --dump-plugin | grep -sq zstd=yes; then
  46     allocators="$allocators zstd"
  47 fi
  48
  49 # Compare the data parameter to expected output.  The second parameter
  50 # (expected) is a literal snippet of Python.  It cannot contain single
  51 # quote characters, but anything else is fine.
  52 do_test ()
  53 {
  54     data="$1"
  55     py="$2"
  56     shift 2
  57
  58     for allocator in $allocators; do
  59         nbdkit -U - -v -D data.AST=1 \
  60                data "$data" \
  61                allocator=$allocator "$@" \
  62                --run '
  63                   nbdsh --uri "$uri" -c '\''expected='"$py"\'' -c "
  64 size = h.get_size()
  65 if size > 0:
  66     actual = h.pread(size, 0)
  67 else:
  68     actual = b\"\"
  69 def trunc(s): return s[:128] + (s[128:] and b\"...\")
  70 print(\"actual:   %r\" % trunc(actual))
  71 print(\"expected: %r\" % trunc(expected))
  72 assert actual == expected
  73 "'
  74     done
  75 }
  76
  77 #----------------------------------------------------------------------
  78 # The tests.
  79
  80 # Basic types.
  81 do_test '' 'b""'
  82 do_test '0' 'b"\x00"'
  83 do_test '1' 'b"\x01"'
  84 do_test '65' 'b"A"'
  85 do_test '128' 'b"\x80"'
  86 do_test '0xff' 'b"\xff"'
  87 do_test '0x31' 'b"1"'
  88 do_test '017' 'b"\x0f"'
  89 do_test '0 0' 'b"\x00\x00"'
  90 do_test '0 1' 'b"\x00\x01"'
  91 do_test '1 0' 'b"\x01\x00"'
  92 do_test '1 2' 'b"\x01\x02"'
  93 do_test '1 0xcc' 'b"\x01\xcc"'
  94
  95 do_test 'le16:1' 'b"\x01\x00"'
  96 do_test 'be16:1' 'b"\x00\x01"'
  97 do_test 'le32:1' 'b"\x01\x00\x00\x00"'
  98 do_test 'be32:1' 'b"\x00\x00\x00\x01"'
  99 do_test 'le64:1' 'b"\x01\x00\x00\x00\x00\x00\x00\x00"'
 100 do_test 'be64:1' 'b"\x00\x00\x00\x00\x00\x00\x00\x01"'
 101 do_test 'le16:0xffff' 'b"\xff\xff"'
 102 do_test 'be16:0xffff' 'b"\xff\xff"'
 103 do_test 'le32:0xffffffff' 'b"\xff\xff\xff\xff"'
 104 do_test 'be32:0xffffffff' 'b"\xff\xff\xff\xff"'
 105 do_test 'le64:0xffffffffffffffff' 'b"\xff\xff\xff\xff\xff\xff\xff\xff"'
 106 do_test 'be64:0xffffffffffffffff' 'b"\xff\xff\xff\xff\xff\xff\xff\xff"'
 107
 108 do_test '""' 'b""'
 109 do_test '"foo"' 'b"foo"'
 110 do_test '"foo\x00"' 'b"foo\x00"'
 111 do_test '"\x00foo\x00"' 'b"\x00foo\x00"'
 112 do_test ' "hello" ( "\"" "\\" )*2 "\x01\x02\x03" "\n" ' \
 113         'b"hello\"\\\"\\\x01\x02\x03\n"'
 114
 115 do_test '@4 "\x00"' 'b"\x00\x00\x00\x00\x00"'
 116 do_test '@+4 "\x00"' 'b"\x00\x00\x00\x00\x00"'
 117 do_test '@+4 @-1 "\x00"' 'b"\x00\x00\x00\x00"'
 118 do_test '1 @^4 "\x00"' 'b"\x01\x00\x00\x00\x00"'
 119
 120 #----------------------------------------------------------------------
 121 # Comments.
 122 do_test '#' 'b""'
 123 do_test '# ignore' 'b""'
 124 do_test '# ignore
 125          1 2 3' 'b"\1\2\3"'
 126
 127 #----------------------------------------------------------------------
 128 # Nested expressions and repeat.
 129
 130 # Simple nested expression without any operator.
 131 do_test '( 0x55 0xAA )' 'b"\x55\xAA"'
 132
 133 # Check that *1 is optimized correctly.
 134 do_test '( 0x55 0xAA )*1' 'b"\x55\xAA"'
 135
 136 # Repeated nest.
 137 do_test '( 0x55 0xAA )*4' 'b"\x55\xAA" * 4'
 138
 139 # Doubly-nested.
 140 do_test '( @4 ( 0x21 )*4 )*4' 'b"\0\0\0\0!!!!"*4'
 141
 142 # Nest + offset alignment at the end.
 143 do_test '( "Hello" @^8 )*2' 'b"Hello\0\0\0Hello\0\0\0"'
 144
 145 # Nest with only an offset.
 146 do_test '( @4 )' 'bytearray(4)'
 147 do_test '( @4 )*4 1' 'bytearray(16) + b"\1"'
 148 do_test '( @7 )*4' 'bytearray(28)'
 149
 150 # Nested offsets apply only to the nested expression.
 151 do_test '1 (@2 2) (@3 3)' 'b"\1\0\0\2\0\0\0\3"'
 152
 153 # These should all expand to nothing.  They are here to test corner
 154 # cases in the parser and optimizer.
 155 do_test '
 156 () ()*2 ( () ) ( ()*2 ) ( () () )*2
 157 ()*2[:0] ()[:0]*2 (()[:0]*2)[:0]*2
 158 ()*0 ( 1 2 3 )*0
 159 ()*1*2*3
 160 () -> \a
 161 \a (\a) (\a)*2 (\a \a) (\a*2 \a)
 162 \a*2[:0] \a[:0]*2
 163 ' 'b""'
 164
 165 # In nbdkit <= 1.27.5 this caused allocator=zstd to crash.
 166 do_test '0*10' 'bytearray(10)'
 167
 168 #----------------------------------------------------------------------
 169 # Test various optimizations preserve the meaning.
 170
 171 # expr*X*Y is equivalent to expr*(X*Y)
 172 do_test '1*2*3' 'b"\1" * 6'
 173 do_test '1*2*3*4' 'b"\1" * 24'
 174
 175 # ( ( expr ) ) optimized to ( expr )
 176 do_test '( ( ( ( 1 2 ) ) ) )' 'b"\1\2"'
 177
 178 # string*N is sometimes optimized to N copies of string.
 179 do_test '"foo"*2' 'b"foo" * 2'
 180 do_test '"foo"*2*2' 'b"foo" * 4'
 181 do_test '"foo"*2*50' 'b"foo" * 100'
 182
 183 # ( const ) is sometimes optimized to const
 184 do_test '( "foo" )' 'b"foo"'
 185 do_test '( <(echo hello) )' 'b"hello\n"'
 186 do_test '( $hello )' 'b"hello"' hello=' "hello" '
 187
 188 # Single byte * N is optimized to a fill.
 189 do_test '1*100000' 'b"\1" * 100000'
 190 do_test '"1"*100000' 'b"1" * 100000'
 191
 192 # Zero fill should overwrite existing data.
 193 do_test '1*1000 @100 0*100' 'b"\1" * 100 + bytearray(100) + b"\1" * 800'
 194
 195 # Zero fill should extend the disk.
 196 do_test '1*1000 @100 0*1000' 'b"\1" * 100 + bytearray(1000)'
 197
 198 # Combining adjacent bytes, strings and fills.
 199 do_test '1 2 "3"' 'b"\x01\x023"'
 200 do_test '1 2 "3"*3 4' 'b"\x01\x02333\x04"'
 201
 202 #----------------------------------------------------------------------
 203 # Assignments.
 204 do_test '
 205 # Assign to \a in the outer scope.
 206 (0x31 0x32) -> \a
 207 (0x35 0x36) -> \b
 208 (
 209   # Assign to \a and \c in the inner scope.
 210   (0x33 0x34) -> \a
 211   (0x37 0x38) -> \c
 212   \a \a \b \c
 213 )
 214 # The end of the inner scope should restore the outer
 215 # scope definition of \a.
 216 \a \b
 217 ' 'b"343456781256"'
 218
 219 # Test environment capture in -> assignments.
 220 do_test '
 221 # In the global scope, assign \1 = 1
 222 1 -> \1
 223
 224 # Create a few more assignments.  These should make no difference
 225 # to anything.
 226 "foo" -> \foo
 227 42 -> \a-number
 228 2 -> \2
 229
 230 # Capture the assignment to \1 (= 1) in a nested expression.
 231 # Note this is parsed as EXPR_ASSIGN ("\test", EXPR_EXPR (EXPR_NAME "\1"))
 232 ( \1 ) -> \test
 233
 234 # Obviously this should evaluate to 1.
 235 \test
 236
 237 # Since assignments only last to the end of the scope in which they
 238 # are created, the assignment inside the nested scope here should make
 239 # no difference, \test should still evaluate to 1.
 240 ( 2 -> \1 )  \test
 241
 242 # In a nested scope, change \1 and evaluate \test.
 243 # It should still be 1 because of the captured environment from above.
 244 ( 3 -> \1  \test )
 245
 246 # In the global scope, change \1 and evaluate \test.
 247 # It should still be 1 because of the captured environment from above.
 248 4 -> \1  \test
 249
 250 # Now reassign \test to the new value of \1 (= 4) and evaluate.
 251 # It should now be the new value (4).
 252 ( \1 ) -> \test
 253 \test
 254 ' 'b"\1\1\1\1\4"'
 255
 256 #----------------------------------------------------------------------
 257 # Slices.
 258 do_test '( $hello )[:4]' 'b"Hell"' hello=' "Hello" '
 259 do_test '( "Hello" )[3:]' 'b"lo"'
 260 do_test '( "Hello" )[3:5]' 'b"lo"'
 261
 262 # With the new parser it should work without the parens too.
 263 do_test '"Hello"[:]' 'b"Hello"'
 264 do_test '$hello[:]' 'b"Hello"' hello=' "Hello" '
 265 do_test '$hello[:4]' 'b"Hell"' hello=' "Hello" '
 266 do_test '"Hello"[3:]' 'b"lo"'
 267 do_test '"Hello"[3:5]' 'b"lo"'
 268
 269 # Zero length slices are optimized out.  The first index is ignored.
 270 do_test '"Hello"[:0]' 'b""'
 271 do_test '"Hello"[99:99]' 'b""'
 272
 273 #----------------------------------------------------------------------
 274 # Scripts.
 275 do_test '<( for i in `seq 0 7`; do printf "%04d" $i; done )' \
 276         'b"00000001000200030004000500060007"'
 277 do_test '<( i=0
 278                 while :; do
 279                     printf "%04d" $i; i=$((i+1))
 280                 done )[:65]' \
 281         'b"00000001000200030004000500060007000800090010001100120013001400150"'
 282
 283 #----------------------------------------------------------------------
 284 # Variables.
 285
 286 # Check there are no environment variables that might
 287 # interfere with the test.
 288 unset a
 289 unset b
 290 unset c
 291
 292 # Unknown variable should fail.
 293 if nbdkit -U - data ' $a $b $c ' --run 'exit 0'; then
 294     echo "$0: expected unknown variables to fail"
 295     exit 1
 296 fi
 297
 298 # Set environment variables to patterns.
 299 export a=' 1 2 '
 300 export b=' 3 4 '
 301 export c=' 5*5 '
 302 do_test ' $a $b $c ' 'b"\1\2\3\4\5\5\5\5\5"'
 303
 304 # Same but using command line parameters.
 305 unset a
 306 unset b
 307 unset c
 308 do_test ' $a $b $c ' 'b"\1\2\3\4\5\5\5\5\5"' \
 309         a=' 1 2 ' b=' 3 4 ' c=' 5*5 '
 310
 311 # Same but using a mix.
 312 export a='BLAH'
 313 export b=' 3 4 '
 314 export c='FOO'
 315 do_test ' $a $b $c ' 'b"\1\2\3\4\5\5\5\5\5"' \
 316         a=' 1 2 ' c=' 5*5 '
 317
 318 # Variables referencing variables.
 319 unset a
 320 unset b
 321 unset c
 322 do_test ' $a $b $c ' 'b"\1\2\3\4\5\5\5\5\5"' \
 323         a=' 1 2 ' b=' 3 4 ' c=' $d*5 ' d=' 5 '
 324
 325 # Badly formatted variable should fail.
 326 if nbdkit -U - data ' $a ' a='NONSENSE' --run 'exit 0'; then
 327     echo "$0: expected unknown variables to fail"
 328     exit 1
 329 fi
 330
 331 # Using an extra parameter without data= should fail.
 332 if nbdkit -U - data raw='' a='NONSENSE' --run 'exit 0'; then
 333     echo "$0: expected extra params to fail with !data"
 334     exit 1
 335 fi
 336
 337 unset a
 338 unset b
 339 unset c
 340
 341 #----------------------------------------------------------------------
 342 # Some tests at offsets.
 343 #
 344 # Most of the tests above fit into a single page for sparse and zstd
 345 # allocators (32K).  It could be useful to test at page boundaries.
 346
 347 do_test '@32766 1 2 3'   'b"\0"*32766 + b"\1\2\3"'
 348 do_test '@32766 1*6'     'b"\0"*32766 + b"\1"*6'
 349 do_test '@32766 1*32800' 'b"\0"*32766 + b"\1"*32800'
 350
 351 # Since we do sparseness detection, automatically trimming whole
 352 # pages if they are zero, this should be interesting:
 353 do_test '@32766 1*5 @65534 2*5 @32768 0*32768' \
 354         'b"\0"*32766 + b"\1\1" + b"\0"*32768 + b"\2\2\2"'