test/json/json-generator.tcl

   1 #!/usr/bin/tclsh
   2 #
   3 # Generate SQL that will populate an SQLite database with about 100 megabytes
   4 # of pseudo-random JSON text.
   5 #
   6 #     tclsh json-generator.tcl | sqlite3 json110mb.db
   7 #
   8 # srand() is used to initialize the random seed so that the same JSON
   9 # is generated for every run.
  10 #
  11 expr srand(12345678)
  12 set wordlist {
  13    ability   able      abroad    access    account   act
  14    action    active    actor     add       address   adept
  15    adroit    advance   advice    affect    age       ageless
  16    agency    agent     agile     agree     air       airfare
  17    airline   airport   alert     almond    alpha     always
  18    amend     amount    amplify   analyst   anchor    angel
  19    angelic   angle     ankle     annual    answer    antique
  20    anybody   anyhow    appeal    apple     apricot   apt
  21    area      argon     arm       army      arrival   arsenic
  22    art       artful    article   arugula   aside     ask
  23    aspect    assist    assume    atom      atone     attempt
  24    author    autumn    average   avocado   award     awl
  25    azure     back      bacon     bag       bagel     bake
  26    baker     balance   ball      balloon   bamboo    banana
  27    band      banjo     bank      barium    base      basil
  28    basin     basis     basket    bass      bat       bath
  29    battery   beach     beak      bean      bear      bearcub
  30    beauty    beef      beet      beige     being     bell
  31    belly     belt      bench     bend      benefit   best
  32    beta      better    beyond    bicycle   bid       big
  33    bike      bill      bird      biscuit   bismuth   bisque
  34    bit       black     blank     blest     blind     bliss
  35    block     bloom     blue      board     boat      body
  36    bokchoy   bone      bonus     book      bookish   boot
  37    border    boron     boss      bossy     bottle    bottom
  38    bow       bowl      bowtie    box       brain     brainy
  39    branch    brave     bravely   bread     break     breath
  40    breezy    brick     bridge    brie      brief     briefly
  41    bright    broad     broil     bromine   bronze    brother
  42    brow      brown     brush     buddy     budget    buffalo
  43    bug       bugle     bull      bunch     burger    burly
  44    burrito   bus       busy      butter    button    buy
  45    buyer     byte      cab       cabbage   cabinet   cable
  46    cadet     cadmium   caesium   cake      calcium   caliper
  47    call      caller    calm      calmly    camera    camp
  48    can       canary    cancel    candle    candy     cap
  49    capable   caper     capital   captain   car       carbon
  50    card      care      career    careful   carp      carpet
  51    carrot    carry     case      cash      cassava   casual
  52    cat       catch     catfish   catsear   catsup    cause
  53    cave      celery    cell      century   chain     chair
  54    chalk     chance    change    channel   chapter   chard
  55    charge    charity   chart     check     cheddar   cheery
  56    cheese    chicken   chicory   chiffon   child     chin
  57    chip      chives    choice    chowder   chum      church
  58    circle    city      claim     clam      class     classic
  59    classy    clay      clean     cleaner   clear     clearly
  60    clerk     click     client    climate   clock     clorine
  61    closet    clothes   cloud     clown     club      clue
  62    cluster   coach     coast     coat      cobbler   cobolt
  63    cod       code      coffee    colby     cold      collar
  64    college   comb      combine   comet     comfort   command
  65    comment   common    company   complex   concept   concern
  66    concert   conduit   consist   contact   contest   context
  67    control   convert   cook      cookie    copilot   copper
  68    copy      coral     cordial   corn      corner    corny
  69    correct   cost      count     counter   country   county
  70    couple    courage   course    court     cover     cow
  71    cowbird   crab      crack     craft     crash     crazy
  72    cream     credit    creek     cress     crevice   crew
  73    crimson   croaker   crop      cross     crowd     cube
  74    cuckoo    cuisine   culture   cup       current   curve
  75    cut       cyan      cycle     dagger    daily     dance
  76    dare      darter    data      date      day       daylily
  77    deal      dear      dearly    debate    debit     decade
  78    decimal   deep      deft      deftly    degree    delay
  79    deluxe    deposit   depth     design    desk      detail
  80    device    dew       diamond   diet      dig       dill
  81    dinner    dip       direct    dirt      dish      disk
  82    display   diver     divide    divine    doctor    dodger
  83    donut     door      dot       double    dough     draft
  84    drag      dragon    drama     draw      drawer    drawing
  85    dream     drill     drink     drive     driver    drop
  86    drum      dry       dryer     drywall   duck      due
  87    dump      dusk      dust      duty      dye       eagle
  88    ear       earring   earth     ease      east      easy
  89    eat       economy   edge      editor    eel       effect
  90    effort    egg       eight     elbow     elegant   element
  91    elf       elk       email     emerald   employ    end
  92    endive    endless   energy    engine    enjoy     enter
  93    entry     equal     equip     error     escape    essay
  94    eternal   evening   event     exam      example   excuse
  95    exit      expert    extent    extreme   eye       face
  96    fact      factor    factual   fail      failure   fair
  97    fajita    fall      family    fan       fang      farm
  98    farmer    fat       fault     feature   feed      feel
  99    feeling   fench     fennel    festive   few       fiber
 100    field     fig       figure    file      fill      film
 101    filter    final     finance   finding   finger    finish
 102    fire      fish      fishing   fit       fitting   five
 103    fix       flier     flight    floor     floral    florine
 104    flour     flow      flower    fly       flying    focus
 105    fold      folding   food      foot      force     forest
 106    forever   forgive   form      formal    format    fortune
 107    forum     frame     free      freedom   freely    fresh
 108    friend    frog      front     fruit     fuchsia   fuel
 109    fun       funny     future    gain      galaxy    gallium
 110    game      gamma     gap       garage    garden    garlic
 111    gas       gate      gather    gauge     gear      gem
 112    gene      general   gentle    gently    gherkin   ghost
 113    gift      give      glad      glass     gleeful   glossy
 114    glove     glue      goal      goat      goby      gold
 115    goldeye   golf      good      gouda     goulash   gourd
 116    grab      grace     grade     gram      grand     grape
 117    grapes    grass     gravy     gray      great     green
 118    grits     grocery   ground    group     grouper   grout
 119    growth    guard     guave     guess     guest     guide
 120    guitar    gumbo     guppy     habit     hacksaw   haddock
 121    hafnium   hagfish   hair      half      halibut   hall
 122    hammer    hand      handle    handy     hanger    happy
 123    hat       havarti   hay       haybale   head      health
 124    healthy   hearing   heart     hearty    heat      heavy
 125    heel      height    helium    hello     help      helpful
 126    herald    herring   hide      high      highly    highway
 127    hill      hip       hipster   hire      history   hit
 128    hoki      hold      hole      holiday   holly     home
 129    honest    honey     hook      hope      hopeful   horizon
 130    horn      horse     host      hotel     hour      house
 131    housing   human     humane    humor     hunt      hurry
 132    ice       icecube   icefish   icy       idea      ideal
 133    image     impact    impress   inch      income    indigo
 134    initial   inkpen    insect    inside    intense   invite
 135    iodine    iridium   iron      island    issue     item
 136    ivory     jacket    jargon    javelin   jello     jelly
 137    jewel     job       jocund    join      joint     joke
 138    jovial    joy       joyful    joyous    judge     juice
 139    jump      junior    jury      just      justice   kale
 140    keel      keep      kelp      ketchup   key       keyhole
 141    keyway    khaki     kick      kid       kidney    kiloohm
 142    kind      kindly    king      kitchen   kite      kiwi
 143    knee      knife     krill     krypton   kumquat   lab
 144    lace      lack      ladder    lake      lamp      lamprey
 145    land      laser     laugh     law       lawn      lawyer
 146    layer     lead      leader    leading   leaf      leafy
 147    league    leather   leave     lecture   leek      leg
 148    lemon     length    lentil    lesson    let       letter
 149    lettuce   level     library   life      lift      light
 150    lily      lime      limit     line      linen     link
 151    lip       list      listen    lithium   lively    living
 152    lizard    load      loan      lobster   local     lock
 153    log       long      longfin   look      lotus     love
 154    lovely    loving    low       lucid     luck      luffa
 155    lunch     lung      machine   magenta   magnet    mail
 156    main      major     make      mall      manager   mango
 157    manner    many      map       march     market    maroon
 158    martian   master    match     math      matter    maximum
 159    maybe     meal      meaning   meat      media     medium
 160    meet      meeting   melody    melon     member    memory
 161    mention   menu      mercury   merry     mess      message
 162    messy     metal     meter     method    micron    middle
 163    might     mile      milk      mind      mine      minimum
 164    minnow    minor     mint      minute    mirror    miss
 165    mission   misty     mix       mixer     mixture   mobile
 166    mode      model     moment    monitor   monk      month
 167    moon      moray     morning   most      motor     mouse
 168    mouth     move      mover     movie     much      mud
 169    mudfish   muffin    mullet    munster   muon      muscle
 170    music     mustard   nail      name      nation    native
 171    natural   nature    navy      neat      neatly    nebula
 172    neck      needle    neon      nerve     net       network
 173    neutron   news      nibble    nice      nickel    night
 174    niobium   nobody    noise     noodle    normal    north
 175    nose      note      nothing   notice    nova      novel
 176    number    nurse     nursery   oar       object    offer
 177    office    officer   oil       okay      okra      old
 178    olive     one       onion     open      opening   opinion
 179    option    orange    orbit     orchid    order     oregano
 180    other     ounce     outcome   outside   oven      owner
 181    oxygen    oyster    pace      pack      package   page
 182    pager     paint     pair      pale      pan       pancake
 183    papaya    paper     pardon    parent    park      parking
 184    parsley   parsnip   part      partner   party     pass
 185    passage   past      pasta     path      patient   pattern
 186    pause     pay       pea       peace     peach     peacock
 187    peahen    peak      peanut    pear      pearl     pen
 188    penalty   pencil    pension   people    pepper    perch
 189    perfect   period    permit    person    phase     phone
 190    photo     phrase    physics   piano     pick      picture
 191    pie       piece     pigeon    pike      pilot     pin
 192    pink      pinkie    pious     pipe      pitch     pizza
 193    place     plan      plane     planet    plant     planter
 194    plastic   plate     play      player    playful   plenty
 195    pliers    plum      pod       poem      poet      poetry
 196    point     police    policy    pollock   pony      pool
 197    pop       popover   poptart   pork      port      portal
 198    post      pot       potato    pound     powder    power
 199    present   press     price     pride     primary   print
 200    prior     private   prize     problem   process   produce
 201    product   profile   profit    program   project   promise
 202    prompt    proof     proper    protein   proton    public
 203    puff      puffer    pull      pumpkin   pup       pupfish
 204    pure      purple    purpose   push      put       quality
 205    quark     quarter   quiet     quill     quit      quote
 206    rabbit    raccoon   race      radiant   radio     radish
 207    radium    radon     rain      rainbow   raise     ramp
 208    ranch     range     rasp      rate      ratio     ray
 209    razor     reach     read      reading   real      reality
 210    reason    recipe    record    recover   red       redeem
 211    reed      reef      refuse    region    regret    regular
 212    relaxed   release   relief    relish    remote    remove
 213    rent      repair    repeat    reply     report    request
 214    reserve   resist    resolve   resort    rest      result
 215    return    reveal    review    reward    ribbon    rice
 216    rich      ride      ridge     right     ring      rise
 217    risk      river     rivet     road      roast     rock
 218    rocket    role      roll      roof      room      rope
 219    rose      rough     roughy    round     row       royal
 220    rub       ruby      rudder    ruin      rule      run
 221    runner    rush      rust      sacred    saddle    safe
 222    safety    sail      salad     salami    sale      salmon
 223    salt      sample    sand      sander    sandy     sauce
 224    save      saving    saw       scale     scampi    scene
 225    scheme    school    score     screen    script    sea
 226    search    season    seat      second    secret    sector
 227    seemly    self      sell      senate    senior    sense
 228    series    serve     set       shake     shape     share
 229    shark     shell     shift     shine     shiny     ship
 230    shock     shoe      shoot     shop      shovel    show
 231    side      sign      signal    silk      silly     silver
 232    simple    sing      singer    single    sink      site
 233    size      skill     skin      sky       slate     sleep
 234    sleepy    slice     slide     slip      smart     smell
 235    smelt     smile     smoke     smooth    snap      snipe
 236    snow      snowy     sock      socket    sodium    soft
 237    softly    soil      sole      solid     song      sorrel
 238    sort      soul      sound     soup      source    south
 239    space     spare     speech    speed     spell     spend
 240    sphere    spice     spider    spirit    spite     split
 241    spoon     sport     spot      spray     spread    spring
 242    squab     square    squash    stable    staff     stage
 243    stand     staple    star      start     state     status
 244    stay      steak     steel     step      stern     stew
 245    stick     still     stock     stone     stop      store
 246    storm     story     strain    street    stress    strike
 247    string    stroke    strong    studio    study     stuff
 248    style     sugar     suit      sulfur    summer    sun
 249    sunny     sunset    super     superb    surf      survey
 250    sweet     swim      swing     switch    symbol    system
 251    table     tackle    tail      tale      talk      tan
 252    tank      tap       tape      target    task      taste
 253    tau       tea       teach     teal      team      tear
 254    tell      ten       tender    tennis    tent      term
 255    test      tetra     text      thanks    theme     theory
 256    thing     think     thread    throat    thumb     ticket
 257    tidy      tie       tiger     till      time      timely
 258    tin       tip       title     toast     today     toe
 259    tomato    tone      tongue    tool      tooth     top
 260    topic     total     touch     tough     tour      towel
 261    tower     town      track     trade     train     trash
 262    travel    tray      treat     tree      trick     trip
 263    trout     trowel    truck     trupet    trust     truth
 264    try       tube      tuna      tune      turf      turkey
 265    turn      turnip    tutor     tux       tweet     twist
 266    two       type      union     unique    unit      upbeat
 267    upper     use       useful    user      usual     valley
 268    value     van       vase      vast      veil      vein
 269    velvet    verse     very      vessel    vest      video
 270    view      violet    visit     visual    vivid     voice
 271    volume    vowel     voyage    waffle    wait      wake
 272    walk      wall      warm      warmth    wasabi    wash
 273    watch     water     wave      wax       way       wealth
 274    wear      web       wedge     week      weekly    weight
 275    west      whale     what      wheat     wheel     when
 276    where     while     who       whole     why       will
 277    win       wind      window    wing      winner    winter
 278    wire      wish      witty     wolf      wonder    wood
 279    wool      woolly    word      work      worker    world
 280    worry     worth     worthy    wrap      wrench    wrist
 281    writer    xenon     yak       yam       yard      yarrow
 282    year      yearly    yellow    yew       yogurt    young
 283    youth     zebra     zephyr    zinc      zone      zoo
 284 }
 285 set nwordlist [llength $wordlist]
 286
 287 proc random_char {} {
 288   return [string index \
 289              "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" \
 290              [expr {int(rand()*52)}]]
 291 }
 292 proc random_label {} {
 293   set label [random_char]
 294   while {rand()>0.8} {
 295     append label [random_char]
 296   }
 297   if {rand()>0.9} {append label -}
 298   append label [format %d [expr {int(rand()*100)}]]
 299   return $label
 300 }
 301 proc random_numeric {} {
 302   set n [expr {(rand()*2-1.0)*1e6}]
 303   switch [expr {int(rand()*6)}] {
 304     0  {set format %.3f}
 305     1  {set format %.6E}
 306     2  {set format %.4e}
 307     default  {set format %g}
 308   }
 309   return [format $format $n]
 310 }
 311
 312
 313 proc random_json {limit indent} {
 314   global nwordlist wordlist
 315   set res {}
 316   if {$indent==0 || ($limit>0 && rand()>0.5)} {
 317     incr limit -1
 318     incr indent 2
 319     set n [expr {int(rand()*5)+1}]
 320     if {$n==5} {incr n [expr {int(rand()*10)}]}
 321     if {rand()>0.5} {
 322       set res \173\n
 323       for {set i 0} {$i<$n} {incr i} {
 324         append res [string repeat { } $indent]
 325         if {rand()>0.8} {
 326           if {rand()>0.5} {
 327             set sep ":\n   [string repeat { } $indent]"
 328           } else {
 329             set sep " : "
 330           }
 331         } else {
 332           set sep :
 333         }
 334         append res \"[random_label]\"$sep[random_json $limit $indent]
 335         if {$i<$n-1} {append res ,}
 336         append res \n
 337       }
 338       incr indent -2
 339       append res [string repeat { } $indent]
 340       append res \175
 341       return $res
 342     } else {
 343       set res \[\n
 344       for {set i 0} {$i<$n} {incr i} {
 345         append res [string repeat { } $indent]
 346         append res [random_json $limit $indent]
 347         if {$i<$n-1} {append res ,}
 348         append res \n
 349       }
 350       incr indent -2
 351       append res [string repeat { } $indent]
 352       append res \]
 353       return $res
 354     }
 355   } elseif {rand()>0.9} {
 356     if {rand()>0.7} {return "true"}
 357     if {rand()>0.5} {return "false"}
 358     return "null"
 359   } elseif {rand()>0.5} {
 360     return [random_numeric]
 361   } else {
 362     set res \"
 363     set n [expr {int(rand()*4)+1}]
 364     if {$n>=4} {set n [expr {$n+int(rand()*6)}]}
 365     for {set i 0} {$i<$n} {incr i} {
 366       if {rand()<0.05} {
 367         set w [random_numeric]
 368       } else {
 369         set k [expr {int(rand()*$nwordlist)}]
 370         set w [lindex $wordlist $k]
 371       }
 372       if {rand()<0.07} {
 373          set w \\\"$w\\\"
 374       }
 375       if {$i<$n-1} {
 376         switch [expr {int(rand()*9)}] {
 377           0       {set sp {, }}
 378           1       {set sp "\\n "}
 379           2       {set sp "-"}
 380           default {set sp { }}
 381         }
 382         append res $w$sp
 383       } else {
 384         append res $w
 385         if {rand()<0.2} {append res .}
 386       }
 387     }
 388     return $res\"
 389   }
 390 }
 391
 392 puts "CREATE TABLE IF NOT EXISTS data1(x JSON);"
 393 puts "BEGIN;"
 394 set sz 0
 395 for {set i 0} {$sz<100000000} {incr i} {
 396   set j [random_json 7 0]
 397   incr sz [string length $j]
 398   puts "INSERT INTO data1(x) VALUES('$j');"
 399 }
 400 puts "COMMIT;"
 401 puts "SELECT sum(length(x)) FROM data1;"