audioProcessing.js

   1 /*
   2  * audioProcessing.js
   3  *
   4  * Copyright (C) 2016 R.J.J.H. van Son (r.j.j.h.vanson@gmail.com)
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version 2
   9  * of the License, or (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You can find a copy of the GNU General Public License at
  17  * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
  18  */
  19
  20 var mimeTypes = {
  21         "wav": "audio/wav",
  22         "mp3": "audio/mpeg",
  23         "flac": "audio/flac",
  24         "ogg": "audio/ogg",
  25         "spx": "audio/ogg",
  26         "aif": "audio/aifc",
  27         "tsv": "text/tsv",
  28         "csv": "text/csv"
  29 };
  30 // Global variables
  31 var recordedBlob, recordedBlobURL;
  32 var recordedArray;
  33 var recordedDuration;
  34 var recordedPitchTier;
  35 var windowStart = windowEnd = 0;
  36 var recordedSampleRate = 0;
  37 var currentAudioWindow = undefined;
  38 var audioDatabaseName = "Audio";
  39 var examplesDatabaseName = "Examples";
  40
  41 var clearRecording = function () {
  42         recordedBlob = undefined;
  43         recordedBlobURL = undefined;
  44         recordedArray = undefined;
  45         currentAudioWindow = undefined;
  46         recordedSampleRate = recordedDuration = undefined;
  47         recordedPitchTier = undefined;
  48 };
  49
  50 /*
  51  *
  52  * Audio processing code
  53  *
  54  */
  55
  56 // Only initialize ONCE
  57 // On Safari/iOS use: webkitAudioContext
  58 // webkitAudioContext must be "resume()"ed with a tap (Record/Play?)
  59 var AudioContextObject = window.AudioContext || window.webkitAudioContext;
  60 var audioContext = new AudioContextObject();
  61
  62 // Decode the audio blob
  63 var audioProcessing_decodedArray;
  64 function processAudio (blob) {
  65         var audioReader = new FileReader();
  66         audioReader.onload = function(){
  67                 audioContext.resume();  // Is this necessary for Safari?
  68                 var arrayBuffer = audioReader.result;
  69                 audioContext.decodeAudioData(arrayBuffer, decodedDone, function(e){ console.log("Error with decoding audio data");if(e){console.log(e)}});
  70         };
  71         audioReader.readAsArrayBuffer(blob);
  72 };
  73
  74 // You need a function "processRecordedSound ()"
  75 // Set some switches to prevent stored data are reused
  76 sessionStorage["recorded"] = "false";
  77 var retrievedData = false;
  78 function decodedDone(decoded) {
  79         var typedArray = new Float32Array(decoded.length);
  80         typedArray = decoded.getChannelData(0);
  81         var currentArray = typedArray;
  82         recordedSampleRate = decoded.sampleRate;
  83         recordedDuration = decoded.duration;
  84         var length = decoded.length;
  85
  86         // Process and draw audio
  87         recordedArray = cut_silent_margins (currentArray, recordedSampleRate);
  88         currentAudioWindow = recordedArray;
  89         recordedDuration = recordedArray.length / recordedSampleRate;
  90         windowStart = 0;
  91         windowEnd = recordedDuration;
  92
  93         // make sure this function is defined!!!
  94         if (!retrievedData) sessionStorage["recorded"] = "true";
  95         processRecordedSound ();
  96         sessionStorage["recorded"] = "false";
  97         // Note this one should be switched AFTER processRecordedSound has been called.
  98         retrievedData = false;
  99 };
 100
 101 function play_soundArray (soundArray, sampleRate, start, end) {
 102         var startSample = start > 0 ? Math.floor(start * sampleRate) : 0;
 103         var endSample = end > 0 ? Math.ceil(end * sampleRate) : soundArray.length;
 104         if (startSample > soundArray.length || endSample > soundArray.length) {
 105                 startSample = 0;
 106                 endSample = soundArray.length;
 107         };
 108         var soundBuffer = audioContext.createBuffer(1, endSample - startSample, sampleRate);
 109         var buffer = soundBuffer.getChannelData(0);
 110         for (var i = 0; i < (endSample - startSample); i++) {
 111              buffer[i] = soundArray[startSample + i];
 112         };
 113
 114         // Get an AudioBufferSourceNode.
 115         // This is the AudioNode to use when we want to play an AudioBuffer
 116         var source = audioContext.createBufferSource();
 117         // set the buffer in the AudioBufferSourceNode
 118         source.buffer = soundBuffer;
 119         // connect the AudioBufferSourceNode to the
 120         // destination so we can hear the sound
 121         source.connect(audioContext.destination);
 122         // start the source playing
 123         source.start();
 124 };
 125
 126 /*
 127  * Convert typed mono array to WAV blob
 128  *
 129  */
 130 function writeUTFBytes(view, offset, string){
 131   var lng = string.length;
 132   for (var i = 0; i < lng; i++){
 133     view.setUint8(offset + i, string.charCodeAt(i));
 134   }
 135 }
 136
 137 // Write selection [start, end] to a WAV blob
 138 function arrayToBlob (array, start, end, sampleRate) {
 139         if(!array) return;
 140         if(end <= 0) end = array.length;
 141         var window = array.slice(start*sampleRate, end*sampleRate);
 142
 143         // create the buffer and view to create the .WAV file
 144         var buffer = new ArrayBuffer(44 + window.length * 2);
 145         var view = new DataView(buffer);
 146
 147         // write the WAV container, check spec at: https://ccrma.stanford.edu/courses/422/projects/WaveFormat/
 148         // RIFF chunk descriptor
 149         writeUTFBytes(view, 0, 'RIFF');
 150         view.setUint32(4, 44 + window.length * 2, true);
 151         writeUTFBytes(view, 8, 'WAVE');
 152         // FMT sub-chunk
 153         writeUTFBytes(view, 12, 'fmt ');
 154         view.setUint32(16, 16, true);
 155         view.setUint16(20, 1, true);
 156         // mono (2 channels)
 157         view.setUint16(22, 1, true);
 158         view.setUint32(24, sampleRate, true);
 159         view.setUint32(28, sampleRate * 2, true);
 160         view.setUint16(32, 2, true);
 161         view.setUint16(34, 16, true);
 162         // data sub-chunk
 163         writeUTFBytes(view, 36, 'data');
 164         view.setUint32(40, window.length * 2, true);
 165
 166         // write the PCM samples
 167         var lng = window.length;
 168         var index = 44;
 169         var volume = 1;
 170         for (var i = 0; i < lng; i++){
 171             view.setInt16(index, window[i] * (0x7FFF * volume), true);
 172             index += 2;
 173         }
 174
 175         // our final binary blob that we can hand off
 176         var blob = new Blob ( [ view ], { type : 'audio/wav' } );
 177
 178         return blob;
 179 };
 180
 181 // Set up window
 182 function setupGaussWindow (sampleRate, fMin) {
 183         var lagMax = (fMin > 0) ? 1/fMin : 1/75;
 184         var windowDuration = lagMax * 6;
 185         var windowSigma = 1/6;
 186         var window = new Float32Array(windowDuration * sampleRate);
 187         var windowCenter = (windowDuration * sampleRate -1) / 2;
 188         for (var i = 0; i < windowDuration * sampleRate; ++i) {
 189                 var exponent = -0.5 * Math.pow( (i - windowCenter)/(windowSigma * windowCenter), 2);
 190                 window [i] = Math.exp(exponent);
 191         };
 192
 193         return window;
 194 };
 195
 196 function getWindowRMS (window) {
 197         var windowSumSq = 0;
 198         var windowRMS = 0;
 199         if (window && window.length > 0) {
 200                 for (var i = 0; i < window.length; ++i) {
 201                         windowSumSq += window [i]*window [i];
 202                 };
 203                 windowRMS = Math.sqrt(windowSumSq/window.length);
 204         };
 205
 206         return windowRMS;
 207 };
 208
 209 // Cut off the silent margins
 210 // ISSUE: After the first recording, there is a piece at the start missing.
 211 // This is now cut off
 212 function cut_silent_margins (typedArray, sampleRate) {
 213         // Find part with sound
 214         var silentMargin = 0.1;
 215         // Silence thresshold is -20 dB
 216         var thressHoldDb = 25;
 217         // Stepsize
 218         var dT = 0.01;
 219         var soundLength = typedArray.length;
 220
 221         // There is sometimes (often) a delay before recording is started
 222         var firstNonZero = 0;
 223         while (firstNonZero < typedArray.length && (isNaN(typedArray[firstNonZero]) || typedArray[firstNonZero] == 0)) {
 224                 ++firstNonZero
 225         };
 226
 227         // Calculation intensity
 228         var currentIntensity = calculate_Intensity (typedArray, sampleRate, 75, 600, 0.01);
 229         var maxInt = Math.max.apply(Math, currentIntensity);
 230         var silenceThresshold = maxInt - thressHoldDb;
 231
 232         var firstFrame = 0;
 233         while (firstFrame < currentIntensity.length && currentIntensity[firstFrame] < silenceThresshold) {
 234                 ++firstFrame;
 235         };
 236
 237         var lastFrame = currentIntensity.length - 1;
 238         while (lastFrame > 0 && currentIntensity[lastFrame] < silenceThresshold) {
 239                 --lastFrame;
 240         };
 241
 242         if ((firstFrame >= currentIntensity.length - silentMargin/dT) || (lastFrame <= silentMargin/dT) || lastFrame <= firstFrame) {
 243                 firstFrame = 0;
 244                 lastFrame = currentIntensity.length - 1;
 245         };
 246
 247         // Convert frames to samples
 248         var firstSample = (firstFrame * dT - silentMargin) * sampleRate;
 249         var lastSample = ((lastFrame + 1) * dT + silentMargin) * sampleRate;
 250         if (firstSample < 0) firstSample = 0;
 251         // Remove non-recorded part froms tart
 252         if (firstSample < firstNonZero) firstSample = firstNonZero;
 253         if (lastSample >= soundLength) lastSample = soundLength - 1;
 254
 255         var newLength = Math.ceil(lastSample - firstSample);
 256         var soundArray = new Float32Array(newLength);
 257         for (var i = 0; i < newLength; ++i) {
 258                 // Also, get rid of NaN's
 259                 soundArray [i] = !isNaN(typedArray[firstSample + i]) ? typedArray[firstSample + i] : 0;
 260         };
 261         return soundArray;
 262 };
 263
 264 // Calculate autocorrelation in a window (array!!!) around time
 265 // You should divide the result by the autocorrelation of the window!!!
 266 //
 267 // David Weenink: De autocorrelatie moet gecontroleerd worden.
 268 // Ik denk niet dat hij veel sneller kan.
 269 function autocorrelation (sound, sampleRate, time, window) {
 270         // Calculate FFT
 271         // This is stil just the power in dB.
 272         var soundLength = sound.length;
 273         var windowLength = (window) ? window.length : soundLength;
 274         var FFT_N = Math.pow(2, Math.ceil(Math.log2(windowLength))) * 2;
 275         var input = new Float32Array(FFT_N * 2);
 276         var output = new Float32Array(FFT_N * 2);
 277         // The window can get outside of the sound itself
 278         var offset = Math.floor(time * sampleRate - Math.ceil(windowLength/2));
 279         if (window) {
 280                 for (var i = 0; i < FFT_N; ++i) {
 281                         input [2*i] = (i < windowLength && (offset + i) > 0 && (offset + i) < soundLength) ? sound [offset + i] * window [i] : 0;
 282                         input [2*i + 1] = 0;
 283                 };
 284         } else {
 285                 for (var i = 0; i < FFT_N; ++i) {
 286                         input [2*i] = (i < windowLength && (offset + i) > 0 && (offset + i) < soundLength) ? sound [offset + i] : 0;
 287                         input [2*i + 1] = 0;
 288                 };
 289         };
 290         var fft = new FFT.complex(FFT_N, false);
 291         var ifft = new FFT.complex(FFT_N, true);
 292         fft.simple(output, input, 1)
 293
 294         // Calculate H * H(cj)
 295         // Scale per frequency
 296         for(var i = 0; i < FFT_N; ++ i) {
 297                 var squareValue = (output[2*i]*output[2*i] + output[2*i+1]*output[2*i+1]);
 298                 input[2*i] = squareValue;
 299                 input[2*i+1] = 0;
 300                 output[2*i] = 0;
 301                 output[2*i+1] = 0;
 302         };
 303
 304         ifft.simple(output, input, 1);
 305
 306         var autocorr = new Float32Array(FFT_N);
 307         for(var i = 0; i < FFT_N; ++ i) {
 308                  autocorr[i] = output[2*i] / windowLength;
 309         };
 310
 311         return autocorr;
 312 };
 313
 314 /*
 315  *
 316  * David Weenink, here are the pitch routines:
 317  * - Autocorrelation peak picking for candidates
 318  * - Pitch tracking for selecting the best candidate
 319  *
 320  */
 321
 322 // Take a spectrum and return a list with peaks
 323 //
 324 // David Weenink: De peak picker moet geoptimaliseerd worden.
 325 // liefst met een "echt" peak picking algoritme. Dit is een simplistische
 326 // 5 punts differentiatie (4 verschillen) die halverwege door nul gaan
 327 // dwz, twee verschillen > 0 gevolgd door twee verschillen < 0.
 328 // Ik gebruik nu een 3 punts parabool om het maximum te vinden,
 329 // maar ik vermoed dat een 5 punts kwadratische fit misschien beter is?
 330 function autocorrelationPeakPicker (autocorr, sampleRate, fMin, fMax) {
 331         var thressHold = 0.001;
 332         var resultArray = [];
 333         // Find the pitch candidates
 334         var lagMin = (fMax > 0) ? 1/fMax : 1/600;
 335         var lagMax = (fMin > 0) ? 1/fMin : 1/60;
 336         var startLag = Math.floor(lagMin * sampleRate);
 337         var endLag = Math.ceil(lagMax * sampleRate);
 338         var bestLag = 0;
 339         var bestAmp = -100;
 340         // 5 point differentiation
 341         var diffAmp = [];
 342         var halfLength = 2;
 343         for (var i=-halfLength; i<halfLength; ++i){
 344                 diffAmp[i+halfLength] = autocorr [startLag+i+1] - autocorr [startLag+i];
 345         };
 346         var peaks = [];
 347         for (var j = startLag; j <= endLag; ++j) {
 348                 for (var i=0;  i< diffAmp.length-1; ++i){
 349                         diffAmp[i] = diffAmp[i+1];
 350                 };
 351                 diffAmp[diffAmp.length-1] = autocorr [j+halfLength] - autocorr [j+halfLength-1];
 352                 // A peak is found
 353                 if(autocorr [j] > thressHold && diffAmp[0] > 0 && diffAmp[1] > 0 && diffAmp[2] <= 0 && diffAmp[3] <= 0) {
 354                         // 4 point linear regression with zero-crossing centered around 0
 355                         var linReg = linearRegression(diffAmp, [-1.5, -0.5, 0.5, 1.5]);
 356                         var offset =  - linReg.intercept/linReg.slope;
 357                         var zeroCrossing = j + offset;
 358                         // Get the maximum using a 3-point quadratic interpolation.
 359                         // I would prefer a 5-point least RMSE quadratic fit
 360                         var pC = threePointParabola ([autocorr [j-1], autocorr [j], autocorr [j+1]], [-1, 0, 1]);
 361                         var maxValue = pC.a * offset*offset + pC.b*offset + pC.c;
 362                         peaks.push({x: sampleRate/zeroCrossing, y: maxValue});
 363                 };
 364         };
 365         // Switch to low to high ordering
 366         peaks.reverse();
 367         return peaks;
 368 };
 369
 370 // Return pitch array
 371 // Return a list of points with {t, candidates} "pairs"
 372 function calculate_Pitch (sound, sampleRate, fMin, fMax, dT) {
 373         var duration = sound.length / sampleRate;
 374         var pitchArray = [];
 375
 376         // Set up window and calculate Autocorrelation of window
 377         var windowDuration = (fMin > 0) ? 1/fMin : 1/60;
 378         windowDuration *= 6;
 379         var window = setupGaussWindow (sampleRate, fMin);
 380         var windowRMS = getWindowRMS (window);
 381
 382         // calculate the autocorrelation of the window
 383         var windowAutocorr = autocorrelation (window, sampleRate, windowDuration / 2, 0);
 384
 385         // Step through the sound
 386         for (var t = 0; t < duration; t += dT) {
 387                 var autocorr = autocorrelation (sound, sampleRate, t, window);
 388                 for (var i = 0; i < autocorr.length; ++i) {
 389                         autocorr [i] /= (windowAutocorr [i]) ? (windowAutocorr [i] * windowRMS) : 0;
 390                 };
 391
 392                 // Find the pitch candidates
 393                 var pitchCandidates = autocorrelationPeakPicker (autocorr, sampleRate, fMin, fMax);
 394                 // unvoiced
 395                 if(pitchCandidates.length == 0)pitchCandidates.push({x: 0, y: 0});
 396                 pitchArray.push({x: t, values: pitchCandidates});
 397         };
 398         return pitchArray;
 399 };
 400
 401 // PitchTier definition
 402 function Tier () {
 403         // data
 404         this.xmin = Infinity;
 405         this.xmax = -Infinity;
 406         this.valuemin = Infinity;
 407         this.valuemax = -Infinity;
 408         this.dT = undefined;
 409         this.size = undefined;
 410         this.time = [];
 411         this.values = [];
 412
 413         // access functions
 414         this.valueSeries = function () {return this.values; };
 415         this.timeSeries = function () {return this.time; };
 416         this.item = function (i) {
 417                 return {x: this.time [i], value: this.values [i]};
 418         };
 419         this.writeItem = function (i, item) {
 420                 if ( i < this.time.length ) {
 421                         this.time [i] = item.x;
 422                         if(item.x < this.xmin)this.xmin = item.x;
 423                         if(item.x > this.xmax)this.xmax = item.x;
 424                         this.values [i] = item.value;
 425                         if(item.value < this.valuemin) this.valuemin = item.value;
 426                         if(item.value > this.valuemax) this.valuemax = item.value;
 427                         return i;
 428                 } else {
 429                         console.log("Item "+i+" does not exist");
 430                         return false;
 431                 }
 432         };
 433         this.pushItem = function (item) {
 434                 this.time.push(item.x);
 435                 if(item.x < this.xmin)this.xmin = item.x;
 436                 if(item.x > this.xmax)this.xmax = item.x;
 437                 this.values.push(item.value);
 438                 if(item.value < this.valuemin) this.valuemin = item.value;
 439                 if(item.value > this.valuemax) this.valuemax = item.value;
 440                 this.size = this.time.length;
 441                 return this.size;
 442         };
 443 };
 444
 445 // Pitch tracking and candidate selection
 446 function toPitchTierPitchJS (sound, sampleRate, fMin, fMax, dT) {
 447         var duration = sampleRate > 0 ? sound.length / sampleRate : 0;
 448
 449
 450         // Set up window and calculate Autocorrelation of window
 451         var windowDuration = (fMin > 0) ? 1/fMin : 1/60;
 452         windowDuration *= 2;
 453         windowLength = sampleRate * windowDuration
 454
 455         var pitchTier = new Tier();
 456         pitchTier.xmin = 0;
 457         pitchTier.dT = dT;
 458
 459         /* Create a new pitch detector */
 460         var pitch = new PitchAnalyzer(sampleRate);
 461
 462         // Step through the sound
 463         for (var t = 0; t < duration; t += dT) {
 464                 var startSample = Math.floor(sampleRate * (t + dT - windowDuration)/2);
 465                 var endSample = startSample + windowLength;
 466                 var audioBuffer = new Float32Array(windowLength);
 467                 for(var i = 0; i < windowLength; ++i){
 468                         if(startSample + i < 0 || startSample + i >= sound.length){
 469                                 audioBuffer[i] = 0;
 470                         } else {
 471                                 audioBuffer[i] = sound [startSample + i];
 472                         };
 473                 };
 474
 475                 /* Copy samples to the internal buffer */
 476                 pitch.input(audioBuffer);
 477                 /* Process the current input in the internal buffer */
 478                 pitch.process();
 479
 480                 // Find the pitch candidates
 481                 var tone = pitch.findTone();
 482                 var pitchCandidates = [];
 483                 var F0 = 0;
 484                 if (tone === null) {
 485                         F0 = 0;
 486                 } else {
 487                         F0 = tone.freq;
 488                 };
 489                 pitchTier.pushItem ({x: t, value: F0});
 490
 491         console.log("toPitchTier: "+t+" - "+F0+" - "+windowDuration+" - "+audioBuffer.length);
 492
 493         };
 494
 495         return pitchTier;
 496 }
 497
 498 function toPitchTier (sound, sampleRate, fMin, fMax, dT) {
 499         var pitchArray = calculate_Pitch (sound, sampleRate, fMin, fMax, dT);
 500         var duration = sampleRate > 0 ? sound.length / sampleRate : 0;
 501         var points = [];
 502         var timeSeries = [];
 503         var valueSeries = [];
 504
 505         // Select the best pitch candidates using tracking etc.
 506         var bestTrack = viterbi (pitchArray);
 507
 508         var pitchTier = new Tier();
 509         pitchTier.xmin = 0;
 510         pitchTier.dT = dT;
 511         for (var i=0; i < pitchArray.length; ++ i) {
 512                 var bestValue = pitchArray[i].values[bestTrack[i]].x;
 513                 pitchTier.pushItem ({x: pitchArray [i].x, value: bestValue});
 514         };
 515
 516         return pitchTier;
 517 }
 518
 519 // Viterbi pitch tracking
 520 // Takes an array of pitch candidates and returns a list of
 521 // the best candidate for each frame
 522 //
 523 // David Weenink:
 524 // Deze viterbi functie moet versneld en geoptimaliseerd worden.
 525 // De kostenfunctie is nu willekeurig gekozen.
 526 function viterbi (pitchArray) {
 527         var costsList = [];
 528         var backpointerList = [];
 529         for (var i=0; i < pitchArray.length; ++i) {
 530                 var sumWeights = 0;
 531                 var pitchCandidates = pitchArray[i].values;
 532                 var prevCandidates = pitchArray[i-1] && pitchArray[i-1].values ? pitchArray[i-1].values : [{x:0, y:0}];
 533                 var previousCosts = i>0 ? costsList[i-1] : [0];
 534                 var candidateCosts = [];
 535                 var candidateBackpointers = [];
 536                 // Initialize variables
 537                 for (var j=0; j <pitchCandidates.length; ++j) {
 538                         sumWeights += pitchCandidates[j].y;
 539                         candidateCosts.push(0);
 540                         candidateBackpointers.push(-1);
 541                 };
 542                 // Find best continuation for each candidate
 543                 for (var j=0; j <pitchCandidates.length; ++j) {
 544                         weight = sumWeights > 0 ? 1 - (pitchCandidates[j].y / sumWeights) : 1;
 545                         // Initialize to handle voiceless previous frame
 546                         minCost = Infinity;
 547                         bestBackpointer = 0;
 548                         // The cost function is distance^2 / relative height of peak
 549                         for (var k=0; k<prevCandidates.length; ++k) {
 550                                 // Previous costs of this precurser
 551                                 var newCost = previousCosts[k];
 552                                 // Cost added
 553                                 if(prevCandidates[k].x > 0 && pitchCandidates[j].x > 0) {
 554                                         // Squared difference relative to average between points.
 555                                         newCost += weight * Math.pow(2*(prevCandidates[k].x - pitchCandidates[j].x)/(prevCandidates[k].x + pitchCandidates[j].x), 2);
 556                                 } else {
 557                                         newCost += weight;
 558                                 };
 559                                 if(newCost < minCost) {
 560                                         minCost = newCost;
 561                                         bestBackpointer = k;
 562                                 };
 563                         };
 564                         candidateCosts[j] = minCost;
 565                         candidateBackpointers[j] = bestBackpointer;
 566                 };
 567                 costsList.push(candidateCosts);
 568                 backpointerList.push(candidateBackpointers);
 569         };
 570         // Trace back best pitch track
 571         var lastItem = costsList.length - 1;
 572         var costsCandidates = costsList[lastItem];
 573         var minCost = costsCandidates[0];
 574         var endPoint = 0;
 575         // Get the end point with the lowest cost
 576         for (j=1; j<costsCandidates.length; ++j) {
 577                 if(costsCandidates[j] < minCost) {
 578                         minCost = costsCandidates[j];
 579                         endPoint = j;
 580                 };
 581         };
 582         var resultTrack = [endPoint];
 583         var lastBackpointer = endPoint;
 584         for (var i=backpointerList.length - 1; i>0; --i) {
 585                 lastBackpointer = backpointerList[i][lastBackpointer];
 586                 resultTrack.push(lastBackpointer);
 587         };
 588
 589         // The result track is reversed
 590         return resultTrack.reverse();
 591 };
 592
 593
 594 // DTW between two pitchTiers
 595 //
 596 // David Weenink:
 597 // Dit is de DTW functie. Die moet geoptimaliseerd en versneld worden
 598 // De kosten (inclusief voor voicing sprongen) zijn nu willekeurig gekozen
 599 function toDTW (pitchTier1, pitchTier2) {
 600         var horCost = 2;
 601         var verCost = 2;
 602         var diagonalCost = 1;
 603         var voicingCost = 100;
 604         var dtw = {distance: 0, path: [], matrix: undefined};
 605         var costMatrix = [];
 606         var backpointerMatrix = [];
 607
 608         var pitch1 = pitchTier1.valueSeries();
 609         var pitch2 = pitchTier2.valueSeries();
 610
 611         // Initialize the cost and backpointer matrices
 612         for (var i=0; i<pitch1.length; ++i) {
 613                 var costColumn = [];
 614                 var bpColumn = [];
 615                 for (var j=0; j<pitch2.length; ++j) {
 616                         costColumn.push(0);
 617                         bpColumn.push(0);
 618                 };
 619                 costMatrix.push(costColumn);
 620                 backpointerMatrix.push(bpColumn);
 621         };
 622
 623         // Fill the cost and backpointer matrices
 624         // i = pitch1
 625         // j = pitch2
 626         for (var i=0; i<pitch1.length; ++i) {
 627                 for (var j=0; j<pitch2.length; ++j) {
 628                         var newCost = 0;
 629                         // backpointer [i-1][j]=-1, [i][j-1]=1, [i-1][j-1]=0
 630                         var backpointer = 0;
 631                         if(j > 0) {
 632                                 if(i > 0) {
 633                                         var newHorCost = costMatrix[i-1][j] + horCost * dtwCostFunction(pitch1[i-1], pitch2[j], voicingCost);
 634                                         var newVerCost = costMatrix[i][j-1] + verCost * dtwCostFunction(pitch1[i], pitch2[j-1], voicingCost);
 635                                         var newDiaCost = costMatrix[i-1][j-1] + diagonalCost * dtwCostFunction(pitch1[i-1], pitch2[j-1], voicingCost);
 636                                         var minCost = Math.min(newHorCost, newVerCost, newDiaCost);
 637                                         if(newDiaCost == minCost) {
 638                                                 newCost = newDiaCost;
 639                                                 backpointer = 0;
 640                                         } else if (newHorCost == minCost) {
 641                                                 newCost = newHorCost;
 642                                                 backpointer = -1;
 643                                         } else if (newVerCost == minCost) {
 644                                                 newCost = newVerCost;
 645                                                 backpointer = 1;
 646                                         } else {
 647                                                 console.log("ERROR: No DTW cost");
 648                                         };
 649                                 } else {
 650                                         // i == 0
 651                                         newCost = costMatrix[0][j-1];
 652                                         newCost += horCost * dtwCostFunction(pitch1[0], pitch2[j], voicingCost);
 653                                         backpointer = 1;
 654                                 };
 655                         } else if (i > 0) {
 656                                 // j == 0
 657                                 newCost = costMatrix[i-1][0];
 658                                 newCost += horCost * dtwCostFunction(pitch1[i], pitch2[0], voicingCost);
 659                                 backpointer = -1;
 660                         } else {
 661                                 // i == 0 and j == 0, corner case
 662                                 newCost = 0;
 663                                 newCost += horCost * dtwCostFunction(pitch1[0], pitch2[0], voicingCost);
 664                                 backpointer = 0;
 665                         };
 666                         costMatrix[i][j] = newCost;
 667                         backpointerMatrix[i][j] = backpointer;
 668                 };
 669         };
 670         dtw.matrix = costMatrix;
 671
 672         // Start at upper right corner and trace back the path
 673         var i = pitch1.length-1;
 674         var j = pitch2.length-1;
 675         dtw.distance = costMatrix[i][j];
 676
 677         var path = [];
 678         while (i > 0 || j > 0) {
 679                 path.push({i: i, j: j});
 680                 var step = backpointerMatrix[i][j];
 681                 if(step == 0) {
 682                         i -= 1;
 683                         j -= 1;
 684                 } else if (step == -1) {
 685                         i -= 1;
 686                 } else if (step == 1) {
 687                         j -= 1;
 688                 } else {
 689                         console.log("ERROR: wrong step");
 690                         return [];
 691                 };
 692         };
 693         path.push({i: 0, j: 0});
 694         dtw.path = path.reverse();
 695
 696         return dtw;
 697 };
 698
 699
 700 // David Weenink:
 701 // Dit is de DTW kostenfunctie. hier kan een betere ingevoerd worden.
 702 // Deze is nu willekeurig gekozen
 703 function dtwCostFunction (value1, value2, voicingCost) {
 704         if(value1 > 0 && value2 > 0)return Math.pow(value1-value2, 2);
 705         if(value1 == 0 && value2 == 0)return 0;
 706         if(value1 == 0 || value2 == 0)return Math.pow(voicingCost, 2);
 707 };
 708
 709 // Calculate the (RMS) power in a time window
 710 function getPower (sound, sampleRate, time, window) {
 711         var soundLength = sound.length;
 712         var duration = sound.length / sampleRate;
 713         var windowLength = (window) ? window.length : soundLength;
 714         var sumSquare = 0;
 715         var windowSUM = 0;
 716         // The window can get outside of the sound itself
 717         var offset = Math.floor(time * sampleRate - Math.ceil(windowLength/2));
 718         if (window) {
 719                 for (var i = 0; i < windowLength; ++i) {
 720                         var value = ((offset + i) > 0 && (offset + i) < soundLength) ? sound [offset + i] * window [i] : 0;
 721                         sumSquare += value*value;
 722                         windowSUM += window [i]*window [i];
 723                 };
 724         } else {
 725                 for (var i = 0; i < windowLength; ++i) {
 726                         var value = ((offset + i) > 0 && (offset + i) < soundLength) ? sound [offset + i] : 0;
 727                         sumSquare += value*value;
 728                         windowSUM += 1;
 729                 };
 730         };
 731
 732         if (windowSUM <= 0) windowSUM = 1;
 733         var power = sumSquare / windowSUM;
 734         return power;
 735 };
 736
 737 function calculate_Intensity (sound, sampleRate, fMin, fMax, dT) {
 738         var duration = sound.length / sampleRate;
 739         var bitSize = 16;
 740         var maxPower = Math.round(20*Math.log10(Math.pow(2,bitSize-1)));
 741         var quantNoise = Math.round(20*Math.log10(0.5));
 742         var dynamicRange = maxPower - quantNoise;
 743         var lagMin = (fMax > 0) ? 1/fMax : 1/600;
 744         var lagMax = (fMin > 0) ? 1/fMin : 1/60;
 745         var intensity = new Float32Array(Math.floor(duration / dT));
 746
 747         // Set up window
 748         var windowDuration = lagMax * 6;
 749         var window = setupGaussWindow (sampleRate, fMin);
 750
 751         // Step through the sound
 752         var i = 0;
 753         for (var t = 0; t < duration; t += dT) {
 754                 var power = getPower (sound, sampleRate, t, window);
 755                 var powerdB = (power > 0) ? dynamicRange + Math.log10(power) * 10 : 0;
 756                 if (i < intensity.length) intensity [i] = powerdB;
 757                 ++i;
 758         };
 759
 760         return intensity;
 761 };
 762
 763
 764 // load the sound from a URL
 765 function load_audio(url) {
 766         var request = new XMLHttpRequest();
 767         request.open('GET', url, true);
 768         request.responseType = 'arraybuffer';
 769         // When loaded decode the data and store the audio buffer in memory
 770         request.onload = function() {
 771                 processAudio (request.response);
 772         }
 773         request.send();
 774 }
 775
 776 /*
 777  *
 778  * Use: var perc = get_percentiles (points, function (a, b) { return a.value-b.value;}, function(a) { return a.value <= 0;}, [0.05, 0.50, 0.95]);
 779  *
 780  */
 781 function get_percentiles (points, compare, remove, percentiles) {
 782         var sortList = points.slice();
 783         var ax;
 784         while (sortList.length > 0 && (ax = sortList.indexOf(undefined)) !== -1) {
 785             sortList.splice(ax, 1);
 786     }
 787         var result = [];
 788         sortList.sort(compare);
 789         var sortListLength = sortList.length
 790         for (var i = sortListLength-1; i >= 0; --i) {
 791                 if (remove(sortList[i])) {
 792                         sortList.splice(i, 1);
 793                 };
 794         };
 795         for (var i = 0; i < percentiles.length; ++i) {
 796                 var perc = percentiles[i];
 797                 if (perc > 1) perc /= 100;
 798                 var newPercentile = {value: undefined, percentile: 0};
 799                 var bin = Math.ceil(perc * sortList.length) - 1;
 800                 bin = bin < 0 ? 0 : (bin >= sortList.length ? sortList.length : bin);
 801                 newPercentile.value = sortList[bin];
 802                 newPercentile.percentile = percentiles[i];
 803                 result.push(newPercentile)
 804         };
 805         return result;
 806 };
 807
 808 // return the minim and maximum and their times
 809 function get_time_of_minmax (tier) {
 810         var min = Infinity;
 811         var max = -Infinity;
 812         var tmin = tmax = 0;
 813         for (var i = 0; i < tier.size; ++i) {
 814                 var item = tier.item(i);
 815                 var currentValue = item.value;
 816                 var currentTime = item.x;
 817                 if (currentValue < min) {
 818                         min = currentValue;
 819                         tmin = currentTime;
 820                 };
 821                 if (currentValue > max) {
 822                         max = currentValue;
 823                         tmax = currentTime;
 824                 };
 825         };
 826         return {min: min, max: max, tmin: tmin, tmax: tmax};
 827 };
 828
 829 /*
 830  *
 831  * Use IndexedDB as a store for audio "files"
 832  *
 833  */
 834
 835 // Use IndexedDB as an Audio storage
 836 function saveCurrentAudioWindow (collection, map, fileName) {
 837         if (!currentAudioWindow || currentAudioWindow.length <= 0 || ! recordedSampleRate || recordedSampleRate <= 0) return;
 838         var blob = arrayToBlob (currentAudioWindow, 0, 0, recordedSampleRate);
 839         if (collection && collection.length > 0 && map && map.length > 0 && fileName && fileName.length > 0) {
 840                 addAudioBlob(collection, map, fileName, blob);
 841         };
 842 };
 843
 844 var indexedDBversion = 2;
 845 function getCurrentAudioWindow (collection, map, name) {
 846         var request = indexedDB.open(audioDatabaseName, indexedDBversion);
 847         request.onerror = function(event) {
 848           alert("Use of IndexedDB not allowed");
 849         };
 850         request.onsuccess = function(event) {
 851                 db = this.result;
 852                 var key = (map.length > 0) ? collection+"/"+map+"/"+name : collection+"/"+name;
 853                 var request = db.transaction(["Recordings"], "readwrite")
 854                         .objectStore("Recordings")
 855                         .get(key);
 856
 857                 request.onsuccess = function(event) {
 858                         var record = this.result;
 859                         if(record) {
 860                                 if(record.audio){
 861                                         // processAudio is resolved asynchronously, reset retrievedData when it is finished
 862                                         retrievedData = true;
 863                                         processAudio (record.audio);
 864                                 };
 865                         };
 866                 };
 867
 868                 // Data not found
 869                 request.onerror = function(event) {
 870                         console.log("Unable to retrieve data: "+map+"/"+name+" cannot be found");
 871                 };
 872
 873         };
 874         request.onerror = function(event) {
 875                 console.log("Error: ", event);
 876         }
 877         request.onupgradeneeded = function(event) {
 878                 var db = this.result;
 879                 // Create an objectStore to hold audio blobs.
 880                 initializeObjectStore (db, collection);
 881         };
 882 };
 883
 884 function getAndPlayExample (wordlist, name, playBlob, otherPlay) {
 885         var request = indexedDB.open(examplesDatabaseName, indexedDBversion);
 886         request.onerror = function(event) {
 887           alert("Use of IndexedDB not allowed");
 888         };
 889         request.onsuccess = function(event) {
 890                 db = this.result;
 891                 var key = "Wordlists"+"/"+wordlist+"/"+name;
 892                 var request = db.transaction(["Recordings"], "readwrite")
 893                         .objectStore("Recordings")
 894                         .get(key);
 895
 896                 request.onsuccess = function(event) {
 897                         var record = this.result;
 898                         if(record) {
 899                                 if(record.audio){
 900                                         // playBlob is run asychronously
 901                                         playBlob(record.audio);
 902                                 };
 903                         };
 904                         if((!record || !record.audio) && otherPlay) otherPlay();
 905                 };
 906
 907                 // Data not found
 908                 request.onerror = function(event) {
 909                         if(otherPlay) otherPlay();
 910                         console.log("Unable to retrieve data: "+map+"/"+name+" cannot be found");
 911                 };
 912
 913         };
 914         request.onerror = function(event) {
 915                 console.log("Error: ", event);
 916         }
 917         request.onupgradeneeded = function(event) {
 918                 var db = this.result;
 919                 // Create an objectStore to hold audio blobs.
 920                 initializeObjectStore (db, "Worklists");
 921         };
 922 };
 923
 924 function getCurrentMetaData (collection, processData) {
 925         var request = indexedDB.open(audioDatabaseName, indexedDBversion);
 926         request.onerror = function(event) {
 927           alert("Use of IndexedDB not allowed");
 928         };
 929         request.onsuccess = function(event) {
 930                 db = this.result;
 931                 var request = db.transaction(["Recordings"], "readwrite")
 932                         .objectStore("Recordings")
 933                         .get(collection+"/"+collection+".tsv");
 934
 935                 request.onsuccess = function(event) {
 936                         var record = this.result;
 937                         if(record) {
 938                                 if(record.audio){
 939                                         // This is a text blob
 940                                         if (processData) {
 941                                                 var objectList = csvblob2objectlist(record.audio, processData);
 942                                         };
 943                                 };
 944                         } else {
 945                                 var date = new Date().toLocaleString();
 946                                 var blob = new Blob([''], { type: 'text/tsv', endings: 'native' });
 947                                 var customerObjectStore = db.transaction(["Recordings"], "readwrite").objectStore("Recordings");
 948                                 var request = customerObjectStore.add({ collection: collection, map: "", name: collection+".tsv", date: date, audio: blob }, collection+"/"+collection+".tsv");
 949
 950                                 request.onsuccess = function(event) {
 951                                         console.log("Success: ", this.result, " ", date);
 952                                 };
 953
 954                                 request.onerror = function(event) {
 955                                         console.log("Unable to add data: "+collection+"/"+collection+".tsv"+" cannot be created or updated");
 956                                 };
 957                         };
 958                 };
 959
 960                 // Data not found
 961                 request.onerror = function(event) {
 962                         console.log("Unable to retrieve metadata file: "+collection+"/"+collection+".tsv cannot be found");
 963                 };
 964
 965         };
 966         request.onerror = function(event) {
 967                 console.log("Error: ", event);
 968         }
 969         request.onupgradeneeded = function(event) {
 970                 var db = this.result;
 971                 // Create an objectStore to hold audio blobs.
 972                 initializeObjectStore (db, collection);
 973         };
 974 };
 975
 976 // Create an objectStore to hold audio blobs.
 977 function initializeObjectStore (db, collection) {
 978         var objectStore = db.createObjectStore("Recordings");
 979         objectStore.createIndex("collection", "collection", { unique: false });
 980         objectStore.createIndex("map", "map", { unique: false });
 981         // initialize metadata file
 982         // Use transaction oncomplete to make sure the objectStore creation is
 983         // finished before adding data into it.
 984         objectStore.transaction.oncomplete = function(event) {
 985                 // Store values in the newly created objectStore.
 986                 var date = new Date().toLocaleString();
 987                 var customerObjectStore = db.transaction(["Recordings"], "readwrite").objectStore("Recordings");
 988                 // create empty text blob
 989                 var blob = new Blob([''], { type: 'text/tsv', endings: 'native' });
 990                 var request = customerObjectStore.add({ collection: collection, map: "", name: collection+".tsv", date: date, audio: blob }, collection+"/"+collection+".tsv");
 991
 992                 request.onsuccess = function(event) {
 993                         console.log("Success: ", this.result, " ", date);
 994                 };
 995
 996                 request.onerror = function(event) {
 997                         console.log("Unable to add data: "+collection+"/"+collection+".tsv"+" cannot be created or updated");
 998                 };
 999         };
1000 };
1001
1002 // Use IndexedDB as an Audio storage
1003 // Remove entries that have the same name
1004 // The structure is: Directory, Filename, Binary data
1005 function addAudioBlob(collection, map, name, blob) {
1006         addFileBlob(audioDatabaseName, collection, map, name, blob);
1007 };
1008
1009 function addExamplesBlob(wordlist, name, blob) {
1010         addFileBlob(examplesDatabaseName, "Wordlists", wordlist, name, blob);
1011 };
1012
1013 function addFileBlob(databaseName, collection, map, name, blob) {
1014         var date = new Date().toLocaleString();
1015         var db;
1016         var request = indexedDB.open(databaseName, indexedDBversion);
1017         request.onerror = function(event) {
1018           alert("Use of IndexedDB not allowed");
1019         };
1020         request.onsuccess = function(event) {
1021                 db = this.result;
1022                 var key = (map.length > 0) ? collection+"/"+map+"/"+name : collection+"/"+name;
1023                 var request = db.transaction(["Recordings"], "readwrite")
1024                         .objectStore("Recordings")
1025                         .put({ collection: collection, map: map, name: name, date: date, audio: blob }, key);
1026
1027                 request.onsuccess = function(event) {
1028                         console.log("Success: ", this.result, " ", date);
1029
1030                 };
1031
1032                 // If data already exist, update it
1033                 request.onerror = function(event) {
1034                         console.log("Unable to add data: "+key+" cannot be created or updated");
1035                 };
1036
1037         };
1038
1039         request.onupgradeneeded = function(event) {
1040                 var db = this.result;
1041                 // Create an objectStore to hold audio blobs.
1042                 var objectStore = db.createObjectStore("Recordings");
1043                 objectStore.createIndex("collection", "collection", { unique: false });
1044                 objectStore.createIndex("map", "map", { unique: false });
1045                 // Use transaction oncomplete to make sure the objectStore creation is
1046                 // finished before adding data into it.
1047                 objectStore.transaction.oncomplete = function(event) {
1048                         // Store values in the newly created objectStore.
1049                         var date = new Date().toLocaleString();
1050                         if (!(name && collection)) return;
1051                         var customerObjectStore = db.transaction(["Recordings"], "readwrite").objectStore("Recordings");
1052
1053                         var request2 = customerObjectStore.add({ collection: collection, map: map, name: name, date: date, audio: blob }, collection+"/"+map+"/"+name);
1054                         request2.onsuccess = function(event) {
1055                                 console.log("Success: ", this.result, " ", date);
1056                         };
1057
1058                         request2.onerror = function(event) {
1059                                 console.log("Unable to add data: "+collection+"/"+map+"/"+name+" cannot be created or updated");
1060                         };
1061
1062                         var tsvBlob = new Blob([''], { type: 'text/tsv', endings: 'native' });
1063                         var request1 = customerObjectStore.add({ collection: collection, map: "", name: collection+".tsv", date: date, audio: tsvBlob }, collection+"/"+collection+".tsv");
1064                         request1.onsuccess = function(event) {
1065                                 console.log("Success: ", this.result, " ", date);
1066                         };
1067
1068                         request1.onerror = function(event) {
1069                                 console.log("Unable to add data: "+collection+"/"+map+"/"+name+" cannot be created or updated");
1070                         };
1071                 };
1072         };
1073 };
1074
1075 // CSV is a list of objects, each with the same properties
1076 function writeCSV(collection, csvList) {
1077         var map = "";
1078         var name = collection+".tsv";
1079         var blob = objectlist2csvblob (csvList);
1080         addAudioBlob(collection, map, name, blob);
1081 };
1082
1083 // Iterate over all records
1084 function getAllRecords (collection, processRecords) {
1085         var collectRecords = [];
1086         var db;
1087         var request = indexedDB.open(audioDatabaseName, indexedDBversion);
1088         request.onerror = function(event) {
1089           alert("Use of IndexedDB not allowed");
1090         };
1091         request.onsuccess = function(event) {
1092                 db = this.result;
1093                 var objectStore = db.transaction("Recordings").objectStore("Recordings");
1094                 var index = objectStore.index("collection");
1095                 index.openCursor().onsuccess = function(event) {
1096                   var cursor = event.target.result;
1097                   if (cursor) {
1098                         if (!collection || cursor.value.collection == collection) collectRecords.push(cursor.value);
1099                     cursor.continue();
1100                   } else {
1101                         processRecords(collectRecords);
1102                   };
1103                 };
1104         };
1105
1106         request.onupgradeneeded = function(event) {
1107                 var db = this.result;
1108                 // Create an objectStore to hold audio blobs.
1109                 initializeObjectStore (db, collection)
1110         };
1111 };
1112
1113
1114 // Initialize Audio storage
1115 function initializeDataStorage (collection) {
1116         getCurrentMetaData (collection, false);
1117 };
1118
1119 // Remove a collection
1120 function removeCollection (collection, complete) {
1121         removeSubsetInDB (audioDatabaseName, collection, false, complete);
1122 };
1123
1124 // Remove a wordlist
1125 function removeExamples (wordlistName, complete) {
1126         removeSubsetInDB (examplesDatabaseName, false, wordlistName, complete);
1127 };
1128
1129 function removeSubsetInDB (databaseName, collection, map, complete) {
1130         var db;
1131         var request = indexedDB.open(databaseName, indexedDBversion);
1132         request.onerror = function(event) {
1133           alert("Use of IndexedDB not allowed");
1134         };
1135         request.onsuccess = function(event) {
1136                 db = this.result;
1137                 var objectStore = db.transaction("Recordings", "readwrite").objectStore("Recordings");
1138                 var index = collection ? objectStore.index("collection") : objectStore.index("map");
1139                 index.openCursor().onsuccess = function(event) {
1140                   var cursor = event.target.result;
1141                   if (cursor) {
1142                           var deleteCursor = false;
1143                           deleteCursor = deleteCursor || !(collection || map);
1144                           deleteCursor = deleteCursor || (!map && (collection && cursor.value.collection == collection));
1145                           deleteCursor = deleteCursor || (!collection && (map && cursor.value.map == map));
1146                           deleteCursor = deleteCursor || ((collection && cursor.value.collection == collection) && (map && cursor.value.map == map));
1147                         if (deleteCursor) {
1148                                 var value = cursor.value;
1149                         var request = cursor.delete();
1150                         request.onsuccess = function() {
1151                                         console.log('Delete', value);
1152                                 };
1153                         } else {
1154                                 if(complete)complete(collection);
1155                         };
1156                     cursor.continue();
1157                   };
1158                 };
1159         };
1160
1161         request.onupgradeneeded = function(event) {
1162                 var db = this.result;
1163                 // Create an objectStore to hold audio blobs.
1164                 initializeObjectStore (db, collection)
1165         };
1166
1167 };
1168
1169 // Remove Audio storage, including ALL data
1170 function clearDataStorage (databaseName, storeName) {
1171         var db;
1172         var request = indexedDB.open(databaseName, 1);
1173         request.onerror = function(event) {
1174           alert("Use of IndexedDB not allowed");
1175         };
1176         request.onsuccess = function(event) {
1177                 db = this.result;
1178                 var request = db.transaction([storeName], "readwrite")
1179                         .objectStore(storeName)
1180                         .clear();
1181
1182                 request.onsuccess = function(event) {
1183                         console.log("Success: ", storeName);
1184
1185                 };
1186
1187                 // If data already exist, update it
1188                 request.onerror = function(event) {
1189                         console.log("Unable to clear "+storeName);
1190                 };
1191         };
1192 };
1193
1194
1195 // Remove Audio storage, including ALL data
1196 function deleteDatabase (databaseName) {
1197         var req = indexedDB.deleteDatabase(databaseName);
1198         req.onsuccess = function () {
1199             console.log("Deleted database successfully");
1200         };
1201         req.onerror = function () {
1202             console.log("Couldn't delete database");
1203         };
1204         req.onblocked = function () {
1205             console.log("Couldn't delete database due to the operation being blocked");
1206         };
1207 };
1208
1209 /*
1210  *
1211  * Comma/Tab-Separated-Values
1212  *
1213  */
1214
1215 // Convert a list of objects to a csv blob
1216 function objectlist2csvblob (csvList) {
1217         var headerList = Object.keys(csvList[0]);
1218         var text = headerList.join("\t") + "\n";
1219         for (var i=0; i<csvList.length; ++i) {
1220                 var valueList = [];
1221                 for (var p=0; p<headerList.length; ++p) {
1222                         valueList.push(csvList[i][headerList[p]]);
1223                 };
1224                 text += valueList.join("\t") + "\n";
1225         };
1226         var blob = new Blob ([text], {type: 'text/tsv', endings: 'native'});
1227         return blob;
1228 };
1229
1230 function csvblob2objectlist (blob, handleList) {
1231         if(blob.type != "text/tsv") return undefined;
1232         var reader = new FileReader();
1233         reader.addEventListener("loadend", function() {
1234            // reader.result contains the contents of blob as a typed array
1235                 var lines = reader.result.split(/\r\n|\n/);
1236                 var columnNames = lines[0].split(/\t/);
1237                 var objectList = [];
1238                 for (var i=1; i<lines.length; ++i) {
1239                         if(lines[i].match(/\t/)) {
1240                                 var values = lines[i].split(/\t/);
1241                                 var record = {};
1242                                 for(var p=0; p<columnNames.length; ++p) {
1243                                         record[columnNames[p]] = values[p];
1244                                 };
1245                                 objectList.push(record);
1246                         };
1247                 };
1248                 handleList(objectList);
1249         });
1250         reader.readAsText(blob);
1251 };
1252
1253 // Convert csv into an object. Must contain a property .key
1254 function csvblob2object (blob, handleObject) {
1255         if(blob.type != "text/tsv") return {};
1256         var reader = new FileReader();
1257         reader.addEventListener("loadend", function() {
1258            // reader.result contains the contents of blob as a typed array
1259                 var lines = reader.result.split(/\r\n|\n/);
1260                 var columnNames = lines[0].split(/\t/);
1261                 var object = {};
1262                 for (var i=1; i<lines.length; ++i) {
1263                         if(lines[i].match(/\t/)) {
1264                                 var values = lines[i].split(/\t/);
1265                                 var record = {};
1266                                 for(var p=0; p<columnNames.length; ++p) {
1267                                         record[columnNames[p]] = values[p];
1268                                 };
1269                                 object[record.key] = record;
1270                         };
1271                 };
1272                 handleObject(object);
1273         });
1274         reader.readAsText(blob);
1275 };
1276
1277 function csvList2object (csvList) {
1278         var object = {};
1279         for (var i=0; i<csvList.length; ++i) {
1280                 object[csvList[i].key] = csvList[i];
1281         };
1282         return object;
1283 };
1284
1285 function object2csvList (csvObject) {
1286         var keyList = Object.keys(csvObject);
1287         var csvList = [];
1288         for (var i=0; i<keyList.length; ++i) {
1289                 csvList.push(csvObject[keyList[i]]);
1290         };
1291         return csvList;
1292 };
1293
1294 // Do linear regression
1295 // y = [<y-values]]
1296 // y = [<x-values]]
1297 // return {slope: , intercept: , r2: }
1298 //
1299 // (by: Trent Richardson,
1300 //      http://trentrichardson.com/2010/04/06/compute-linear-regressions-in-javascript/)
1301 function linearRegression(y,x){
1302                 var lr = {};
1303                 var n = y.length;
1304                 var sum_x = 0;
1305                 var sum_y = 0;
1306                 var sum_xy = 0;
1307                 var sum_xx = 0;
1308                 var sum_yy = 0;
1309
1310                 for (var i = 0; i < y.length; i++) {
1311
1312                         sum_x += x[i];
1313                         sum_y += y[i];
1314                         sum_xy += (x[i]*y[i]);
1315                         sum_xx += (x[i]*x[i]);
1316                         sum_yy += (y[i]*y[i]);
1317                 }
1318
1319                 lr['slope'] = (n * sum_xy - sum_x * sum_y) / (n*sum_xx - sum_x * sum_x);
1320                 lr['intercept'] = (sum_y - lr.slope * sum_x)/n;
1321                 lr['r2'] = Math.pow((n*sum_xy - sum_x*sum_y)/Math.sqrt((n*sum_xx-sum_x*sum_x)*(n*sum_yy-sum_y*sum_y)),2);
1322
1323                 return lr;
1324 };
1325
1326 // find y = a*x^2 + b*x + c from three points y=[], x=[]
1327 // I would prefer a 5-point fit
1328 function threePointParabola (y, x) {
1329     var a = ((y[1]-y[0])*(x[0]-x[2]) + (y[2]-y[0])*(x[1]-x[0]))/((x[0]-x[2])*(x[1]*x[1]-x[0]*x[0]) + (x[1]-x[0])*(x[2]*x[2]-x[0]*x[0]))
1330     var b = ((y[1] - y[0]) - a*(x[1]*x[1] - x[0]*x[0])) / (x[1]-x[0])
1331     var c = y[0] - a*x[0]*x[0] - b*x[0];
1332     return {a: a, b: b, c: c};
1333 };