apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 #define UNTAGGED "<Untagged>"
  99
 100 /* Previous path when scanning directory tree recursively. */
 101 static char curpath[TAG_MAXLEN+32];
 102
 103 /* Used when removing duplicates. */
 104 static char *tempbuf;     /* Allocated when needed. */
 105 static long tempbufidx;   /* Current location in buffer. */
 106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 107 static long tempbuf_left; /* Buffer space left. */
 108 static long tempbuf_pos;
 109
 110 #define SORTED_TAGS_COUNT 8
 111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 114     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 115 /* Tags we want to get sorted (loaded to the tempbuf). */
 116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 117     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 118     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 119
 120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 122     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 123     (1LU << tag_albumartist) | (1LU << tag_grouping))
 124
 125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 126 static const char *tags_str[] = { "artist", "album", "genre", "title",
 127     "filename", "composer", "comment", "albumartist", "grouping", "year",
 128     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 129     "playtime", "lastplayed", "commitid", "mtime" };
 130
 131 /* Status information of the tagcache. */
 132 static struct tagcache_stat tc_stat;
 133
 134 /* Queue commands. */
 135 enum tagcache_queue {
 136     Q_STOP_SCAN = 0,
 137     Q_START_SCAN,
 138     Q_IMPORT_CHANGELOG,
 139     Q_UPDATE,
 140     Q_REBUILD,
 141
 142     /* Internal tagcache command queue. */
 143     CMD_UPDATE_MASTER_HEADER,
 144     CMD_UPDATE_NUMERIC,
 145 };
 146
 147 struct tagcache_command_entry {
 148     int32_t command;
 149     int32_t idx_id;
 150     int32_t tag;
 151     int32_t data;
 152 };
 153
 154 #ifndef __PCTOOL__
 155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 156 static volatile int command_queue_widx = 0;
 157 static volatile int command_queue_ridx = 0;
 158 static struct mutex command_queue_mutex;
 159 #endif
 160
 161 /* Tag database structures. */
 162
 163 /* Variable-length tag entry in tag files. */
 164 struct tagfile_entry {
 165     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 166     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 167     char tag_data[0];  /* Begin of the tag data */
 168 };
 169
 170 /* Fixed-size tag entry in master db index. */
 171 struct index_entry {
 172     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 173     int32_t flag;                /* Status flags */
 174 };
 175
 176 /* Header is the same in every file. */
 177 struct tagcache_header {
 178     int32_t magic;       /* Header version number */
 179     int32_t datasize;    /* Data size in bytes */
 180     int32_t entry_count; /* Number of entries in this file */
 181 };
 182
 183 struct master_header {
 184     struct tagcache_header tch;
 185     int32_t serial; /* Increasing counting number */
 186     int32_t commitid; /* Number of commits so far */
 187     int32_t dirty;
 188 };
 189
 190 /* For the endianess correction */
 191 static const char *tagfile_entry_ec   = "ll";
 192 /**
 193  Note: This should be (1 + TAG_COUNT) amount of l's.
 194  */
 195 static const char *index_entry_ec     = "lllllllllllllllllllll";
 196
 197 static const char *tagcache_header_ec = "lll";
 198 static const char *master_header_ec   = "llllll";
 199
 200 static struct master_header current_tcmh;
 201
 202 #ifdef HAVE_TC_RAMCACHE
 203 /* Header is created when loading database to ram. */
 204 struct ramcache_header {
 205     struct master_header h;      /* Header from the master index */
 206     struct index_entry *indices; /* Master index file content */
 207     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 208     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 209 };
 210
 211 # ifdef HAVE_EEPROM_SETTINGS
 212 struct statefile_header {
 213     struct ramcache_header *hdr;
 214     struct tagcache_stat tc_stat;
 215 };
 216 # endif
 217
 218 /* Pointer to allocated ramcache_header */
 219 static struct ramcache_header *hdr;
 220 #endif
 221
 222 /**
 223  * Full tag entries stored in a temporary file waiting
 224  * for commit to the cache. */
 225 struct temp_file_entry {
 226     long tag_offset[TAG_COUNT];
 227     short tag_length[TAG_COUNT];
 228     long flag;
 229
 230     long data_length;
 231 };
 232
 233 struct tempbuf_id_list {
 234     long id;
 235     struct tempbuf_id_list *next;
 236 };
 237
 238 struct tempbuf_searchidx {
 239     long idx_id;
 240     char *str;
 241     int seek;
 242     struct tempbuf_id_list idlist;
 243 };
 244
 245 /* Lookup buffer for fixing messed up index while after sorting. */
 246 static long commit_entry_count;
 247 static long lookup_buffer_depth;
 248 static struct tempbuf_searchidx **lookup;
 249
 250 /* Used when building the temporary file. */
 251 static int cachefd = -1, filenametag_fd;
 252 static int total_entry_count = 0;
 253 static int data_size = 0;
 254 static int processed_dir_count;
 255
 256 /* Thread safe locking */
 257 static volatile int write_lock;
 258 static volatile int read_lock;
 259
 260 static bool delete_entry(long idx_id);
 261
 262 const char* tagcache_tag_to_str(int tag)
 263 {
 264     return tags_str[tag];
 265 }
 266
 267 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 268 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 269 {
 270     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 271 }
 272
 273 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 274 {
 275     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 276 }
 277
 278 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 279 {
 280     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 281 }
 282
 283 #ifdef HAVE_DIRCACHE
 284 /**
 285  * Returns true if specified flag is still present, i.e., dircache
 286  * has not been reloaded.
 287  */
 288 static bool is_dircache_intact(void)
 289 {
 290     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 291 }
 292 #endif
 293
 294 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 295 {
 296     int fd;
 297     char buf[MAX_PATH];
 298     int rc;
 299
 300     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 301         return -1;
 302
 303     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 304
 305     fd = open(buf, write ? O_RDWR : O_RDONLY);
 306     if (fd < 0)
 307     {
 308         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 309         tc_stat.ready = false;
 310         return fd;
 311     }
 312
 313     /* Check the header. */
 314     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 315     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 316     {
 317         logf("header error");
 318         tc_stat.ready = false;
 319         close(fd);
 320         return -2;
 321     }
 322
 323     return fd;
 324 }
 325
 326 static int open_master_fd(struct master_header *hdr, bool write)
 327 {
 328     int fd;
 329     int rc;
 330
 331     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 332     if (fd < 0)
 333     {
 334         logf("master file open failed for R/W");
 335         tc_stat.ready = false;
 336         return fd;
 337     }
 338
 339     tc_stat.econ = false;
 340
 341     /* Check the header. */
 342     rc = read(fd, hdr, sizeof(struct master_header));
 343     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 344     {
 345         /* Success. */
 346         return fd;
 347     }
 348
 349     /* Trying to read again, this time with endianess correction enabled. */
 350     lseek(fd, 0, SEEK_SET);
 351
 352     rc = ecread(fd, hdr, 1, master_header_ec, true);
 353     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 354     {
 355         logf("header error");
 356         tc_stat.ready = false;
 357         close(fd);
 358         return -2;
 359     }
 360
 361     tc_stat.econ = true;
 362
 363     return fd;
 364 }
 365
 366 #ifndef __PCTOOL__
 367 static bool do_timed_yield(void)
 368 {
 369     /* Sorting can lock up for quite a while, so yield occasionally */
 370     static long wakeup_tick = 0;
 371     if (TIME_AFTER(current_tick, wakeup_tick))
 372     {
 373         wakeup_tick = current_tick + (HZ/4);
 374         yield();
 375         return true;
 376     }
 377     return false;
 378 }
 379 #endif
 380
 381 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 382 static long find_entry_ram(const char *filename,
 383                            const struct dircache_entry *dc)
 384 {
 385     static long last_pos = 0;
 386     int i;
 387
 388     /* Check if we tagcache is loaded into ram. */
 389     if (!tc_stat.ramcache)
 390         return -1;
 391
 392     if (dc == NULL)
 393         dc = dircache_get_entry_ptr(filename);
 394
 395     if (dc == NULL)
 396     {
 397         logf("tagcache: file not found.");
 398         return -1;
 399     }
 400
 401     try_again:
 402
 403     if (last_pos > 0)
 404         i = last_pos;
 405     else
 406         i = 0;
 407
 408     for (; i < hdr->h.tch.entry_count; i++)
 409     {
 410         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 411         {
 412             last_pos = MAX(0, i - 3);
 413             return i;
 414         }
 415
 416         do_timed_yield();
 417     }
 418
 419     if (last_pos > 0)
 420     {
 421         last_pos = 0;
 422         goto try_again;
 423     }
 424
 425     return -1;
 426 }
 427 #endif
 428
 429 static long find_entry_disk(const char *filename, bool localfd)
 430 {
 431     struct tagcache_header tch;
 432     static long last_pos = -1;
 433     long pos_history[POS_HISTORY_COUNT];
 434     long pos_history_idx = 0;
 435     bool found = false;
 436     struct tagfile_entry tfe;
 437     int fd;
 438     char buf[TAG_MAXLEN+32];
 439     int i;
 440     int pos = -1;
 441
 442     if (!tc_stat.ready)
 443         return -2;
 444
 445     fd = filenametag_fd;
 446     if (fd < 0 || localfd)
 447     {
 448         last_pos = -1;
 449         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 450             return -1;
 451     }
 452
 453     check_again:
 454
 455     if (last_pos > 0)
 456         lseek(fd, last_pos, SEEK_SET);
 457     else
 458         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 459
 460     while (true)
 461     {
 462         pos = lseek(fd, 0, SEEK_CUR);
 463         for (i = pos_history_idx-1; i >= 0; i--)
 464             pos_history[i+1] = pos_history[i];
 465         pos_history[0] = pos;
 466
 467         if (ecread_tagfile_entry(fd, &tfe)
 468             != sizeof(struct tagfile_entry))
 469         {
 470             break ;
 471         }
 472
 473         if (tfe.tag_length >= (long)sizeof(buf))
 474         {
 475             logf("too long tag #1");
 476             close(fd);
 477             if (!localfd)
 478                 filenametag_fd = -1;
 479             last_pos = -1;
 480             return -2;
 481         }
 482
 483         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 484         {
 485             logf("read error #2");
 486             close(fd);
 487             if (!localfd)
 488                 filenametag_fd = -1;
 489             last_pos = -1;
 490             return -3;
 491         }
 492
 493         if (!strcasecmp(filename, buf))
 494         {
 495             last_pos = pos_history[pos_history_idx];
 496             found = true;
 497             break ;
 498         }
 499
 500         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 501             pos_history_idx++;
 502     }
 503
 504     /* Not found? */
 505     if (!found)
 506     {
 507         if (last_pos > 0)
 508         {
 509             last_pos = -1;
 510             logf("seek again");
 511             goto check_again;
 512         }
 513
 514         if (fd != filenametag_fd || localfd)
 515             close(fd);
 516         return -4;
 517     }
 518
 519     if (fd != filenametag_fd || localfd)
 520         close(fd);
 521
 522     return tfe.idx_id;
 523 }
 524
 525 static int find_index(const char *filename)
 526 {
 527     long idx_id = -1;
 528
 529 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 530     if (tc_stat.ramcache && is_dircache_intact())
 531         idx_id = find_entry_ram(filename, NULL);
 532 #endif
 533
 534     if (idx_id < 0)
 535         idx_id = find_entry_disk(filename, true);
 536
 537     return idx_id;
 538 }
 539
 540 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 541 {
 542     int idx_id;
 543
 544     if (!tc_stat.ready)
 545         return false;
 546
 547     idx_id = find_index(filename);
 548     if (idx_id < 0)
 549         return false;
 550
 551     if (!tagcache_search(tcs, tag_filename))
 552         return false;
 553
 554     tcs->entry_count = 0;
 555     tcs->idx_id = idx_id;
 556
 557     return true;
 558 }
 559
 560 static bool get_index(int masterfd, int idxid,
 561                       struct index_entry *idx, bool use_ram)
 562 {
 563     bool localfd = false;
 564
 565     if (idxid < 0)
 566     {
 567         logf("Incorrect idxid: %d", idxid);
 568         return false;
 569     }
 570
 571 #ifdef HAVE_TC_RAMCACHE
 572     if (tc_stat.ramcache && use_ram)
 573     {
 574         if (hdr->indices[idxid].flag & FLAG_DELETED)
 575             return false;
 576
 577 # ifdef HAVE_DIRCACHE
 578         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 579             || is_dircache_intact())
 580 #endif
 581         {
 582             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 583             return true;
 584         }
 585     }
 586 #else
 587     (void)use_ram;
 588 #endif
 589
 590     if (masterfd < 0)
 591     {
 592         struct master_header tcmh;
 593
 594         localfd = true;
 595         masterfd = open_master_fd(&tcmh, false);
 596         if (masterfd < 0)
 597             return false;
 598     }
 599
 600     lseek(masterfd, idxid * sizeof(struct index_entry)
 601           + sizeof(struct master_header), SEEK_SET);
 602     if (ecread_index_entry(masterfd, idx)
 603         != sizeof(struct index_entry))
 604     {
 605         logf("read error #3");
 606         if (localfd)
 607             close(masterfd);
 608
 609         return false;
 610     }
 611
 612     if (localfd)
 613         close(masterfd);
 614
 615     if (idx->flag & FLAG_DELETED)
 616         return false;
 617
 618     return true;
 619 }
 620
 621 #ifndef __PCTOOL__
 622
 623 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 624 {
 625     /* We need to exclude all memory only flags & tags when writing to disk. */
 626     if (idx->flag & FLAG_DIRCACHE)
 627     {
 628         logf("memory only flags!");
 629         return false;
 630     }
 631
 632 #ifdef HAVE_TC_RAMCACHE
 633     /* Only update numeric data. Writing the whole index to RAM by memcpy
 634      * destroys dircache pointers!
 635      */
 636     if (tc_stat.ramcache)
 637     {
 638         int tag;
 639         struct index_entry *idx_ram = &hdr->indices[idxid];
 640
 641         for (tag = 0; tag < TAG_COUNT; tag++)
 642         {
 643             if (TAGCACHE_IS_NUMERIC(tag))
 644             {
 645                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 646             }
 647         }
 648
 649         /* Don't touch the dircache flag or attributes. */
 650         idx_ram->flag = (idx->flag & 0x0000ffff)
 651             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 652     }
 653 #endif
 654
 655     lseek(masterfd, idxid * sizeof(struct index_entry)
 656           + sizeof(struct master_header), SEEK_SET);
 657     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 658     {
 659         logf("write error #3");
 660         logf("idxid: %d", idxid);
 661         return false;
 662     }
 663
 664     return true;
 665 }
 666
 667 #endif /* !__PCTOOL__ */
 668
 669 static bool open_files(struct tagcache_search *tcs, int tag)
 670 {
 671     if (tcs->idxfd[tag] < 0)
 672     {
 673         char fn[MAX_PATH];
 674
 675         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 676         tcs->idxfd[tag] = open(fn, O_RDONLY);
 677     }
 678
 679     if (tcs->idxfd[tag] < 0)
 680     {
 681         logf("File not open!");
 682         return false;
 683     }
 684
 685     return true;
 686 }
 687
 688 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 689                      int tag, char *buf, long size)
 690 {
 691     struct tagfile_entry tfe;
 692     long seek;
 693
 694     *buf = '\0';
 695
 696     if (TAGCACHE_IS_NUMERIC(tag))
 697         return false;
 698
 699     seek = idx->tag_seek[tag];
 700     if (seek < 0)
 701     {
 702         logf("Retrieve failed");
 703         return false;
 704     }
 705
 706 #ifdef HAVE_TC_RAMCACHE
 707     if (tcs->ramsearch)
 708     {
 709         struct tagfile_entry *ep;
 710
 711 # ifdef HAVE_DIRCACHE
 712         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 713             && is_dircache_intact())
 714         {
 715             dircache_copy_path((struct dircache_entry *)seek,
 716                                buf, size);
 717             return true;
 718         }
 719         else
 720 # endif
 721         if (tag != tag_filename)
 722         {
 723             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 724             strlcpy(buf, ep->tag_data, size);
 725
 726             return true;
 727         }
 728     }
 729 #endif
 730
 731     if (!open_files(tcs, tag))
 732         return false;
 733
 734     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 735     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 736         != sizeof(struct tagfile_entry))
 737     {
 738         logf("read error #5");
 739         return false;
 740     }
 741
 742     if (tfe.tag_length >= size)
 743     {
 744         logf("too small buffer");
 745         return false;
 746     }
 747
 748     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 749         tfe.tag_length)
 750     {
 751         logf("read error #6");
 752         return false;
 753     }
 754
 755     buf[tfe.tag_length] = '\0';
 756
 757     return true;
 758 }
 759
 760 static long check_virtual_tags(int tag, const struct index_entry *idx)
 761 {
 762     long data = 0;
 763
 764     switch (tag)
 765     {
 766         case tag_virt_length_sec:
 767             data = (idx->tag_seek[tag_length]/1000) % 60;
 768             break;
 769
 770         case tag_virt_length_min:
 771             data = (idx->tag_seek[tag_length]/1000) / 60;
 772             break;
 773
 774         case tag_virt_playtime_sec:
 775             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 776             break;
 777
 778         case tag_virt_playtime_min:
 779             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 780             break;
 781
 782         case tag_virt_autoscore:
 783             if (idx->tag_seek[tag_length] == 0
 784                 || idx->tag_seek[tag_playcount] == 0)
 785             {
 786                 data = 0;
 787             }
 788             else
 789             {
 790                 /* A straight calculus gives:
 791                      autoscore = 100 * playtime / length / playcout (1)
 792                    Now, consider the euclidian division of playtime by length:
 793                      playtime = alpha * length + beta
 794                    With:
 795                      0 <= beta < length
 796                    Now, (1) becomes:
 797                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 798                    Both terms should be small enough to avoid any overflow
 799                 */
 800                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 801                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 802                 data /=  idx->tag_seek[tag_playcount];
 803             }
 804             break;
 805
 806         /* How many commits before the file has been added to the DB. */
 807         case tag_virt_entryage:
 808             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 809             break;
 810
 811         default:
 812             data = idx->tag_seek[tag];
 813     }
 814
 815     return data;
 816 }
 817
 818 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 819 {
 820     struct index_entry idx;
 821
 822     if (!tc_stat.ready)
 823         return false;
 824
 825     if (!TAGCACHE_IS_NUMERIC(tag))
 826         return -1;
 827
 828     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 829         return -2;
 830
 831     return check_virtual_tags(tag, &idx);
 832 }
 833
 834 inline static bool str_ends_with(const char *str1, const char *str2)
 835 {
 836     int str_len = strlen(str1);
 837     int clause_len = strlen(str2);
 838
 839     if (clause_len > str_len)
 840         return false;
 841
 842     return !strcasecmp(&str1[str_len - clause_len], str2);
 843 }
 844
 845 inline static bool str_oneof(const char *str, const char *list)
 846 {
 847     const char *sep;
 848     int l, len = strlen(str);
 849
 850     while (*list)
 851     {
 852         sep = strchr(list, '|');
 853         l = sep ? (long)sep - (long)list : (int)strlen(list);
 854         if ((l==len) && !strncasecmp(str, list, len))
 855             return true;
 856         list += sep ? l + 1 : l;
 857     }
 858
 859     return false;
 860 }
 861
 862 static bool check_against_clause(long numeric, const char *str,
 863                                  const struct tagcache_search_clause *clause)
 864 {
 865     if (clause->numeric)
 866     {
 867         switch (clause->type)
 868         {
 869             case clause_is:
 870                 return numeric == clause->numeric_data;
 871             case clause_is_not:
 872                 return numeric != clause->numeric_data;
 873             case clause_gt:
 874                 return numeric > clause->numeric_data;
 875             case clause_gteq:
 876                 return numeric >= clause->numeric_data;
 877             case clause_lt:
 878                 return numeric < clause->numeric_data;
 879             case clause_lteq:
 880                 return numeric <= clause->numeric_data;
 881             default:
 882                 logf("Incorrect numeric tag: %d", clause->type);
 883         }
 884     }
 885     else
 886     {
 887         switch (clause->type)
 888         {
 889             case clause_is:
 890                 return !strcasecmp(clause->str, str);
 891             case clause_is_not:
 892                 return strcasecmp(clause->str, str);
 893             case clause_gt:
 894                 return 0>strcasecmp(clause->str, str);
 895             case clause_gteq:
 896                 return 0>=strcasecmp(clause->str, str);
 897             case clause_lt:
 898                 return 0<strcasecmp(clause->str, str);
 899             case clause_lteq:
 900                 return 0<=strcasecmp(clause->str, str);
 901             case clause_contains:
 902                 return (strcasestr(str, clause->str) != NULL);
 903             case clause_not_contains:
 904                 return (strcasestr(str, clause->str) == NULL);
 905             case clause_begins_with:
 906                 return (strcasestr(str, clause->str) == str);
 907             case clause_not_begins_with:
 908                 return (strcasestr(str, clause->str) != str);
 909             case clause_ends_with:
 910                 return str_ends_with(str, clause->str);
 911             case clause_not_ends_with:
 912                 return !str_ends_with(str, clause->str);
 913             case clause_oneof:
 914                 return str_oneof(str, clause->str);
 915
 916             default:
 917                 logf("Incorrect tag: %d", clause->type);
 918         }
 919     }
 920
 921     return false;
 922 }
 923
 924 static bool check_clauses(struct tagcache_search *tcs,
 925                           struct index_entry *idx,
 926                           struct tagcache_search_clause **clause, int count)
 927 {
 928     int i;
 929
 930 #ifdef HAVE_TC_RAMCACHE
 931     if (tcs->ramsearch)
 932     {
 933         /* Go through all conditional clauses. */
 934         for (i = 0; i < count; i++)
 935         {
 936             struct tagfile_entry *tfe;
 937             int seek;
 938             char buf[256];
 939             char *str = NULL;
 940
 941             seek = check_virtual_tags(clause[i]->tag, idx);
 942
 943             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 944             {
 945                 if (clause[i]->tag == tag_filename)
 946                 {
 947                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 948                     str = buf;
 949                 }
 950                 else
 951                 {
 952                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 953                     str = tfe->tag_data;
 954                 }
 955             }
 956
 957             if (!check_against_clause(seek, str, clause[i]))
 958                 return false;
 959         }
 960     }
 961     else
 962 #endif
 963     {
 964         /* Check for conditions. */
 965         for (i = 0; i < count; i++)
 966         {
 967             struct tagfile_entry tfe;
 968             int seek;
 969             char str[256];
 970
 971             seek = check_virtual_tags(clause[i]->tag, idx);
 972
 973             memset(str, 0, sizeof str);
 974             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 975             {
 976                 int fd = tcs->idxfd[clause[i]->tag];
 977                 lseek(fd, seek, SEEK_SET);
 978                 ecread_tagfile_entry(fd, &tfe);
 979                 if (tfe.tag_length >= (int)sizeof(str))
 980                 {
 981                     logf("Too long tag read!");
 982                     break ;
 983                 }
 984
 985                 read(fd, str, tfe.tag_length);
 986
 987                 /* Check if entry has been deleted. */
 988                 if (str[0] == '\0')
 989                     break;
 990             }
 991
 992             if (!check_against_clause(seek, str, clause[i]))
 993                 return false;
 994         }
 995     }
 996
 997     return true;
 998 }
 999
1000 bool tagcache_check_clauses(struct tagcache_search *tcs,
1001                             struct tagcache_search_clause **clause, int count)
1002 {
1003     struct index_entry idx;
1004
1005     if (count == 0)
1006         return true;
1007
1008     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1009         return false;
1010
1011     return check_clauses(tcs, &idx, clause, count);
1012 }
1013
1014 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1015 {
1016     int i;
1017
1018     /* If uniq buffer is not defined we must return true for search to work. */
1019     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1020         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1021     {
1022         return true;
1023     }
1024
1025     for (i = 0; i < tcs->unique_list_count; i++)
1026     {
1027         /* Return false if entry is found. */
1028         if (tcs->unique_list[i] == id)
1029             return false;
1030     }
1031
1032     if (tcs->unique_list_count < tcs->unique_list_capacity)
1033     {
1034         tcs->unique_list[i] = id;
1035         tcs->unique_list_count++;
1036     }
1037
1038     return true;
1039 }
1040
1041 static bool build_lookup_list(struct tagcache_search *tcs)
1042 {
1043     struct index_entry entry;
1044     int i, j;
1045
1046     tcs->seek_list_count = 0;
1047
1048 #ifdef HAVE_TC_RAMCACHE
1049     if (tcs->ramsearch
1050 # ifdef HAVE_DIRCACHE
1051         && (tcs->type != tag_filename || is_dircache_intact())
1052 # endif
1053         )
1054     {
1055         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1056         {
1057             struct tagcache_seeklist_entry *seeklist;
1058             struct index_entry *idx = &hdr->indices[i];
1059             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1060                 break ;
1061
1062             /* Skip deleted files. */
1063             if (idx->flag & FLAG_DELETED)
1064                 continue;
1065
1066             /* Go through all filters.. */
1067             for (j = 0; j < tcs->filter_count; j++)
1068             {
1069                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1070                 {
1071                     break ;
1072                 }
1073             }
1074
1075             if (j < tcs->filter_count)
1076                 continue ;
1077
1078             /* Check for conditions. */
1079             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1080                 continue;
1081
1082             /* Add to the seek list if not already in uniq buffer. */
1083             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1084                 continue;
1085
1086             /* Lets add it. */
1087             seeklist = &tcs->seeklist[tcs->seek_list_count];
1088             seeklist->seek = idx->tag_seek[tcs->type];
1089             seeklist->flag = idx->flag;
1090             seeklist->idx_id = i;
1091             tcs->seek_list_count++;
1092         }
1093
1094         tcs->seek_pos = i;
1095
1096         return tcs->seek_list_count > 0;
1097     }
1098 #endif
1099
1100     if (tcs->masterfd < 0)
1101     {
1102         struct master_header tcmh;
1103         tcs->masterfd = open_master_fd(&tcmh, false);
1104     }
1105
1106     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1107             sizeof(struct master_header), SEEK_SET);
1108
1109     while (ecread_index_entry(tcs->masterfd, &entry)
1110            == sizeof(struct index_entry))
1111     {
1112         struct tagcache_seeklist_entry *seeklist;
1113
1114         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1115             break ;
1116
1117         i = tcs->seek_pos;
1118         tcs->seek_pos++;
1119
1120         /* Check if entry has been deleted. */
1121         if (entry.flag & FLAG_DELETED)
1122             continue;
1123
1124         /* Go through all filters.. */
1125         for (j = 0; j < tcs->filter_count; j++)
1126         {
1127             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1128                 break ;
1129         }
1130
1131         if (j < tcs->filter_count)
1132             continue ;
1133
1134         /* Check for conditions. */
1135         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1136             continue;
1137
1138         /* Add to the seek list if not already in uniq buffer. */
1139         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1140             continue;
1141
1142         /* Lets add it. */
1143         seeklist = &tcs->seeklist[tcs->seek_list_count];
1144         seeklist->seek = entry.tag_seek[tcs->type];
1145         seeklist->flag = entry.flag;
1146         seeklist->idx_id = i;
1147         tcs->seek_list_count++;
1148
1149         yield();
1150     }
1151
1152     return tcs->seek_list_count > 0;
1153 }
1154
1155
1156 static void remove_files(void)
1157 {
1158     int i;
1159     char buf[MAX_PATH];
1160
1161     tc_stat.ready = false;
1162     tc_stat.ramcache = false;
1163     tc_stat.econ = false;
1164     remove(TAGCACHE_FILE_MASTER);
1165     for (i = 0; i < TAG_COUNT; i++)
1166     {
1167         if (TAGCACHE_IS_NUMERIC(i))
1168             continue;
1169
1170         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1171         remove(buf);
1172     }
1173 }
1174
1175
1176 static bool check_all_headers(void)
1177 {
1178     struct master_header myhdr;
1179     struct tagcache_header tch;
1180     int tag;
1181     int fd;
1182
1183     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1184         return false;
1185
1186     close(fd);
1187     if (myhdr.dirty)
1188     {
1189         logf("tagcache is dirty!");
1190         return false;
1191     }
1192
1193     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1194
1195     for (tag = 0; tag < TAG_COUNT; tag++)
1196     {
1197         if (TAGCACHE_IS_NUMERIC(tag))
1198             continue;
1199
1200         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1201             return false;
1202
1203         close(fd);
1204     }
1205
1206     return true;
1207 }
1208
1209 bool tagcache_is_busy(void)
1210 {
1211     return read_lock || write_lock;
1212 }
1213
1214 bool tagcache_search(struct tagcache_search *tcs, int tag)
1215 {
1216     struct tagcache_header tag_hdr;
1217     struct master_header   master_hdr;
1218     int i;
1219
1220     while (read_lock)
1221         sleep(1);
1222
1223     memset(tcs, 0, sizeof(struct tagcache_search));
1224     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1225         return false;
1226
1227     tcs->position = sizeof(struct tagcache_header);
1228     tcs->type = tag;
1229     tcs->seek_pos = 0;
1230     tcs->list_position = 0;
1231     tcs->seek_list_count = 0;
1232     tcs->filter_count = 0;
1233     tcs->masterfd = -1;
1234
1235     for (i = 0; i < TAG_COUNT; i++)
1236         tcs->idxfd[i] = -1;
1237
1238 #ifndef HAVE_TC_RAMCACHE
1239     tcs->ramsearch = false;
1240 #else
1241     tcs->ramsearch = tc_stat.ramcache;
1242     if (tcs->ramsearch)
1243     {
1244         tcs->entry_count = hdr->entry_count[tcs->type];
1245     }
1246     else
1247 #endif
1248     {
1249         /* Always open as R/W so we can pass tcs to functions that modify data also
1250          * without failing. */
1251         tcs->masterfd = open_master_fd(&master_hdr, true);
1252         if (tcs->masterfd < 0)
1253             return false;
1254
1255         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1256         {
1257             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1258             if (tcs->idxfd[tcs->type] < 0)
1259                 return false;
1260
1261             tcs->entry_count = tag_hdr.entry_count;
1262         }
1263         else
1264         {
1265             tcs->entry_count = master_hdr.tch.entry_count;
1266         }
1267     }
1268
1269     tcs->valid = true;
1270     tcs->initialized = true;
1271     write_lock++;
1272
1273     return true;
1274 }
1275
1276 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1277                                  void *buffer, long length)
1278 {
1279     tcs->unique_list = (unsigned long *)buffer;
1280     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1281     tcs->unique_list_count = 0;
1282 }
1283
1284 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1285                                 int tag, int seek)
1286 {
1287     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1288         return false;
1289
1290     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1291         return false;
1292
1293     tcs->filter_tag[tcs->filter_count] = tag;
1294     tcs->filter_seek[tcs->filter_count] = seek;
1295     tcs->filter_count++;
1296
1297     return true;
1298 }
1299
1300 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1301                                 struct tagcache_search_clause *clause)
1302 {
1303     int i;
1304
1305     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1306     {
1307         logf("Too many clauses");
1308         return false;
1309     }
1310
1311     /* Check if there is already a similar filter in present (filters are
1312      * much faster than clauses).
1313      */
1314     for (i = 0; i < tcs->filter_count; i++)
1315     {
1316         if (tcs->filter_tag[i] == clause->tag)
1317             return true;
1318     }
1319
1320     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1321     {
1322         char buf[MAX_PATH];
1323
1324         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1325         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1326     }
1327
1328     tcs->clause[tcs->clause_count] = clause;
1329     tcs->clause_count++;
1330
1331     return true;
1332 }
1333
1334 static bool get_next(struct tagcache_search *tcs)
1335 {
1336     static char buf[TAG_MAXLEN+32];
1337     struct tagfile_entry entry;
1338     long flag = 0;
1339
1340     if (!tcs->valid || !tc_stat.ready)
1341         return false;
1342
1343     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1344 #ifdef HAVE_TC_RAMCACHE
1345         && !tcs->ramsearch
1346 #endif
1347         )
1348         return false;
1349
1350     /* Relative fetch. */
1351     if (tcs->filter_count > 0 || tcs->clause_count > 0
1352         || TAGCACHE_IS_NUMERIC(tcs->type)
1353 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1354         /* We need to retrieve flag status for dircache. */
1355         || (tcs->ramsearch && tcs->type == tag_filename)
1356 #endif
1357         )
1358     {
1359         struct tagcache_seeklist_entry *seeklist;
1360
1361         /* Check for end of list. */
1362         if (tcs->list_position == tcs->seek_list_count)
1363         {
1364             tcs->list_position = 0;
1365
1366             /* Try to fetch more. */
1367             if (!build_lookup_list(tcs))
1368             {
1369                 tcs->valid = false;
1370                 return false;
1371             }
1372         }
1373
1374         seeklist = &tcs->seeklist[tcs->list_position];
1375         flag = seeklist->flag;
1376         tcs->position = seeklist->seek;
1377         tcs->idx_id = seeklist->idx_id;
1378         tcs->list_position++;
1379     }
1380     else
1381     {
1382         if (tcs->entry_count == 0)
1383         {
1384             tcs->valid = false;
1385             return false;
1386         }
1387
1388         tcs->entry_count--;
1389     }
1390
1391     tcs->result_seek = tcs->position;
1392
1393     if (TAGCACHE_IS_NUMERIC(tcs->type))
1394     {
1395         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1396         tcs->result = buf;
1397         tcs->result_len = strlen(buf) + 1;
1398         return true;
1399     }
1400
1401     /* Direct fetch. */
1402 #ifdef HAVE_TC_RAMCACHE
1403     if (tcs->ramsearch)
1404     {
1405
1406 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1407         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1408             && is_dircache_intact())
1409         {
1410             dircache_copy_path((struct dircache_entry *)tcs->position,
1411                                buf, sizeof buf);
1412             tcs->result = buf;
1413             tcs->result_len = strlen(buf) + 1;
1414             tcs->ramresult = false;
1415
1416             return true;
1417         }
1418         else
1419 #endif
1420         if (tcs->type != tag_filename)
1421         {
1422             struct tagfile_entry *ep;
1423
1424             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1425             tcs->result = ep->tag_data;
1426             tcs->result_len = strlen(tcs->result) + 1;
1427             tcs->idx_id = ep->idx_id;
1428             tcs->ramresult = true;
1429
1430             /* Increase position for the next run. This may get overwritten. */
1431             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1432
1433             return true;
1434         }
1435     }
1436 #endif
1437
1438     if (!open_files(tcs, tcs->type))
1439     {
1440         tcs->valid = false;
1441         return false;
1442     }
1443
1444     /* Seek stream to the correct position and continue to direct fetch. */
1445     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1446
1447     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1448     {
1449         logf("read error #5");
1450         tcs->valid = false;
1451         return false;
1452     }
1453
1454     if (entry.tag_length > (long)sizeof(buf))
1455     {
1456         tcs->valid = false;
1457         logf("too long tag #2");
1458         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1459         return false;
1460     }
1461
1462     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1463     {
1464         tcs->valid = false;
1465         logf("read error #4");
1466         return false;
1467     }
1468
1469     /**
1470      Update the position for the next read (this may be overridden
1471      if filters or clauses are being used).
1472      */
1473     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1474     tcs->result = buf;
1475     tcs->result_len = strlen(tcs->result) + 1;
1476     tcs->idx_id = entry.idx_id;
1477     tcs->ramresult = false;
1478
1479     return true;
1480 }
1481
1482 bool tagcache_get_next(struct tagcache_search *tcs)
1483 {
1484     while (get_next(tcs))
1485     {
1486         if (tcs->result_len > 1)
1487             return true;
1488     }
1489
1490     return false;
1491 }
1492
1493 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1494                        int tag, char *buf, long size)
1495 {
1496     struct index_entry idx;
1497
1498     *buf = '\0';
1499     if (!get_index(tcs->masterfd, idxid, &idx, true))
1500         return false;
1501
1502     return retrieve(tcs, &idx, tag, buf, size);
1503 }
1504
1505 static bool update_master_header(void)
1506 {
1507     struct master_header myhdr;
1508     int fd;
1509
1510     if (!tc_stat.ready)
1511         return false;
1512
1513     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1514         return false;
1515
1516     myhdr.serial = current_tcmh.serial;
1517     myhdr.commitid = current_tcmh.commitid;
1518     myhdr.dirty = current_tcmh.dirty;
1519
1520     /* Write it back */
1521     lseek(fd, 0, SEEK_SET);
1522     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1523     close(fd);
1524
1525 #ifdef HAVE_TC_RAMCACHE
1526     if (hdr)
1527     {
1528         hdr->h.serial = current_tcmh.serial;
1529         hdr->h.commitid = current_tcmh.commitid;
1530         hdr->h.dirty = current_tcmh.dirty;
1531     }
1532 #endif
1533
1534     return true;
1535 }
1536
1537 #if 0
1538
1539 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1540 {
1541     struct tagentry *entry;
1542
1543     if (tcs->type != tag_title)
1544         return ;
1545
1546     /* We will need reserve buffer for this. */
1547     if (tcs->ramcache)
1548     {
1549         struct tagfile_entry *ep;
1550
1551         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1552         tcs->seek_list[tcs->seek_list_count];
1553     }
1554
1555     entry = find_entry_ram();
1556
1557 }
1558 #endif
1559
1560 void tagcache_search_finish(struct tagcache_search *tcs)
1561 {
1562     int i;
1563
1564     if (!tcs->initialized)
1565         return;
1566
1567     if (tcs->masterfd >= 0)
1568     {
1569         close(tcs->masterfd);
1570         tcs->masterfd = -1;
1571     }
1572
1573     for (i = 0; i < TAG_COUNT; i++)
1574     {
1575         if (tcs->idxfd[i] >= 0)
1576         {
1577             close(tcs->idxfd[i]);
1578             tcs->idxfd[i] = -1;
1579         }
1580     }
1581
1582     tcs->ramsearch = false;
1583     tcs->valid = false;
1584     tcs->initialized = 0;
1585     if (write_lock > 0)
1586         write_lock--;
1587 }
1588
1589 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1590 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1591 {
1592     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1593 }
1594
1595 static long get_tag_numeric(const struct index_entry *entry, int tag)
1596 {
1597     return check_virtual_tags(tag, entry);
1598 }
1599
1600 static char* get_tag_string(const struct index_entry *entry, int tag)
1601 {
1602     char* s = get_tag(entry, tag)->tag_data;
1603     return strcmp(s, UNTAGGED) ? s : NULL;
1604 }
1605
1606 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1607 {
1608     struct index_entry *entry;
1609     int idx_id;
1610
1611     if (!tc_stat.ready || !tc_stat.ramcache)
1612         return false;
1613
1614     /* Find the corresponding entry in tagcache. */
1615     idx_id = find_entry_ram(filename, NULL);
1616     if (idx_id < 0)
1617         return false;
1618
1619     entry = &hdr->indices[idx_id];
1620
1621     memset(id3, 0, sizeof(struct mp3entry));
1622
1623     id3->title        = get_tag_string(entry, tag_title);
1624     id3->artist       = get_tag_string(entry, tag_artist);
1625     id3->album        = get_tag_string(entry, tag_album);
1626     id3->genre_string = get_tag_string(entry, tag_genre);
1627     id3->composer     = get_tag_string(entry, tag_composer);
1628     id3->comment      = get_tag_string(entry, tag_comment);
1629     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1630     id3->grouping     = get_tag_string(entry, tag_grouping);
1631
1632     id3->length     = get_tag_numeric(entry, tag_length);
1633     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1634     id3->rating     = get_tag_numeric(entry, tag_rating);
1635     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1636     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1637     id3->year       = get_tag_numeric(entry, tag_year);
1638
1639     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1640     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1641     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1642     if (id3->bitrate == 0)
1643         id3->bitrate = 1;
1644
1645     return true;
1646 }
1647 #endif
1648
1649 static inline void write_item(const char *item)
1650 {
1651     int len = strlen(item) + 1;
1652
1653     data_size += len;
1654     write(cachefd, item, len);
1655 }
1656
1657 static int check_if_empty(char **tag)
1658 {
1659     int length;
1660
1661     if (*tag == NULL || **tag == '\0')
1662     {
1663         *tag = UNTAGGED;
1664         return sizeof(UNTAGGED); /* Tag length */
1665     }
1666
1667     length = strlen(*tag);
1668     if (length > TAG_MAXLEN)
1669     {
1670         logf("over length tag: %s", *tag);
1671         length = TAG_MAXLEN;
1672         (*tag)[length] = '\0';
1673     }
1674
1675     return length + 1;
1676 }
1677
1678 #define ADD_TAG(entry,tag,data) \
1679     /* Adding tag */ \
1680     entry.tag_offset[tag] = offset; \
1681     entry.tag_length[tag] = check_if_empty(data); \
1682     offset += entry.tag_length[tag]
1683 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1684  * idea, as it uses lots of stack and is called from a recursive function
1685  * (check_dir).
1686  */
1687 static void __attribute__ ((noinline)) add_tagcache(char *path,
1688                                                     unsigned long mtime
1689 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1690                                                     ,const struct dircache_entry *dc
1691 #endif
1692                                                    )
1693 {
1694     struct mp3entry id3;
1695     struct temp_file_entry entry;
1696     bool ret;
1697     int fd;
1698     int idx_id = -1;
1699     char tracknumfix[3];
1700     int offset = 0;
1701     int path_length = strlen(path);
1702     bool has_albumartist;
1703     bool has_grouping;
1704
1705 #ifdef SIMULATOR
1706     /* Crude logging for the sim - to aid in debugging */
1707     int logfd = open(ROCKBOX_DIR "/database.log",
1708                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1709     if (logfd >= 0) {
1710         write(logfd, path, strlen(path));
1711         write(logfd, "\n", 1);
1712         close(logfd);
1713     }
1714 #endif
1715
1716     if (cachefd < 0)
1717         return ;
1718
1719     /* Check for overlength file path. */
1720     if (path_length > TAG_MAXLEN)
1721     {
1722         /* Path can't be shortened. */
1723         logf("Too long path: %s", path);
1724         return ;
1725     }
1726
1727     /* Check if the file is supported. */
1728     if (probe_file_format(path) == AFMT_UNKNOWN)
1729         return ;
1730
1731     /* Check if the file is already cached. */
1732 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1733     if (tc_stat.ramcache && is_dircache_intact())
1734     {
1735         idx_id = find_entry_ram(path, dc);
1736     }
1737 #endif
1738
1739     /* Be sure the entry doesn't exist. */
1740     if (filenametag_fd >= 0 && idx_id < 0)
1741         idx_id = find_entry_disk(path, false);
1742
1743     /* Check if file has been modified. */
1744     if (idx_id >= 0)
1745     {
1746         struct index_entry idx;
1747
1748         /* TODO: Mark that the index exists (for fast reverse scan) */
1749         //found_idx[idx_id/8] |= idx_id%8;
1750
1751         if (!get_index(-1, idx_id, &idx, true))
1752         {
1753             logf("failed to retrieve index entry");
1754             return ;
1755         }
1756
1757         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1758         {
1759             /* No changes to file. */
1760             return ;
1761         }
1762
1763         /* Metadata might have been changed. Delete the entry. */
1764         logf("Re-adding: %s", path);
1765         if (!delete_entry(idx_id))
1766         {
1767             logf("delete_entry failed: %d", idx_id);
1768             return ;
1769         }
1770     }
1771
1772     fd = open(path, O_RDONLY);
1773     if (fd < 0)
1774     {
1775         logf("open fail: %s", path);
1776         return ;
1777     }
1778
1779     memset(&id3, 0, sizeof(struct mp3entry));
1780     memset(&entry, 0, sizeof(struct temp_file_entry));
1781     memset(&tracknumfix, 0, sizeof(tracknumfix));
1782     ret = get_metadata(&id3, fd, path);
1783     close(fd);
1784
1785     if (!ret)
1786         return ;
1787
1788     logf("-> %s", path);
1789
1790     /* Generate track number if missing. */
1791     if (id3.tracknum <= 0)
1792     {
1793         const char *p = strrchr(path, '.');
1794
1795         if (p == NULL)
1796             p = &path[strlen(path)-1];
1797
1798         while (*p != '/')
1799         {
1800             if (isdigit(*p) && isdigit(*(p-1)))
1801             {
1802                 tracknumfix[1] = *p--;
1803                 tracknumfix[0] = *p;
1804                 break;
1805             }
1806             p--;
1807         }
1808
1809         if (tracknumfix[0] != '\0')
1810         {
1811             id3.tracknum = atoi(tracknumfix);
1812             /* Set a flag to indicate track number has been generated. */
1813             entry.flag |= FLAG_TRKNUMGEN;
1814         }
1815         else
1816         {
1817             /* Unable to generate track number. */
1818             id3.tracknum = -1;
1819         }
1820     }
1821
1822     /* Numeric tags */
1823     entry.tag_offset[tag_year] = id3.year;
1824     entry.tag_offset[tag_discnumber] = id3.discnum;
1825     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1826     entry.tag_offset[tag_length] = id3.length;
1827     entry.tag_offset[tag_bitrate] = id3.bitrate;
1828     entry.tag_offset[tag_mtime] = mtime;
1829
1830     /* String tags. */
1831     has_albumartist = id3.albumartist != NULL
1832         && strlen(id3.albumartist) > 0;
1833     has_grouping = id3.grouping != NULL
1834         && strlen(id3.grouping) > 0;
1835
1836     ADD_TAG(entry, tag_filename, &path);
1837     ADD_TAG(entry, tag_title, &id3.title);
1838     ADD_TAG(entry, tag_artist, &id3.artist);
1839     ADD_TAG(entry, tag_album, &id3.album);
1840     ADD_TAG(entry, tag_genre, &id3.genre_string);
1841     ADD_TAG(entry, tag_composer, &id3.composer);
1842     ADD_TAG(entry, tag_comment, &id3.comment);
1843     if (has_albumartist)
1844     {
1845         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1846     }
1847     else
1848     {
1849         ADD_TAG(entry, tag_albumartist, &id3.artist);
1850     }
1851     if (has_grouping)
1852     {
1853         ADD_TAG(entry, tag_grouping, &id3.grouping);
1854     }
1855     else
1856     {
1857         ADD_TAG(entry, tag_grouping, &id3.title);
1858     }
1859     entry.data_length = offset;
1860
1861     /* Write the header */
1862     write(cachefd, &entry, sizeof(struct temp_file_entry));
1863
1864     /* And tags also... Correct order is critical */
1865     write_item(path);
1866     write_item(id3.title);
1867     write_item(id3.artist);
1868     write_item(id3.album);
1869     write_item(id3.genre_string);
1870     write_item(id3.composer);
1871     write_item(id3.comment);
1872     if (has_albumartist)
1873     {
1874         write_item(id3.albumartist);
1875     }
1876     else
1877     {
1878         write_item(id3.artist);
1879     }
1880     if (has_grouping)
1881     {
1882         write_item(id3.grouping);
1883     }
1884     else
1885     {
1886         write_item(id3.title);
1887     }
1888     total_entry_count++;
1889 }
1890
1891 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1892 {
1893     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1894     int len = strlen(str)+1;
1895     int i;
1896     unsigned crc32;
1897     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1898     char buf[TAG_MAXLEN+32];
1899
1900     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1901         buf[i] = tolower(str[i]);
1902     buf[i] = '\0';
1903
1904     crc32 = crc_32(buf, i, 0xffffffff);
1905
1906     if (unique)
1907     {
1908         /* Check if the crc does not exist -> entry does not exist for sure. */
1909         for (i = 0; i < tempbufidx; i++)
1910         {
1911             if (crcbuf[-i] != crc32)
1912                 continue;
1913
1914             if (!strcasecmp(str, index[i].str))
1915             {
1916                 if (id < 0 || id >= lookup_buffer_depth)
1917                 {
1918                     logf("lookup buf overf.: %d", id);
1919                     return false;
1920                 }
1921
1922                 lookup[id] = &index[i];
1923                 return true;
1924             }
1925         }
1926     }
1927
1928     /* Insert to CRC buffer. */
1929     crcbuf[-tempbufidx] = crc32;
1930     tempbuf_left -= 4;
1931
1932     /* Insert it to the buffer. */
1933     tempbuf_left -= len;
1934     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1935         return false;
1936
1937     if (id >= lookup_buffer_depth)
1938     {
1939         logf("lookup buf overf. #2: %d", id);
1940         return false;
1941     }
1942
1943     if (id >= 0)
1944     {
1945         lookup[id] = &index[tempbufidx];
1946         index[tempbufidx].idlist.id = id;
1947     }
1948     else
1949         index[tempbufidx].idlist.id = -1;
1950
1951     index[tempbufidx].idlist.next = NULL;
1952     index[tempbufidx].idx_id = idx_id;
1953     index[tempbufidx].seek = -1;
1954     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1955     memcpy(index[tempbufidx].str, str, len);
1956     tempbuf_pos += len;
1957     tempbufidx++;
1958
1959     return true;
1960 }
1961
1962 static int compare(const void *p1, const void *p2)
1963 {
1964     do_timed_yield();
1965
1966     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1967     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1968
1969     if (strcmp(e1->str, UNTAGGED) == 0)
1970     {
1971         if (strcmp(e2->str, UNTAGGED) == 0)
1972             return 0;
1973         return -1;
1974     }
1975     else if (strcmp(e2->str, UNTAGGED) == 0)
1976         return 1;
1977
1978     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1979 }
1980
1981 static int tempbuf_sort(int fd)
1982 {
1983     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1984     struct tagfile_entry fe;
1985     int i;
1986     int length;
1987
1988     /* Generate reverse lookup entries. */
1989     for (i = 0; i < lookup_buffer_depth; i++)
1990     {
1991         struct tempbuf_id_list *idlist;
1992
1993         if (!lookup[i])
1994             continue;
1995
1996         if (lookup[i]->idlist.id == i)
1997             continue;
1998
1999         idlist = &lookup[i]->idlist;
2000         while (idlist->next != NULL)
2001             idlist = idlist->next;
2002
2003         tempbuf_left -= sizeof(struct tempbuf_id_list);
2004         if (tempbuf_left - 4 < 0)
2005             return -1;
2006
2007         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2008         if (tempbuf_pos & 0x03)
2009         {
2010             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2011             tempbuf_left -= 3;
2012             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2013         }
2014         tempbuf_pos += sizeof(struct tempbuf_id_list);
2015
2016         idlist = idlist->next;
2017         idlist->id = i;
2018         idlist->next = NULL;
2019
2020         do_timed_yield();
2021     }
2022
2023     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2024     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2025
2026     for (i = 0; i < tempbufidx; i++)
2027     {
2028         struct tempbuf_id_list *idlist = &index[i].idlist;
2029
2030         /* Fix the lookup list. */
2031         while (idlist != NULL)
2032         {
2033             if (idlist->id >= 0)
2034                 lookup[idlist->id] = &index[i];
2035             idlist = idlist->next;
2036         }
2037
2038         index[i].seek = lseek(fd, 0, SEEK_CUR);
2039         length = strlen(index[i].str) + 1;
2040         fe.tag_length = length;
2041         fe.idx_id = index[i].idx_id;
2042
2043         /* Check the chunk alignment. */
2044         if ((fe.tag_length + sizeof(struct tagfile_entry))
2045             % TAGFILE_ENTRY_CHUNK_LENGTH)
2046         {
2047             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2048                 ((fe.tag_length + sizeof(struct tagfile_entry))
2049                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2050         }
2051
2052 #ifdef TAGCACHE_STRICT_ALIGN
2053         /* Make sure the entry is long aligned. */
2054         if (index[i].seek & 0x03)
2055         {
2056             logf("tempbuf_sort: alignment error!");
2057             return -3;
2058         }
2059 #endif
2060
2061         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2062             sizeof(struct tagfile_entry))
2063         {
2064             logf("tempbuf_sort: write error #1");
2065             return -1;
2066         }
2067
2068         if (write(fd, index[i].str, length) != length)
2069         {
2070             logf("tempbuf_sort: write error #2");
2071             return -2;
2072         }
2073
2074         /* Write some padding. */
2075         if (fe.tag_length - length > 0)
2076             write(fd, "XXXXXXXX", fe.tag_length - length);
2077     }
2078
2079     return i;
2080 }
2081
2082 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2083 {
2084     if (id < 0 || id >= lookup_buffer_depth)
2085         return NULL;
2086
2087     return lookup[id];
2088 }
2089
2090
2091 inline static int tempbuf_find_location(int id)
2092 {
2093     struct tempbuf_searchidx *entry;
2094
2095     entry = tempbuf_locate(id);
2096     if (entry == NULL)
2097         return -1;
2098
2099     return entry->seek;
2100 }
2101
2102 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2103 {
2104     struct master_header tcmh;
2105     struct index_entry idx;
2106     int masterfd;
2107     int masterfd_pos;
2108     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2109     int max_entries;
2110     int entries_processed = 0;
2111     int i, j;
2112     char buf[TAG_MAXLEN];
2113
2114     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2115
2116     logf("Building numeric indices...");
2117     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2118
2119     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2120         return false;
2121
2122     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2123                          SEEK_CUR);
2124     if (masterfd_pos == filesize(masterfd))
2125     {
2126         logf("we can't append!");
2127         close(masterfd);
2128         return false;
2129     }
2130
2131     while (entries_processed < h->entry_count)
2132     {
2133         int count = MIN(h->entry_count - entries_processed, max_entries);
2134
2135         /* Read in as many entries as possible. */
2136         for (i = 0; i < count; i++)
2137         {
2138             struct temp_file_entry *tfe = &entrybuf[i];
2139             int datastart;
2140
2141             /* Read in numeric data. */
2142             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2143                 sizeof(struct temp_file_entry))
2144             {
2145                 logf("read fail #1");
2146                 close(masterfd);
2147                 return false;
2148             }
2149
2150             datastart = lseek(tmpfd, 0, SEEK_CUR);
2151
2152             /**
2153              * Read string data from the following tags:
2154              * - tag_filename
2155              * - tag_artist
2156              * - tag_album
2157              * - tag_title
2158              *
2159              * A crc32 hash is calculated from the read data
2160              * and stored back to the data offset field kept in memory.
2161              */
2162 #define tmpdb_read_string_tag(tag) \
2163     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2164     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2165     { \
2166         logf("read fail: buffer overflow"); \
2167         close(masterfd); \
2168         return false; \
2169     } \
2170     \
2171     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2172         tfe->tag_length[tag]) \
2173     { \
2174         logf("read fail #2"); \
2175         close(masterfd); \
2176         return false; \
2177     } \
2178     \
2179     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2180     lseek(tmpfd, datastart, SEEK_SET)
2181
2182             tmpdb_read_string_tag(tag_filename);
2183             tmpdb_read_string_tag(tag_artist);
2184             tmpdb_read_string_tag(tag_album);
2185             tmpdb_read_string_tag(tag_title);
2186
2187             /* Seek to the end of the string data. */
2188             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2189         }
2190
2191         /* Backup the master index position. */
2192         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2193         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2194
2195         /* Check if we can resurrect some deleted runtime statistics data. */
2196         for (i = 0; i < tcmh.tch.entry_count; i++)
2197         {
2198             /* Read the index entry. */
2199             if (ecread_index_entry(masterfd, &idx)
2200                 != sizeof(struct index_entry))
2201             {
2202                 logf("read fail #3");
2203                 close(masterfd);
2204                 return false;
2205             }
2206
2207             /**
2208              * Skip unless the entry is marked as being deleted
2209              * or the data has already been resurrected.
2210              */
2211             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2212                 continue;
2213
2214             /* Now try to match the entry. */
2215             /**
2216              * To succesfully match a song, the following conditions
2217              * must apply:
2218              *
2219              * For numeric fields: tag_length
2220              * - Full identical match is required
2221              *
2222              * If tag_filename matches, no further checking necessary.
2223              *
2224              * For string hashes: tag_artist, tag_album, tag_title
2225              * - Two of these must match
2226              */
2227             for (j = 0; j < count; j++)
2228             {
2229                 struct temp_file_entry *tfe = &entrybuf[j];
2230
2231                 /* Try to match numeric fields first. */
2232                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2233                     continue;
2234
2235                 /* Now it's time to do the hash matching. */
2236                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2237                 {
2238                     int match_count = 0;
2239
2240                     /* No filename match, check if we can match two other tags. */
2241 #define tmpdb_match(tag) \
2242     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2243         match_count++
2244
2245                     tmpdb_match(tag_artist);
2246                     tmpdb_match(tag_album);
2247                     tmpdb_match(tag_title);
2248
2249                     if (match_count < 2)
2250                     {
2251                         /* Still no match found, give up. */
2252                         continue;
2253                     }
2254                 }
2255
2256                 /* A match found, now copy & resurrect the statistical data. */
2257 #define tmpdb_copy_tag(tag) \
2258     tfe->tag_offset[tag] = idx.tag_seek[tag]
2259
2260                 tmpdb_copy_tag(tag_playcount);
2261                 tmpdb_copy_tag(tag_rating);
2262                 tmpdb_copy_tag(tag_playtime);
2263                 tmpdb_copy_tag(tag_lastplayed);
2264                 tmpdb_copy_tag(tag_commitid);
2265
2266                 /* Avoid processing this entry again. */
2267                 idx.flag |= FLAG_RESURRECTED;
2268
2269                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2270                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2271                 {
2272                     logf("masterfd writeback fail #1");
2273                     close(masterfd);
2274                     return false;
2275                 }
2276
2277                 logf("Entry resurrected");
2278             }
2279         }
2280
2281
2282         /* Restore the master index position. */
2283         lseek(masterfd, masterfd_pos, SEEK_SET);
2284
2285         /* Commit the data to the index. */
2286         for (i = 0; i < count; i++)
2287         {
2288             int loc = lseek(masterfd, 0, SEEK_CUR);
2289
2290             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2291             {
2292                 logf("read fail #3");
2293                 close(masterfd);
2294                 return false;
2295             }
2296
2297             for (j = 0; j < TAG_COUNT; j++)
2298             {
2299                 if (!TAGCACHE_IS_NUMERIC(j))
2300                     continue;
2301
2302                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2303             }
2304             idx.flag = entrybuf[i].flag;
2305
2306             if (idx.tag_seek[tag_commitid])
2307             {
2308                 /* Data has been resurrected. */
2309                 idx.flag |= FLAG_DIRTYNUM;
2310             }
2311             else if (tc_stat.ready && current_tcmh.commitid > 0)
2312             {
2313                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2314                 idx.flag |= FLAG_DIRTYNUM;
2315             }
2316
2317             /* Write back the updated index. */
2318             lseek(masterfd, loc, SEEK_SET);
2319             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2320             {
2321                 logf("write fail");
2322                 close(masterfd);
2323                 return false;
2324             }
2325         }
2326
2327         entries_processed += count;
2328         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2329     }
2330
2331     close(masterfd);
2332
2333     return true;
2334 }
2335
2336 /**
2337  * Return values:
2338  *     > 0   success
2339  *    == 0   temporary failure
2340  *     < 0   fatal error
2341  */
2342 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2343 {
2344     int i;
2345     struct tagcache_header tch;
2346     struct master_header   tcmh;
2347     struct index_entry idxbuf[IDX_BUF_DEPTH];
2348     int idxbuf_pos;
2349     char buf[TAG_MAXLEN+32];
2350     int fd = -1, masterfd;
2351     bool error = false;
2352     int init;
2353     int masterfd_pos;
2354
2355     logf("Building index: %d", index_type);
2356
2357     /* Check the number of entries we need to allocate ram for. */
2358     commit_entry_count = h->entry_count + 1;
2359
2360     masterfd = open_master_fd(&tcmh, false);
2361     if (masterfd >= 0)
2362     {
2363         commit_entry_count += tcmh.tch.entry_count;
2364         close(masterfd);
2365     }
2366     else
2367         remove_files(); /* Just to be sure we are clean. */
2368
2369     /* Open the index file, which contains the tag names. */
2370     fd = open_tag_fd(&tch, index_type, true);
2371     if (fd >= 0)
2372     {
2373         logf("tch.datasize=%ld", tch.datasize);
2374         lookup_buffer_depth = 1 +
2375         /* First part */ commit_entry_count +
2376         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2377     }
2378     else
2379     {
2380         lookup_buffer_depth = 1 +
2381         /* First part */ commit_entry_count +
2382         /* Second part */ 0;
2383     }
2384
2385     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2386     logf("commit_entry_count=%ld", commit_entry_count);
2387
2388     /* Allocate buffer for all index entries from both old and new
2389      * tag files. */
2390     tempbufidx = 0;
2391     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2392
2393     /* Allocate lookup buffer. The first portion of commit_entry_count
2394      * contains the new tags in the temporary file and the second
2395      * part for locating entries already in the db.
2396      *
2397      *  New tags  Old tags
2398      * +---------+---------------------------+
2399      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2400      * +---------+---------------------------+
2401      *
2402      * Old tags are inserted to a temporary buffer with position:
2403      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2404      * And new tags with index:
2405      *     tempbuf_insert(idx, ...);
2406      *
2407      * The buffer is sorted and written into tag file:
2408      *     tempbuf_sort(...);
2409      * leaving master index locations messed up.
2410      *
2411      * That is fixed using the lookup buffer for old tags:
2412      *     new_seek = tempbuf_find_location(old_seek, ...);
2413      * and for new tags:
2414      *     new_seek = tempbuf_find_location(idx);
2415      */
2416     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2417     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2418     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2419
2420     /* And calculate the remaining data space used mainly for storing
2421      * tag data (strings). */
2422     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2423     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2424     {
2425         logf("Buffer way too small!");
2426         return 0;
2427     }
2428
2429     if (fd >= 0)
2430     {
2431         /**
2432          * If tag file contains unique tags (sorted index), we will load
2433          * it entirely into memory so we can resort it later for use with
2434          * chunked browsing.
2435          */
2436         if (TAGCACHE_IS_SORTED(index_type))
2437         {
2438             logf("loading tags...");
2439             for (i = 0; i < tch.entry_count; i++)
2440             {
2441                 struct tagfile_entry entry;
2442                 int loc = lseek(fd, 0, SEEK_CUR);
2443                 bool ret;
2444
2445                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2446                 {
2447                     logf("read error #7");
2448                     close(fd);
2449                     return -2;
2450                 }
2451
2452                 if (entry.tag_length >= (int)sizeof(buf))
2453                 {
2454                     logf("too long tag #3");
2455                     close(fd);
2456                     return -2;
2457                 }
2458
2459                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2460                 {
2461                     logf("read error #8");
2462                     close(fd);
2463                     return -2;
2464                 }
2465
2466                 /* Skip deleted entries. */
2467                 if (buf[0] == '\0')
2468                     continue;
2469
2470                 /**
2471                  * Save the tag and tag id in the memory buffer. Tag id
2472                  * is saved so we can later reindex the master lookup
2473                  * table when the index gets resorted.
2474                  */
2475                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2476                                      + commit_entry_count, entry.idx_id,
2477                                      TAGCACHE_IS_UNIQUE(index_type));
2478                 if (!ret)
2479                 {
2480                     close(fd);
2481                     return -3;
2482                 }
2483                 do_timed_yield();
2484             }
2485             logf("done");
2486         }
2487         else
2488             tempbufidx = tch.entry_count;
2489     }
2490     else
2491     {
2492         /**
2493          * Creating new index file to store the tags. No need to preload
2494          * anything whether the index type is sorted or not.
2495          */
2496         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2497         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2498         if (fd < 0)
2499         {
2500             logf("%s open fail", buf);
2501             return -2;
2502         }
2503
2504         tch.magic = TAGCACHE_MAGIC;
2505         tch.entry_count = 0;
2506         tch.datasize = 0;
2507
2508         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2509             != sizeof(struct tagcache_header))
2510         {
2511             logf("header write failed");
2512             close(fd);
2513             return -2;
2514         }
2515     }
2516
2517     /* Loading the tag lookup file as "master file". */
2518     logf("Loading index file");
2519     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2520
2521     if (masterfd < 0)
2522     {
2523         logf("Creating new DB");
2524         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2525
2526         if (masterfd < 0)
2527         {
2528             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2529             close(fd);
2530             return -2;
2531         }
2532
2533         /* Write the header (write real values later). */
2534         memset(&tcmh, 0, sizeof(struct master_header));
2535         tcmh.tch = *h;
2536         tcmh.tch.entry_count = 0;
2537         tcmh.tch.datasize = 0;
2538         tcmh.dirty = true;
2539         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2540         init = true;
2541         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2542     }
2543     else
2544     {
2545         /**
2546          * Master file already exists so we need to process the current
2547          * file first.
2548          */
2549         init = false;
2550
2551         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2552             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2553         {
2554             logf("header error");
2555             close(fd);
2556             close(masterfd);
2557             return -2;
2558         }
2559
2560         /**
2561          * If we reach end of the master file, we need to expand it to
2562          * hold new tags. If the current index is not sorted, we can
2563          * simply append new data to end of the file.
2564          * However, if the index is sorted, we need to update all tag
2565          * pointers in the master file for the current index.
2566          */
2567         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2568             SEEK_CUR);
2569         if (masterfd_pos == filesize(masterfd))
2570         {
2571             logf("appending...");
2572             init = true;
2573         }
2574     }
2575
2576     /**
2577      * Load new unique tags in memory to be sorted later and added
2578      * to the master lookup file.
2579      */
2580     if (TAGCACHE_IS_SORTED(index_type))
2581     {
2582         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2583         /* h is the header of the temporary file containing new tags. */
2584         logf("inserting new tags...");
2585         for (i = 0; i < h->entry_count; i++)
2586         {
2587             struct temp_file_entry entry;
2588
2589             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2590                 sizeof(struct temp_file_entry))
2591             {
2592                 logf("read fail #3");
2593                 error = true;
2594                 goto error_exit;
2595             }
2596
2597             /* Read data. */
2598             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2599             {
2600                 logf("too long entry!");
2601                 error = true;
2602                 goto error_exit;
2603             }
2604
2605             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2606             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2607                 entry.tag_length[index_type])
2608             {
2609                 logf("read fail #4");
2610                 error = true;
2611                 goto error_exit;
2612             }
2613
2614             if (TAGCACHE_IS_UNIQUE(index_type))
2615                 error = !tempbuf_insert(buf, i, -1, true);
2616             else
2617                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2618
2619             if (error)
2620             {
2621                 logf("insert error");
2622                 goto error_exit;
2623             }
2624
2625             /* Skip to next. */
2626             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2627                     entry.tag_length[index_type], SEEK_CUR);
2628             do_timed_yield();
2629         }
2630         logf("done");
2631
2632         /* Sort the buffer data and write it to the index file. */
2633         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2634         /**
2635          * We need to truncate the index file now. There can be junk left
2636          * at the end of file (however, we _should_ always follow the
2637          * entry_count and don't crash with that).
2638          */
2639         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2640
2641         i = tempbuf_sort(fd);
2642         if (i < 0)
2643             goto error_exit;
2644         logf("sorted %d tags", i);
2645
2646         /**
2647          * Now update all indexes in the master lookup file.
2648          */
2649         logf("updating indices...");
2650         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2651         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2652         {
2653             int j;
2654             int loc = lseek(masterfd, 0, SEEK_CUR);
2655
2656             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2657
2658             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2659                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2660             {
2661                 logf("read fail #5");
2662                 error = true;
2663                 goto error_exit ;
2664             }
2665             lseek(masterfd, loc, SEEK_SET);
2666
2667             for (j = 0; j < idxbuf_pos; j++)
2668             {
2669                 if (idxbuf[j].flag & FLAG_DELETED)
2670                 {
2671                     /* We can just ignore deleted entries. */
2672                     // idxbuf[j].tag_seek[index_type] = 0;
2673                     continue;
2674                 }
2675
2676                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2677                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2678                     + commit_entry_count);
2679
2680                 if (idxbuf[j].tag_seek[index_type] < 0)
2681                 {
2682                     logf("update error: %ld/%d/%ld",
2683                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2684                     error = true;
2685                     goto error_exit;
2686                 }
2687
2688                 do_timed_yield();
2689             }
2690
2691             /* Write back the updated index. */
2692             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2693                         index_entry_ec, tc_stat.econ) !=
2694                 (int)sizeof(struct index_entry)*idxbuf_pos)
2695             {
2696                 logf("write fail");
2697                 error = true;
2698                 goto error_exit;
2699             }
2700         }
2701         logf("done");
2702     }
2703
2704     /**
2705      * Walk through the temporary file containing the new tags.
2706      */
2707     // build_normal_index(h, tmpfd, masterfd, idx);
2708     logf("updating new indices...");
2709     lseek(masterfd, masterfd_pos, SEEK_SET);
2710     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2711     lseek(fd, 0, SEEK_END);
2712     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2713     {
2714         int j;
2715
2716         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2717         if (init)
2718         {
2719             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2720         }
2721         else
2722         {
2723             int loc = lseek(masterfd, 0, SEEK_CUR);
2724
2725             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2726                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2727             {
2728                 logf("read fail #6");
2729                 error = true;
2730                 break ;
2731             }
2732             lseek(masterfd, loc, SEEK_SET);
2733         }
2734
2735         /* Read entry headers. */
2736         for (j = 0; j < idxbuf_pos; j++)
2737         {
2738             if (!TAGCACHE_IS_SORTED(index_type))
2739             {
2740                 struct temp_file_entry entry;
2741                 struct tagfile_entry fe;
2742
2743                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2744                     sizeof(struct temp_file_entry))
2745                 {
2746                     logf("read fail #7");
2747                     error = true;
2748                     break ;
2749                 }
2750
2751                 /* Read data. */
2752                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2753                 {
2754                     logf("too long entry!");
2755                     logf("length=%d", entry.tag_length[index_type]);
2756                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2757                     error = true;
2758                     break ;
2759                 }
2760
2761                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2762                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2763                     entry.tag_length[index_type])
2764                 {
2765                     logf("read fail #8");
2766                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2767                     logf("length=0x%02x", entry.tag_length[index_type]);
2768                     error = true;
2769                     break ;
2770                 }
2771
2772                 /* Write to index file. */
2773                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2774                 fe.tag_length = entry.tag_length[index_type];
2775                 fe.idx_id = tcmh.tch.entry_count + i + j;
2776                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2777                 write(fd, buf, fe.tag_length);
2778                 tempbufidx++;
2779
2780                 /* Skip to next. */
2781                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2782                       entry.tag_length[index_type], SEEK_CUR);
2783             }
2784             else
2785             {
2786                 /* Locate the correct entry from the sorted array. */
2787                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2788                 if (idxbuf[j].tag_seek[index_type] < 0)
2789                 {
2790                     logf("entry not found (%d)", j);
2791                     error = true;
2792                     break ;
2793                 }
2794             }
2795         }
2796
2797         /* Write index. */
2798         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2799                   index_entry_ec, tc_stat.econ) !=
2800             (int)sizeof(struct index_entry)*idxbuf_pos)
2801         {
2802             logf("tagcache: write fail #4");
2803             error = true;
2804             break ;
2805         }
2806
2807         do_timed_yield();
2808     }
2809     logf("done");
2810
2811     /* Finally write the header. */
2812     tch.magic = TAGCACHE_MAGIC;
2813     tch.entry_count = tempbufidx;
2814     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2815     lseek(fd, 0, SEEK_SET);
2816     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2817
2818     if (index_type != tag_filename)
2819         h->datasize += tch.datasize;
2820     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2821     error_exit:
2822
2823     close(fd);
2824     close(masterfd);
2825
2826     if (error)
2827         return -2;
2828
2829     return 1;
2830 }
2831
2832 static bool commit(void)
2833 {
2834     struct tagcache_header tch;
2835     struct master_header   tcmh;
2836     int i, len, rc;
2837     int tmpfd;
2838     int masterfd;
2839 #ifdef HAVE_DIRCACHE
2840     bool dircache_buffer_stolen = false;
2841 #endif
2842     bool local_allocation = false;
2843
2844     logf("committing tagcache");
2845
2846     while (write_lock)
2847         sleep(1);
2848
2849     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2850     if (tmpfd < 0)
2851     {
2852         logf("nothing to commit");
2853         return true;
2854     }
2855
2856
2857     /* Load the header. */
2858     len = sizeof(struct tagcache_header);
2859     rc = read(tmpfd, &tch, len);
2860
2861     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2862     {
2863         logf("incorrect header");
2864         close(tmpfd);
2865         remove(TAGCACHE_FILE_TEMP);
2866         return false;
2867     }
2868
2869     if (tch.entry_count == 0)
2870     {
2871         logf("nothing to commit");
2872         close(tmpfd);
2873         remove(TAGCACHE_FILE_TEMP);
2874         return true;
2875     }
2876
2877 #ifdef HAVE_EEPROM_SETTINGS
2878     remove(TAGCACHE_STATEFILE);
2879 #endif
2880
2881     /* At first be sure to unload the ramcache! */
2882 #ifdef HAVE_TC_RAMCACHE
2883     tc_stat.ramcache = false;
2884 #endif
2885
2886     read_lock++;
2887
2888     /* Try to steal every buffer we can :) */
2889     if (tempbuf_size == 0)
2890         local_allocation = true;
2891
2892 #ifdef HAVE_DIRCACHE
2893     if (tempbuf_size == 0)
2894     {
2895         /* Try to steal the dircache buffer. */
2896         tempbuf = dircache_steal_buffer(&tempbuf_size);
2897         tempbuf_size &= ~0x03;
2898
2899         if (tempbuf_size > 0)
2900         {
2901             dircache_buffer_stolen = true;
2902         }
2903     }
2904 #endif
2905
2906 #ifdef HAVE_TC_RAMCACHE
2907     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2908     {
2909         tempbuf = (char *)(hdr + 1);
2910         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2911         tempbuf_size &= ~0x03;
2912     }
2913 #endif
2914
2915     /* And finally fail if there are no buffers available. */
2916     if (tempbuf_size == 0)
2917     {
2918         logf("delaying commit until next boot");
2919         tc_stat.commit_delayed = true;
2920         close(tmpfd);
2921         read_lock--;
2922         return false;
2923     }
2924
2925     logf("commit %ld entries...", tch.entry_count);
2926
2927     /* Mark DB dirty so it will stay disabled if commit fails. */
2928     current_tcmh.dirty = true;
2929     update_master_header();
2930
2931     /* Now create the index files. */
2932     tc_stat.commit_step = 0;
2933     tch.datasize = 0;
2934     tc_stat.commit_delayed = false;
2935
2936     for (i = 0; i < TAG_COUNT; i++)
2937     {
2938         int ret;
2939
2940         if (TAGCACHE_IS_NUMERIC(i))
2941             continue;
2942
2943         tc_stat.commit_step++;
2944         ret = build_index(i, &tch, tmpfd);
2945         if (ret <= 0)
2946         {
2947             close(tmpfd);
2948             logf("tagcache failed init");
2949             if (ret == 0)
2950                 tc_stat.commit_delayed = true;
2951
2952             tc_stat.commit_step = 0;
2953             read_lock--;
2954             return false;
2955         }
2956     }
2957
2958     if (!build_numeric_indices(&tch, tmpfd))
2959     {
2960         logf("Failure to commit numeric indices");
2961         close(tmpfd);
2962         tc_stat.commit_step = 0;
2963         read_lock--;
2964         return false;
2965     }
2966
2967     close(tmpfd);
2968     remove(TAGCACHE_FILE_TEMP);
2969
2970     tc_stat.commit_step = 0;
2971
2972     /* Update the master index headers. */
2973     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2974     {
2975         read_lock--;
2976         return false;
2977     }
2978
2979     tcmh.tch.entry_count += tch.entry_count;
2980     tcmh.tch.datasize = sizeof(struct master_header)
2981         + sizeof(struct index_entry) * tcmh.tch.entry_count
2982         + tch.datasize;
2983     tcmh.dirty = false;
2984     tcmh.commitid++;
2985
2986     lseek(masterfd, 0, SEEK_SET);
2987     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2988     close(masterfd);
2989
2990     logf("tagcache committed");
2991     tc_stat.ready = check_all_headers();
2992     tc_stat.readyvalid = true;
2993
2994     if (local_allocation)
2995     {
2996         tempbuf = NULL;
2997         tempbuf_size = 0;
2998     }
2999
3000 #ifdef HAVE_DIRCACHE
3001     /* Rebuild the dircache, if we stole the buffer. */
3002     if (dircache_buffer_stolen)
3003         dircache_build(0);
3004 #endif
3005
3006 #ifdef HAVE_TC_RAMCACHE
3007     /* Reload tagcache. */
3008     if (tc_stat.ramcache_allocated > 0)
3009         tagcache_start_scan();
3010 #endif
3011
3012     read_lock--;
3013
3014     return true;
3015 }
3016
3017 static void allocate_tempbuf(void)
3018 {
3019     /* Yeah, malloc would be really nice now :) */
3020 #ifdef __PCTOOL__
3021     tempbuf_size = 32*1024*1024;
3022     tempbuf = malloc(tempbuf_size);
3023 #else
3024     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3025     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3026     audiobuf += tempbuf_size;
3027 #endif
3028 }
3029
3030 static void free_tempbuf(void)
3031 {
3032     if (tempbuf_size == 0)
3033         return ;
3034
3035 #ifdef __PCTOOL__
3036     free(tempbuf);
3037 #else
3038     audiobuf -= tempbuf_size;
3039 #endif
3040     tempbuf = NULL;
3041     tempbuf_size = 0;
3042 }
3043
3044 #ifndef __PCTOOL__
3045
3046 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3047 {
3048     struct index_entry idx;
3049
3050     if (!tc_stat.ready)
3051         return false;
3052
3053     if (!TAGCACHE_IS_NUMERIC(tag))
3054         return false;
3055
3056     if (!get_index(masterfd, idx_id, &idx, false))
3057         return false;
3058
3059     idx.tag_seek[tag] = data;
3060     idx.flag |= FLAG_DIRTYNUM;
3061
3062     return write_index(masterfd, idx_id, &idx);
3063 }
3064
3065 #if 0
3066 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3067                                    int tag, long data)
3068 {
3069     struct master_header myhdr;
3070
3071     if (tcs->masterfd < 0)
3072     {
3073         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3074             return false;
3075     }
3076
3077     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3078 }
3079 #endif
3080
3081 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3082
3083 static bool command_queue_is_full(void)
3084 {
3085     int next;
3086
3087     next = command_queue_widx + 1;
3088     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3089         next = 0;
3090
3091     return (next == command_queue_ridx);
3092 }
3093
3094 static void command_queue_sync_callback(void *data)
3095 {
3096     (void)data;
3097     struct master_header myhdr;
3098     int masterfd;
3099
3100     mutex_lock(&command_queue_mutex);
3101
3102     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3103         return;
3104
3105     while (command_queue_ridx != command_queue_widx)
3106     {
3107         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3108
3109         switch (ce->command)
3110         {
3111             case CMD_UPDATE_MASTER_HEADER:
3112             {
3113                 close(masterfd);
3114                 update_master_header();
3115
3116                 /* Re-open the masterfd. */
3117                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3118                     return;
3119
3120                 break;
3121             }
3122             case CMD_UPDATE_NUMERIC:
3123             {
3124                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3125                 break;
3126             }
3127         }
3128
3129         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3130             command_queue_ridx = 0;
3131     }
3132
3133     close(masterfd);
3134
3135     tc_stat.queue_length = 0;
3136     mutex_unlock(&command_queue_mutex);
3137 }
3138
3139 static void run_command_queue(bool force)
3140 {
3141     if (COMMAND_QUEUE_IS_EMPTY)
3142         return;
3143
3144     if (force || command_queue_is_full())
3145         command_queue_sync_callback(NULL);
3146     else
3147         register_storage_idle_func(command_queue_sync_callback);
3148 }
3149
3150 static void queue_command(int cmd, long idx_id, int tag, long data)
3151 {
3152     while (1)
3153     {
3154         int next;
3155
3156         mutex_lock(&command_queue_mutex);
3157         next = command_queue_widx + 1;
3158         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3159             next = 0;
3160
3161         /* Make sure queue is not full. */
3162         if (next != command_queue_ridx)
3163         {
3164             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3165
3166             ce->command = cmd;
3167             ce->idx_id = idx_id;
3168             ce->tag = tag;
3169             ce->data = data;
3170
3171             command_queue_widx = next;
3172
3173             tc_stat.queue_length++;
3174
3175             mutex_unlock(&command_queue_mutex);
3176             break;
3177         }
3178
3179         /* Queue is full, try again later... */
3180         mutex_unlock(&command_queue_mutex);
3181         sleep(1);
3182     }
3183 }
3184
3185 long tagcache_increase_serial(void)
3186 {
3187     long old;
3188
3189     if (!tc_stat.ready)
3190         return -2;
3191
3192     while (read_lock)
3193         sleep(1);
3194
3195     old = current_tcmh.serial++;
3196     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3197
3198     return old;
3199 }
3200
3201 void tagcache_update_numeric(int idx_id, int tag, long data)
3202 {
3203     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3204 }
3205 #endif /* !__PCTOOL__ */
3206
3207 long tagcache_get_serial(void)
3208 {
3209     return current_tcmh.serial;
3210 }
3211
3212 long tagcache_get_commitid(void)
3213 {
3214     return current_tcmh.commitid;
3215 }
3216
3217 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3218 {
3219     char buf[512];
3220     int i;
3221
3222     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3223     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3224     {
3225         if (*datastr == '\0')
3226             break;
3227
3228         if (*datastr == '"' || *datastr == '\\')
3229             buf[i++] = '\\';
3230
3231         buf[i] = *(datastr++);
3232     }
3233
3234     strcpy(&buf[i], "\" ");
3235
3236     write(fd, buf, i + 2);
3237
3238     return true;
3239 }
3240
3241 #ifndef __PCTOOL__
3242
3243 static bool read_tag(char *dest, long size,
3244                      const char *src, const char *tagstr)
3245 {
3246     int pos;
3247     char current_tag[32];
3248
3249     while (*src != '\0')
3250     {
3251         /* Skip all whitespace */
3252         while (*src == ' ')
3253             src++;
3254
3255         if (*src == '\0')
3256             break;
3257
3258         pos = 0;
3259         /* Read in tag name */
3260         while (*src != '=' && *src != ' ')
3261         {
3262             current_tag[pos] = *src;
3263             src++;
3264             pos++;
3265
3266             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3267                 return false;
3268         }
3269         current_tag[pos] = '\0';
3270
3271         /* Read in tag data */
3272
3273         /* Find the start. */
3274         while (*src != '"' && *src != '\0')
3275             src++;
3276
3277         if (*src == '\0' || *(++src) == '\0')
3278             return false;
3279
3280         /* Read the data. */
3281         for (pos = 0; pos < size; pos++)
3282         {
3283             if (*src == '\0')
3284                 break;
3285
3286             if (*src == '\\')
3287             {
3288                 dest[pos] = *(src+1);
3289                 src += 2;
3290                 continue;
3291             }
3292
3293             dest[pos] = *src;
3294
3295             if (*src == '"')
3296             {
3297                 src++;
3298                 break;
3299             }
3300
3301             if (*src == '\0')
3302                 break;
3303
3304             src++;
3305         }
3306         dest[pos] = '\0';
3307
3308         if (!strcasecmp(tagstr, current_tag))
3309             return true;
3310     }
3311
3312     return false;
3313 }
3314
3315 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3316 {
3317     struct index_entry idx;
3318     char tag_data[TAG_MAXLEN+32];
3319     int idx_id;
3320     long masterfd = (long)parameters;
3321     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3322         tag_commitid };
3323     int i;
3324     (void)line_n;
3325
3326     if (*buf == '#')
3327         return 0;
3328
3329     logf("%d/%s", line_n, buf);
3330     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3331     {
3332         logf("filename missing");
3333         logf("-> %s", buf);
3334         return 0;
3335     }
3336
3337     idx_id = find_index(tag_data);
3338     if (idx_id < 0)
3339     {
3340         logf("entry not found");
3341         return 0;
3342     }
3343
3344     if (!get_index(masterfd, idx_id, &idx, false))
3345     {
3346         logf("failed to retrieve index entry");
3347         return 0;
3348     }
3349
3350     /* Stop if tag has already been modified. */
3351     if (idx.flag & FLAG_DIRTYNUM)
3352         return 0;
3353
3354     logf("import: %s", tag_data);
3355
3356     idx.flag |= FLAG_DIRTYNUM;
3357     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3358     {
3359         int data;
3360
3361         if (!read_tag(tag_data, sizeof tag_data, buf,
3362                       tagcache_tag_to_str(import_tags[i])))
3363         {
3364             continue;
3365         }
3366
3367         data = atoi(tag_data);
3368         if (data < 0)
3369             continue;
3370
3371         idx.tag_seek[import_tags[i]] = data;
3372
3373         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3374             current_tcmh.serial = data + 1;
3375         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3376             current_tcmh.commitid = data + 1;
3377     }
3378
3379     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3380 }
3381
3382 bool tagcache_import_changelog(void)
3383 {
3384     struct master_header myhdr;
3385     struct tagcache_header tch;
3386     int clfd;
3387     long masterfd;
3388     char buf[2048];
3389
3390     if (!tc_stat.ready)
3391         return false;
3392
3393     while (read_lock)
3394         sleep(1);
3395
3396     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3397     if (clfd < 0)
3398     {
3399         logf("failure to open changelog");
3400         return false;
3401     }
3402
3403     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3404     {
3405         close(clfd);
3406         return false;
3407     }
3408
3409     write_lock++;
3410
3411     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3412
3413     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3414                   parse_changelog_line);
3415
3416     close(clfd);
3417     close(masterfd);
3418
3419     if (filenametag_fd >= 0)
3420     {
3421         close(filenametag_fd);
3422         filenametag_fd = -1;
3423     }
3424
3425     write_lock--;
3426
3427     update_master_header();
3428
3429     return true;
3430 }
3431
3432 #endif /* !__PCTOOL__ */
3433
3434 bool tagcache_create_changelog(struct tagcache_search *tcs)
3435 {
3436     struct master_header myhdr;
3437     struct index_entry idx;
3438     char buf[TAG_MAXLEN+32];
3439     char temp[32];
3440     int clfd;
3441     int i, j;
3442
3443     if (!tc_stat.ready)
3444         return false;
3445
3446     if (!tagcache_search(tcs, tag_filename))
3447         return false;
3448
3449     /* Initialize the changelog */
3450     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3451     if (clfd < 0)
3452     {
3453         logf("failure to open changelog");
3454         return false;
3455     }
3456
3457     if (tcs->masterfd < 0)
3458     {
3459         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3460             return false;
3461     }
3462     else
3463     {
3464         lseek(tcs->masterfd, 0, SEEK_SET);
3465         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3466     }
3467
3468     write(clfd, "## Changelog version 1\n", 23);
3469
3470     for (i = 0; i < myhdr.tch.entry_count; i++)
3471     {
3472         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3473         {
3474             logf("read error #9");
3475             tagcache_search_finish(tcs);
3476             close(clfd);
3477             return false;
3478         }
3479
3480         /* Skip until the entry found has been modified. */
3481         if (! (idx.flag & FLAG_DIRTYNUM) )
3482             continue;
3483
3484         /* Skip deleted entries too. */
3485         if (idx.flag & FLAG_DELETED)
3486             continue;
3487
3488         /* Now retrieve all tags. */
3489         for (j = 0; j < TAG_COUNT; j++)
3490         {
3491             if (TAGCACHE_IS_NUMERIC(j))
3492             {
3493                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3494                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3495                 continue;
3496             }
3497
3498             tcs->type = j;
3499             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3500             write_tag(clfd, tagcache_tag_to_str(j), buf);
3501         }
3502
3503         write(clfd, "\n", 1);
3504         do_timed_yield();
3505     }
3506
3507     close(clfd);
3508
3509     tagcache_search_finish(tcs);
3510
3511     return true;
3512 }
3513
3514 static bool delete_entry(long idx_id)
3515 {
3516     int fd = -1;
3517     int masterfd = -1;
3518     int tag, i;
3519     struct index_entry idx, myidx;
3520     struct master_header myhdr;
3521     char buf[TAG_MAXLEN+32];
3522     int in_use[TAG_COUNT];
3523
3524     logf("delete_entry(): %ld", idx_id);
3525
3526 #ifdef HAVE_TC_RAMCACHE
3527     /* At first mark the entry removed from ram cache. */
3528     if (tc_stat.ramcache)
3529         hdr->indices[idx_id].flag |= FLAG_DELETED;
3530 #endif
3531
3532     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3533         return false;
3534
3535     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3536     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3537     {
3538         logf("delete_entry(): read error");
3539         goto cleanup;
3540     }
3541
3542     if (myidx.flag & FLAG_DELETED)
3543     {
3544         logf("delete_entry(): already deleted!");
3545         goto cleanup;
3546     }
3547
3548     myidx.flag |= FLAG_DELETED;
3549     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3550     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3551     {
3552         logf("delete_entry(): write_error #1");
3553         goto cleanup;
3554     }
3555
3556     /* Now check which tags are no longer in use (if any) */
3557     for (tag = 0; tag < TAG_COUNT; tag++)
3558         in_use[tag] = 0;
3559
3560     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3561     for (i = 0; i < myhdr.tch.entry_count; i++)
3562     {
3563         struct index_entry *idxp;
3564
3565 #ifdef HAVE_TC_RAMCACHE
3566         /* Use RAM DB if available for greater speed */
3567         if (tc_stat.ramcache)
3568             idxp = &hdr->indices[i];
3569         else
3570 #endif
3571         {
3572             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3573             {
3574                 logf("delete_entry(): read error #2");
3575                 goto cleanup;
3576             }
3577             idxp = &idx;
3578         }
3579
3580         if (idxp->flag & FLAG_DELETED)
3581             continue;
3582
3583         for (tag = 0; tag < TAG_COUNT; tag++)
3584         {
3585             if (TAGCACHE_IS_NUMERIC(tag))
3586                 continue;
3587
3588             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3589                 in_use[tag]++;
3590         }
3591     }
3592
3593     /* Now delete all tags no longer in use. */
3594     for (tag = 0; tag < TAG_COUNT; tag++)
3595     {
3596         struct tagcache_header tch;
3597         int oldseek = myidx.tag_seek[tag];
3598
3599         if (TAGCACHE_IS_NUMERIC(tag))
3600             continue;
3601
3602         /**
3603          * Replace tag seek with a hash value of the field string data.
3604          * That way runtime statistics of moved or altered files can be
3605          * resurrected.
3606          */
3607 #ifdef HAVE_TC_RAMCACHE
3608         if (tc_stat.ramcache && tag != tag_filename)
3609         {
3610             struct tagfile_entry *tfe;
3611             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3612
3613             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3614             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3615             myidx.tag_seek[tag] = *seek;
3616         }
3617         else
3618 #endif
3619         {
3620             struct tagfile_entry tfe;
3621
3622             /* Open the index file, which contains the tag names. */
3623             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3624                 goto cleanup;
3625
3626             /* Skip the header block */
3627             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3628             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3629             {
3630                 logf("delete_entry(): read error #3");
3631                 goto cleanup;
3632             }
3633
3634             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3635             {
3636                 logf("delete_entry(): read error #4");
3637                 goto cleanup;
3638             }
3639
3640             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3641         }
3642
3643         if (in_use[tag])
3644         {
3645             logf("in use: %d/%d", tag, in_use[tag]);
3646             if (fd >= 0)
3647             {
3648                 close(fd);
3649                 fd = -1;
3650             }
3651             continue;
3652         }
3653
3654 #ifdef HAVE_TC_RAMCACHE
3655         /* Delete from ram. */
3656         if (tc_stat.ramcache && tag != tag_filename)
3657         {
3658             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3659             tagentry->tag_data[0] = '\0';
3660         }
3661 #endif
3662
3663         /* Open the index file, which contains the tag names. */
3664         if (fd < 0)
3665         {
3666             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3667                 goto cleanup;
3668         }
3669
3670         /* Skip the header block */
3671         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3672
3673         /* Debug, print 10 first characters of the tag
3674         read(fd, buf, 10);
3675         buf[10]='\0';
3676         logf("TAG:%s", buf);
3677         lseek(fd, -10, SEEK_CUR);
3678         */
3679
3680         /* Write first data byte in tag as \0 */
3681         write(fd, "", 1);
3682
3683         /* Now tag data has been removed */
3684         close(fd);
3685         fd = -1;
3686     }
3687
3688     /* Write index entry back into master index. */
3689     lseek(masterfd, sizeof(struct master_header) +
3690           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3691     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3692     {
3693         logf("delete_entry(): write_error #2");
3694         goto cleanup;
3695     }
3696
3697     close(masterfd);
3698
3699     return true;
3700
3701     cleanup:
3702     if (fd >= 0)
3703         close(fd);
3704     if (masterfd >= 0)
3705         close(masterfd);
3706
3707     return false;
3708 }
3709
3710 #ifndef __PCTOOL__
3711 /**
3712  * Returns true if there is an event waiting in the queue
3713  * that requires the current operation to be aborted.
3714  */
3715 static bool check_event_queue(void)
3716 {
3717     struct queue_event ev;
3718
3719     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3720     switch (ev.id)
3721     {
3722         case Q_STOP_SCAN:
3723         case SYS_POWEROFF:
3724         case SYS_USB_CONNECTED:
3725             /* Put the event back into the queue. */
3726             queue_post(&tagcache_queue, ev.id, ev.data);
3727             return true;
3728     }
3729
3730     return false;
3731 }
3732 #endif
3733
3734 #ifdef HAVE_TC_RAMCACHE
3735 static bool allocate_tagcache(void)
3736 {
3737     struct master_header tcmh;
3738     int fd;
3739
3740     /* Load the header. */
3741     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3742     {
3743         hdr = NULL;
3744         return false;
3745     }
3746
3747     close(fd);
3748
3749     /**
3750      * Now calculate the required cache size plus
3751      * some extra space for alignment fixes.
3752      */
3753     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3754         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3755     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3756     memset(hdr, 0, sizeof(struct ramcache_header));
3757     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3758     hdr->indices = (struct index_entry *)(hdr + 1);
3759     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3760
3761     return true;
3762 }
3763
3764 # ifdef HAVE_EEPROM_SETTINGS
3765 static bool tagcache_dumpload(void)
3766 {
3767     struct statefile_header shdr;
3768     int fd, rc;
3769     long offpos;
3770     int i;
3771
3772     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3773     if (fd < 0)
3774     {
3775         logf("no tagcache statedump");
3776         return false;
3777     }
3778
3779     /* Check the statefile memory placement */
3780     hdr = buffer_alloc(0);
3781     rc = read(fd, &shdr, sizeof(struct statefile_header));
3782     if (rc != sizeof(struct statefile_header)
3783         /* || (long)hdr != (long)shdr.hdr */)
3784     {
3785         logf("incorrect statefile");
3786         hdr = NULL;
3787         close(fd);
3788         return false;
3789     }
3790
3791     offpos = (long)hdr - (long)shdr.hdr;
3792
3793     /* Lets allocate real memory and load it */
3794     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3795     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3796     close(fd);
3797
3798     if (rc != shdr.tc_stat.ramcache_allocated)
3799     {
3800         logf("read failure!");
3801         hdr = NULL;
3802         return false;
3803     }
3804
3805     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3806
3807     /* Now fix the pointers */
3808     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3809     for (i = 0; i < TAG_COUNT; i++)
3810         hdr->tags[i] += offpos;
3811
3812     return true;
3813 }
3814
3815 static bool tagcache_dumpsave(void)
3816 {
3817     struct statefile_header shdr;
3818     int fd;
3819
3820     if (!tc_stat.ramcache)
3821         return false;
3822
3823     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3824     if (fd < 0)
3825     {
3826         logf("failed to create a statedump");
3827         return false;
3828     }
3829
3830     /* Create the header */
3831     shdr.hdr = hdr;
3832     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3833     write(fd, &shdr, sizeof(struct statefile_header));
3834
3835     /* And dump the data too */
3836     write(fd, hdr, tc_stat.ramcache_allocated);
3837     close(fd);
3838
3839     return true;
3840 }
3841 # endif
3842
3843 static bool load_tagcache(void)
3844 {
3845     struct tagcache_header *tch;
3846     long bytesleft = tc_stat.ramcache_allocated;
3847     struct index_entry *idx;
3848     int rc, fd;
3849     char *p;
3850     int i, tag;
3851
3852 # ifdef HAVE_DIRCACHE
3853     while (dircache_is_initializing())
3854         sleep(1);
3855
3856     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3857 # endif
3858
3859     logf("loading tagcache to ram...");
3860
3861     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3862     if (fd < 0)
3863     {
3864         logf("tagcache open failed");
3865         return false;
3866     }
3867
3868     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3869         != sizeof(struct master_header)
3870         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3871     {
3872         logf("incorrect header");
3873         return false;
3874     }
3875
3876     idx = hdr->indices;
3877
3878     /* Load the master index table. */
3879     for (i = 0; i < hdr->h.tch.entry_count; i++)
3880     {
3881         rc = ecread_index_entry(fd, idx);
3882         if (rc != sizeof(struct index_entry))
3883         {
3884             logf("read error #10");
3885             close(fd);
3886             return false;
3887         }
3888
3889         bytesleft -= sizeof(struct index_entry);
3890         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3891         {
3892             logf("too big tagcache.");
3893             close(fd);
3894             return false;
3895         }
3896
3897         idx++;
3898     }
3899
3900     close(fd);
3901
3902     /* Load the tags. */
3903     p = (char *)idx;
3904     for (tag = 0; tag < TAG_COUNT; tag++)
3905     {
3906         struct tagfile_entry *fe;
3907         char buf[TAG_MAXLEN+32];
3908
3909         if (TAGCACHE_IS_NUMERIC(tag))
3910             continue ;
3911
3912         //p = ((void *)p+1);
3913         p = (char *)((long)p & ~0x03) + 0x04;
3914         hdr->tags[tag] = p;
3915
3916         /* Check the header. */
3917         tch = (struct tagcache_header *)p;
3918         p += sizeof(struct tagcache_header);
3919
3920         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3921             return false;
3922
3923         for (hdr->entry_count[tag] = 0;
3924              hdr->entry_count[tag] < tch->entry_count;
3925              hdr->entry_count[tag]++)
3926         {
3927             long pos;
3928
3929             if (do_timed_yield())
3930             {
3931                 /* Abort if we got a critical event in queue */
3932                 if (check_event_queue())
3933                     return false;
3934             }
3935
3936             fe = (struct tagfile_entry *)p;
3937             pos = lseek(fd, 0, SEEK_CUR);
3938             rc = ecread_tagfile_entry(fd, fe);
3939             if (rc != sizeof(struct tagfile_entry))
3940             {
3941                 /* End of lookup table. */
3942                 logf("read error #11");
3943                 close(fd);
3944                 return false;
3945             }
3946
3947             /* We have a special handling for the filename tags. */
3948             if (tag == tag_filename)
3949             {
3950 # ifdef HAVE_DIRCACHE
3951                 const struct dircache_entry *dc;
3952 # endif
3953
3954                 // FIXME: This is wrong!
3955                 // idx = &hdr->indices[hdr->entry_count[i]];
3956                 idx = &hdr->indices[fe->idx_id];
3957
3958                 if (fe->tag_length >= (long)sizeof(buf)-1)
3959                 {
3960                     read(fd, buf, 10);
3961                     buf[10] = '\0';
3962                     logf("TAG:%s", buf);
3963                     logf("too long filename");
3964                     close(fd);
3965                     return false;
3966                 }
3967
3968                 rc = read(fd, buf, fe->tag_length);
3969                 if (rc != fe->tag_length)
3970                 {
3971                     logf("read error #12");
3972                     close(fd);
3973                     return false;
3974                 }
3975
3976                 /* Check if the entry has already been removed */
3977                 if (idx->flag & FLAG_DELETED)
3978                     continue;
3979
3980                 /* This flag must not be used yet. */
3981                 if (idx->flag & FLAG_DIRCACHE)
3982                 {
3983                     logf("internal error!");
3984                     close(fd);
3985                     return false;
3986                 }
3987
3988                 if (idx->tag_seek[tag] != pos)
3989                 {
3990                     logf("corrupt data structures!");
3991                     close(fd);
3992                     return false;
3993                 }
3994
3995 # ifdef HAVE_DIRCACHE
3996                 if (dircache_is_enabled())
3997                 {
3998                     dc = dircache_get_entry_ptr(buf);
3999                     if (dc == NULL)
4000                     {
4001                         logf("Entry no longer valid.");
4002                         logf("-> %s", buf);
4003                         if (global_settings.tagcache_autoupdate)
4004                             delete_entry(fe->idx_id);
4005                         continue ;
4006                     }
4007
4008                     idx->flag |= FLAG_DIRCACHE;
4009                     idx->tag_seek[tag_filename] = (long)dc;
4010                 }
4011                 else
4012 # endif
4013                 {
4014                     /* This will be very slow unless dircache is enabled
4015                        or target is flash based, but do it anyway for
4016                        consistency. */
4017                     /* Check if entry has been removed. */
4018                     if (global_settings.tagcache_autoupdate)
4019                     {
4020                         if (!file_exists(buf))
4021                         {
4022                             logf("Entry no longer valid.");
4023                             logf("-> %s", buf);
4024                             delete_entry(fe->idx_id);
4025                             continue;
4026                         }
4027                     }
4028                 }
4029
4030                 continue ;
4031             }
4032
4033             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4034             if (bytesleft < 0)
4035             {
4036                 logf("too big tagcache #2");
4037                 logf("tl: %ld", fe->tag_length);
4038                 logf("bl: %ld", bytesleft);
4039                 close(fd);
4040                 return false;
4041             }
4042
4043             p = fe->tag_data;
4044             rc = read(fd, fe->tag_data, fe->tag_length);
4045             p += rc;
4046
4047             if (rc != fe->tag_length)
4048             {
4049                 logf("read error #13");
4050                 logf("rc=0x%04x", rc); // 0x431
4051                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4052                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4053                 logf("tag=0x%02x", tag); // 0x00
4054                 close(fd);
4055                 return false;
4056             }
4057         }
4058         close(fd);
4059     }
4060
4061     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4062     logf("tagcache loaded into ram!");
4063
4064     return true;
4065 }
4066 #endif /* HAVE_TC_RAMCACHE */
4067
4068 static bool check_deleted_files(void)
4069 {
4070     int fd;
4071     char buf[TAG_MAXLEN+32];
4072     struct tagfile_entry tfe;
4073
4074     logf("reverse scan...");
4075     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4076     fd = open(buf, O_RDONLY);
4077
4078     if (fd < 0)
4079     {
4080         logf("%s open fail", buf);
4081         return false;
4082     }
4083
4084     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4085     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4086 #ifndef __PCTOOL__
4087            && !check_event_queue()
4088 #endif
4089            )
4090     {
4091         if (tfe.tag_length >= (long)sizeof(buf)-1)
4092         {
4093             logf("too long tag");
4094             close(fd);
4095             return false;
4096         }
4097
4098         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4099         {
4100             logf("read error #14");
4101             close(fd);
4102             return false;
4103         }
4104
4105         /* Check if the file has already deleted from the db. */
4106         if (*buf == '\0')
4107             continue;
4108
4109         /* Now check if the file exists. */
4110         if (!file_exists(buf))
4111         {
4112             logf("Entry no longer valid.");
4113             logf("-> %s / %ld", buf, tfe.tag_length);
4114             delete_entry(tfe.idx_id);
4115         }
4116     }
4117
4118     close(fd);
4119
4120     logf("done");
4121
4122     return true;
4123 }
4124
4125
4126 /* Note that this function must not be inlined, otherwise the whole point
4127  * of having the code in a separate function is lost.
4128  */
4129 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4130     int *ignore, int *unignore)
4131 {
4132     char newpath[MAX_PATH];
4133
4134     /* check for a database.ignore file */
4135     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4136     *ignore = file_exists(newpath);
4137     /* check for a database.unignore file */
4138     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4139     *unignore = file_exists(newpath);
4140 }
4141
4142
4143 static bool check_dir(const char *dirname, int add_files)
4144 {
4145     DIR *dir;
4146     int len;
4147     int success = false;
4148     int ignore, unignore;
4149
4150     dir = opendir(dirname);
4151     if (!dir)
4152     {
4153         logf("tagcache: opendir(%s) failed", dirname);
4154         return false;
4155     }
4156
4157     /* check for a database.ignore and database.unignore */
4158     check_ignore(dirname, &ignore, &unignore);
4159
4160     /* don't do anything if both ignore and unignore are there */
4161     if (ignore != unignore)
4162         add_files = unignore;
4163
4164     /* Recursively scan the dir. */
4165 #ifdef __PCTOOL__
4166     while (1)
4167 #else
4168     while (!check_event_queue())
4169 #endif
4170     {
4171         struct dirent *entry;
4172
4173         entry = readdir(dir);
4174
4175         if (entry == NULL)
4176         {
4177             success = true;
4178             break ;
4179         }
4180
4181         if (!strcmp((char *)entry->d_name, ".") ||
4182             !strcmp((char *)entry->d_name, ".."))
4183             continue;
4184
4185         yield();
4186
4187         len = strlen(curpath);
4188         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4189                  entry->d_name);
4190
4191         processed_dir_count++;
4192         if (entry->attribute & ATTR_DIRECTORY)
4193             check_dir(curpath, add_files);
4194         else if (add_files)
4195         {
4196             tc_stat.curentry = curpath;
4197
4198             /* Add a new entry to the temporary db file. */
4199             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4200 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4201                          , dir->internal_entry
4202 #endif
4203                          );
4204
4205             /* Wait until current path for debug screen is read and unset. */
4206             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4207                 yield();
4208
4209             tc_stat.curentry = NULL;
4210         }
4211
4212         curpath[len] = '\0';
4213     }
4214
4215     closedir(dir);
4216
4217     return success;
4218 }
4219
4220 void tagcache_screensync_event(void)
4221 {
4222     tc_stat.curentry = NULL;
4223 }
4224
4225 void tagcache_screensync_enable(bool state)
4226 {
4227     tc_stat.syncscreen = state;
4228 }
4229
4230 void tagcache_build(const char *path)
4231 {
4232     struct tagcache_header header;
4233     bool ret;
4234
4235     curpath[0] = '\0';
4236     data_size = 0;
4237     total_entry_count = 0;
4238     processed_dir_count = 0;
4239
4240 #ifdef HAVE_DIRCACHE
4241     while (dircache_is_initializing())
4242         sleep(1);
4243 #endif
4244
4245     logf("updating tagcache");
4246
4247     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4248     if (cachefd >= 0)
4249     {
4250         logf("skipping, cache already waiting for commit");
4251         close(cachefd);
4252         return ;
4253     }
4254
4255     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4256     if (cachefd < 0)
4257     {
4258         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4259         return ;
4260     }
4261
4262     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4263
4264     cpu_boost(true);
4265
4266     logf("Scanning files...");
4267     /* Scan for new files. */
4268     memset(&header, 0, sizeof(struct tagcache_header));
4269     write(cachefd, &header, sizeof(struct tagcache_header));
4270
4271     if (strcmp("/", path) != 0)
4272         strcpy(curpath, path);
4273     ret = check_dir(path, true);
4274
4275     /* Write the header. */
4276     header.magic = TAGCACHE_MAGIC;
4277     header.datasize = data_size;
4278     header.entry_count = total_entry_count;
4279     lseek(cachefd, 0, SEEK_SET);
4280     write(cachefd, &header, sizeof(struct tagcache_header));
4281     close(cachefd);
4282
4283     if (filenametag_fd >= 0)
4284     {
4285         close(filenametag_fd);
4286         filenametag_fd = -1;
4287     }
4288
4289     if (!ret)
4290     {
4291         logf("Aborted.");
4292         cpu_boost(false);
4293         return ;
4294     }
4295
4296     /* Commit changes to the database. */
4297 #ifdef __PCTOOL__
4298     allocate_tempbuf();
4299 #endif
4300     if (commit())
4301     {
4302         remove(TAGCACHE_FILE_TEMP);
4303         logf("tagcache built!");
4304     }
4305 #ifdef __PCTOOL__
4306     free_tempbuf();
4307 #endif
4308
4309 #ifdef HAVE_TC_RAMCACHE
4310     if (hdr)
4311     {
4312         /* Import runtime statistics if we just initialized the db. */
4313         if (hdr->h.serial == 0)
4314             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4315     }
4316 #endif
4317
4318     cpu_boost(false);
4319 }
4320
4321 #ifdef HAVE_TC_RAMCACHE
4322 static void load_ramcache(void)
4323 {
4324     if (!hdr)
4325         return ;
4326
4327     cpu_boost(true);
4328
4329     /* At first we should load the cache (if exists). */
4330     tc_stat.ramcache = load_tagcache();
4331
4332     if (!tc_stat.ramcache)
4333     {
4334         /* If loading failed, it must indicate some problem with the db
4335          * so disable it entirely to prevent further issues. */
4336         tc_stat.ready = false;
4337         hdr = NULL;
4338     }
4339
4340     cpu_boost(false);
4341 }
4342
4343 void tagcache_unload_ramcache(void)
4344 {
4345     tc_stat.ramcache = false;
4346     /* Just to make sure there is no statefile present. */
4347     // remove(TAGCACHE_STATEFILE);
4348 }
4349 #endif
4350
4351 #ifndef __PCTOOL__
4352 static void tagcache_thread(void)
4353 {
4354     struct queue_event ev;
4355     bool check_done = false;
4356
4357     /* If the previous cache build/update was interrupted, commit
4358      * the changes first in foreground. */
4359     cpu_boost(true);
4360     allocate_tempbuf();
4361     commit();
4362     free_tempbuf();
4363
4364 #ifdef HAVE_TC_RAMCACHE
4365 # ifdef HAVE_EEPROM_SETTINGS
4366     if (firmware_settings.initialized && firmware_settings.disk_clean)
4367         check_done = tagcache_dumpload();
4368
4369     remove(TAGCACHE_STATEFILE);
4370 # endif
4371
4372     /* Allocate space for the tagcache if found on disk. */
4373     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4374         allocate_tagcache();
4375 #endif
4376
4377     cpu_boost(false);
4378     tc_stat.initialized = true;
4379
4380     /* Don't delay bootup with the header check but do it on background. */
4381     sleep(HZ);
4382     tc_stat.ready = check_all_headers();
4383     tc_stat.readyvalid = true;
4384
4385     while (1)
4386     {
4387         run_command_queue(false);
4388
4389         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4390
4391         switch (ev.id)
4392         {
4393             case Q_IMPORT_CHANGELOG:
4394                 tagcache_import_changelog();
4395                 break;
4396
4397             case Q_REBUILD:
4398                 remove_files();
4399                 remove(TAGCACHE_FILE_TEMP);
4400                 tagcache_build("/");
4401                 break;
4402
4403             case Q_UPDATE:
4404                 tagcache_build("/");
4405 #ifdef HAVE_TC_RAMCACHE
4406                 load_ramcache();
4407 #endif
4408                 check_deleted_files();
4409                 break ;
4410
4411             case Q_START_SCAN:
4412                 check_done = false;
4413             case SYS_TIMEOUT:
4414                 if (check_done || !tc_stat.ready)
4415                     break ;
4416
4417 #ifdef HAVE_TC_RAMCACHE
4418                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4419                 {
4420                     load_ramcache();
4421                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4422                         tagcache_build("/");
4423                 }
4424                 else
4425 #endif
4426                 if (global_settings.tagcache_autoupdate)
4427                 {
4428                     tagcache_build("/");
4429
4430                     /* This will be very slow unless dircache is enabled
4431                        or target is flash based, but do it anyway for
4432                        consistency. */
4433                     check_deleted_files();
4434                 }
4435
4436                 logf("tagcache check done");
4437
4438                 check_done = true;
4439                 break ;
4440
4441             case Q_STOP_SCAN:
4442                 break ;
4443
4444             case SYS_POWEROFF:
4445                 break ;
4446
4447 #ifndef SIMULATOR
4448             case SYS_USB_CONNECTED:
4449                 logf("USB: TagCache");
4450                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4451                 usb_wait_for_disconnect(&tagcache_queue);
4452                 break ;
4453 #endif
4454         }
4455     }
4456 }
4457
4458 bool tagcache_prepare_shutdown(void)
4459 {
4460     if (tagcache_get_commit_step() > 0)
4461         return false;
4462
4463     tagcache_stop_scan();
4464     while (read_lock || write_lock)
4465         sleep(1);
4466
4467     return true;
4468 }
4469
4470 void tagcache_shutdown(void)
4471 {
4472     /* Flush the command queue. */
4473     run_command_queue(true);
4474
4475 #ifdef HAVE_EEPROM_SETTINGS
4476     if (tc_stat.ramcache)
4477         tagcache_dumpsave();
4478 #endif
4479 }
4480
4481 static int get_progress(void)
4482 {
4483     int total_count = -1;
4484
4485 #ifdef HAVE_DIRCACHE
4486     if (dircache_is_enabled())
4487     {
4488         total_count = dircache_get_entry_count();
4489     }
4490     else
4491 #endif
4492 #ifdef HAVE_TC_RAMCACHE
4493     {
4494         if (hdr && tc_stat.ramcache)
4495             total_count = hdr->h.tch.entry_count;
4496     }
4497 #endif
4498
4499     if (total_count < 0)
4500         return -1;
4501
4502     return processed_dir_count * 100 / total_count;
4503 }
4504
4505 struct tagcache_stat* tagcache_get_stat(void)
4506 {
4507     tc_stat.progress = get_progress();
4508     tc_stat.processed_entries = processed_dir_count;
4509
4510     return &tc_stat;
4511 }
4512
4513 void tagcache_start_scan(void)
4514 {
4515     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4516 }
4517
4518 bool tagcache_update(void)
4519 {
4520     if (!tc_stat.ready)
4521         return false;
4522
4523     queue_post(&tagcache_queue, Q_UPDATE, 0);
4524     return false;
4525 }
4526
4527 bool tagcache_rebuild()
4528 {
4529     queue_post(&tagcache_queue, Q_REBUILD, 0);
4530     return false;
4531 }
4532
4533 void tagcache_stop_scan(void)
4534 {
4535     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4536 }
4537
4538 #ifdef HAVE_TC_RAMCACHE
4539 bool tagcache_is_ramcache(void)
4540 {
4541     return tc_stat.ramcache;
4542 }
4543 #endif
4544
4545 #endif /* !__PCTOOL__ */
4546
4547
4548 void tagcache_init(void)
4549 {
4550     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4551     memset(&current_tcmh, 0, sizeof(struct master_header));
4552     filenametag_fd = -1;
4553     write_lock = read_lock = 0;
4554
4555 #ifndef __PCTOOL__
4556     mutex_init(&command_queue_mutex);
4557     queue_init(&tagcache_queue, true);
4558     create_thread(tagcache_thread, tagcache_stack,
4559                   sizeof(tagcache_stack), 0, tagcache_thread_name
4560                   IF_PRIO(, PRIORITY_BACKGROUND)
4561                   IF_COP(, CPU));
4562 #else
4563     tc_stat.initialized = true;
4564     allocate_tempbuf();
4565     commit();
4566     free_tempbuf();
4567     tc_stat.ready = check_all_headers();
4568 #endif
4569 }
4570
4571 #ifdef __PCTOOL__
4572 void tagcache_reverse_scan(void)
4573 {
4574     logf("Checking for deleted files");
4575     check_deleted_files();
4576 }
4577 #endif
4578
4579 bool tagcache_is_initialized(void)
4580 {
4581     return tc_stat.initialized;
4582 }
4583 bool tagcache_is_usable(void)
4584 {
4585     return tc_stat.initialized && tc_stat.ready;
4586 }
4587 int tagcache_get_commit_step(void)
4588 {
4589     return tc_stat.commit_step;
4590 }
4591 int tagcache_get_max_commit_step(void)
4592 {
4593     return (int)(SORTED_TAGS_COUNT)+1;
4594 }
4595