apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "filefuncs.h"
  77 #include "settings.h"
  78 #include "dir.h"
  79 #include "structec.h"
  80
  81 #ifndef __PCTOOL__
  82 #include "lang.h"
  83 #include "eeprom_settings.h"
  84 #endif
  85
  86 #ifdef __PCTOOL__
  87 #define yield() do { } while(0)
  88 #define sim_sleep(timeout) do { } while(0)
  89 #define do_timed_yield() do { } while(0)
  90 #endif
  91
  92 #ifndef __PCTOOL__
  93 /* Tag Cache thread. */
  94 static struct event_queue tagcache_queue;
  95 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  96 static const char tagcache_thread_name[] = "tagcache";
  97 #endif
  98
  99 /* Previous path when scanning directory tree recursively. */
 100 static char curpath[TAG_MAXLEN+32];
 101
 102 /* Used when removing duplicates. */
 103 static char *tempbuf;     /* Allocated when needed. */
 104 static long tempbufidx;   /* Current location in buffer. */
 105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 106 static long tempbuf_left; /* Buffer space left. */
 107 static long tempbuf_pos;
 108
 109 #define SORTED_TAGS_COUNT 8
 110 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 111 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 112 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 113     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 114 /* Tags we want to get sorted (loaded to the tempbuf). */
 115 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 116     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 117     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 118
 119 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 120 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping))
 123
 124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 125 static const char *tags_str[] = { "artist", "album", "genre", "title",
 126     "filename", "composer", "comment", "albumartist", "grouping", "year",
 127     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 128     "playtime", "lastplayed", "commitid", "mtime" };
 129
 130 /* Status information of the tagcache. */
 131 static struct tagcache_stat tc_stat;
 132
 133 /* Queue commands. */
 134 enum tagcache_queue {
 135     Q_STOP_SCAN = 0,
 136     Q_START_SCAN,
 137     Q_IMPORT_CHANGELOG,
 138     Q_UPDATE,
 139     Q_REBUILD,
 140
 141     /* Internal tagcache command queue. */
 142     CMD_UPDATE_MASTER_HEADER,
 143     CMD_UPDATE_NUMERIC,
 144 };
 145
 146 struct tagcache_command_entry {
 147     int32_t command;
 148     int32_t idx_id;
 149     int32_t tag;
 150     int32_t data;
 151 };
 152
 153 #ifndef __PCTOOL__
 154 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 155 static volatile int command_queue_widx = 0;
 156 static volatile int command_queue_ridx = 0;
 157 static struct mutex command_queue_mutex;
 158 #endif
 159
 160 /* Tag database structures. */
 161
 162 /* Variable-length tag entry in tag files. */
 163 struct tagfile_entry {
 164     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 165     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 166     char tag_data[0];  /* Begin of the tag data */
 167 };
 168
 169 /* Fixed-size tag entry in master db index. */
 170 struct index_entry {
 171     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 172     int32_t flag;                /* Status flags */
 173 };
 174
 175 /* Header is the same in every file. */
 176 struct tagcache_header {
 177     int32_t magic;       /* Header version number */
 178     int32_t datasize;    /* Data size in bytes */
 179     int32_t entry_count; /* Number of entries in this file */
 180 };
 181
 182 struct master_header {
 183     struct tagcache_header tch;
 184     int32_t serial; /* Increasing counting number */
 185     int32_t commitid; /* Number of commits so far */
 186     int32_t dirty;
 187 };
 188
 189 /* For the endianess correction */
 190 static const char *tagfile_entry_ec   = "ll";
 191 /**
 192  Note: This should be (1 + TAG_COUNT) amount of l's.
 193  */
 194 static const char *index_entry_ec     = "lllllllllllllllllllll";
 195
 196 static const char *tagcache_header_ec = "lll";
 197 static const char *master_header_ec   = "llllll";
 198
 199 static struct master_header current_tcmh;
 200
 201 #ifdef HAVE_TC_RAMCACHE
 202 /* Header is created when loading database to ram. */
 203 struct ramcache_header {
 204     struct master_header h;      /* Header from the master index */
 205     struct index_entry *indices; /* Master index file content */
 206     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 207     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 208 };
 209
 210 # ifdef HAVE_EEPROM_SETTINGS
 211 struct statefile_header {
 212     struct ramcache_header *hdr;
 213     struct tagcache_stat tc_stat;
 214 };
 215 # endif
 216
 217 /* Pointer to allocated ramcache_header */
 218 static struct ramcache_header *hdr;
 219 #endif
 220
 221 /**
 222  * Full tag entries stored in a temporary file waiting
 223  * for commit to the cache. */
 224 struct temp_file_entry {
 225     long tag_offset[TAG_COUNT];
 226     short tag_length[TAG_COUNT];
 227     long flag;
 228
 229     long data_length;
 230 };
 231
 232 struct tempbuf_id_list {
 233     long id;
 234     struct tempbuf_id_list *next;
 235 };
 236
 237 struct tempbuf_searchidx {
 238     long idx_id;
 239     char *str;
 240     int seek;
 241     struct tempbuf_id_list idlist;
 242 };
 243
 244 /* Lookup buffer for fixing messed up index while after sorting. */
 245 static long commit_entry_count;
 246 static long lookup_buffer_depth;
 247 static struct tempbuf_searchidx **lookup;
 248
 249 /* Used when building the temporary file. */
 250 static int cachefd = -1, filenametag_fd;
 251 static int total_entry_count = 0;
 252 static int data_size = 0;
 253 static int processed_dir_count;
 254
 255 /* Thread safe locking */
 256 static volatile int write_lock;
 257 static volatile int read_lock;
 258
 259 static bool delete_entry(long idx_id);
 260
 261 const char* tagcache_tag_to_str(int tag)
 262 {
 263     return tags_str[tag];
 264 }
 265
 266 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 267 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 268 {
 269     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 270 }
 271
 272 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 #ifdef HAVE_DIRCACHE
 283 /**
 284  * Returns true if specified flag is still present, i.e., dircache
 285  * has not been reloaded.
 286  */
 287 static bool is_dircache_intact(void)
 288 {
 289     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 290 }
 291 #endif
 292
 293 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 294 {
 295     int fd;
 296     char buf[MAX_PATH], path[MAX_PATH];
 297     const char * file;
 298     int rc;
 299
 300     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 301         return -1;
 302
 303     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 304     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
 305
 306     fd = open(file, write ? O_RDWR : O_RDONLY);
 307     if (fd < 0)
 308     {
 309         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 310         tc_stat.ready = false;
 311         return fd;
 312     }
 313
 314     /* Check the header. */
 315     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 316     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 317     {
 318         logf("header error");
 319         tc_stat.ready = false;
 320         close(fd);
 321         return -2;
 322     }
 323
 324     return fd;
 325 }
 326
 327 static int open_master_fd(struct master_header *hdr, bool write)
 328 {
 329     int fd;
 330     int rc;
 331     char path[MAX_PATH];
 332
 333     fd = open(get_user_file_path(TAGCACHE_FILE_MASTER,
 334                                  IS_FILE|NEED_WRITE,
 335                                  path, sizeof(path)),
 336                     write ? O_RDWR : O_RDONLY);
 337     if (fd < 0)
 338     {
 339         logf("master file open failed for R/W");
 340         tc_stat.ready = false;
 341         return fd;
 342     }
 343
 344     tc_stat.econ = false;
 345
 346     /* Check the header. */
 347     rc = read(fd, hdr, sizeof(struct master_header));
 348     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 349     {
 350         /* Success. */
 351         return fd;
 352     }
 353
 354     /* Trying to read again, this time with endianess correction enabled. */
 355     lseek(fd, 0, SEEK_SET);
 356
 357     rc = ecread(fd, hdr, 1, master_header_ec, true);
 358     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 359     {
 360         logf("header error");
 361         tc_stat.ready = false;
 362         close(fd);
 363         return -2;
 364     }
 365
 366     tc_stat.econ = true;
 367
 368     return fd;
 369 }
 370
 371 #ifndef __PCTOOL__
 372 static bool do_timed_yield(void)
 373 {
 374     /* Sorting can lock up for quite a while, so yield occasionally */
 375     static long wakeup_tick = 0;
 376     if (TIME_AFTER(current_tick, wakeup_tick))
 377     {
 378         wakeup_tick = current_tick + (HZ/4);
 379         yield();
 380         return true;
 381     }
 382     return false;
 383 }
 384 #endif
 385
 386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 387 static long find_entry_ram(const char *filename,
 388                            const struct dircache_entry *dc)
 389 {
 390     static long last_pos = 0;
 391     int i;
 392
 393     /* Check if we tagcache is loaded into ram. */
 394     if (!tc_stat.ramcache)
 395         return -1;
 396
 397     if (dc == NULL)
 398         dc = dircache_get_entry_ptr(filename);
 399
 400     if (dc == NULL)
 401     {
 402         logf("tagcache: file not found.");
 403         return -1;
 404     }
 405
 406     try_again:
 407
 408     if (last_pos > 0)
 409         i = last_pos;
 410     else
 411         i = 0;
 412
 413     for (; i < hdr->h.tch.entry_count; i++)
 414     {
 415         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 416         {
 417             last_pos = MAX(0, i - 3);
 418             return i;
 419         }
 420
 421         do_timed_yield();
 422     }
 423
 424     if (last_pos > 0)
 425     {
 426         last_pos = 0;
 427         goto try_again;
 428     }
 429
 430     return -1;
 431 }
 432 #endif
 433
 434 static long find_entry_disk(const char *filename, bool localfd)
 435 {
 436     struct tagcache_header tch;
 437     static long last_pos = -1;
 438     long pos_history[POS_HISTORY_COUNT];
 439     long pos_history_idx = 0;
 440     bool found = false;
 441     struct tagfile_entry tfe;
 442     int fd;
 443     char buf[TAG_MAXLEN+32];
 444     int i;
 445     int pos = -1;
 446
 447     if (!tc_stat.ready)
 448         return -2;
 449
 450     fd = filenametag_fd;
 451     if (fd < 0 || localfd)
 452     {
 453         last_pos = -1;
 454         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 455             return -1;
 456     }
 457
 458     check_again:
 459
 460     if (last_pos > 0)
 461         lseek(fd, last_pos, SEEK_SET);
 462     else
 463         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 464
 465     while (true)
 466     {
 467         pos = lseek(fd, 0, SEEK_CUR);
 468         for (i = pos_history_idx-1; i >= 0; i--)
 469             pos_history[i+1] = pos_history[i];
 470         pos_history[0] = pos;
 471
 472         if (ecread_tagfile_entry(fd, &tfe)
 473             != sizeof(struct tagfile_entry))
 474         {
 475             break ;
 476         }
 477
 478         if (tfe.tag_length >= (long)sizeof(buf))
 479         {
 480             logf("too long tag #1");
 481             close(fd);
 482             if (!localfd)
 483                 filenametag_fd = -1;
 484             last_pos = -1;
 485             return -2;
 486         }
 487
 488         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 489         {
 490             logf("read error #2");
 491             close(fd);
 492             if (!localfd)
 493                 filenametag_fd = -1;
 494             last_pos = -1;
 495             return -3;
 496         }
 497
 498         if (!strcasecmp(filename, buf))
 499         {
 500             last_pos = pos_history[pos_history_idx];
 501             found = true;
 502             break ;
 503         }
 504
 505         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 506             pos_history_idx++;
 507     }
 508
 509     /* Not found? */
 510     if (!found)
 511     {
 512         if (last_pos > 0)
 513         {
 514             last_pos = -1;
 515             logf("seek again");
 516             goto check_again;
 517         }
 518
 519         if (fd != filenametag_fd || localfd)
 520             close(fd);
 521         return -4;
 522     }
 523
 524     if (fd != filenametag_fd || localfd)
 525         close(fd);
 526
 527     return tfe.idx_id;
 528 }
 529
 530 static int find_index(const char *filename)
 531 {
 532     long idx_id = -1;
 533
 534 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 535     if (tc_stat.ramcache && is_dircache_intact())
 536         idx_id = find_entry_ram(filename, NULL);
 537 #endif
 538
 539     if (idx_id < 0)
 540         idx_id = find_entry_disk(filename, true);
 541
 542     return idx_id;
 543 }
 544
 545 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 546 {
 547     int idx_id;
 548
 549     if (!tc_stat.ready)
 550         return false;
 551
 552     idx_id = find_index(filename);
 553     if (idx_id < 0)
 554         return false;
 555
 556     if (!tagcache_search(tcs, tag_filename))
 557         return false;
 558
 559     tcs->entry_count = 0;
 560     tcs->idx_id = idx_id;
 561
 562     return true;
 563 }
 564
 565 static bool get_index(int masterfd, int idxid,
 566                       struct index_entry *idx, bool use_ram)
 567 {
 568     bool localfd = false;
 569
 570     if (idxid < 0)
 571     {
 572         logf("Incorrect idxid: %d", idxid);
 573         return false;
 574     }
 575
 576 #ifdef HAVE_TC_RAMCACHE
 577     if (tc_stat.ramcache && use_ram)
 578     {
 579         if (hdr->indices[idxid].flag & FLAG_DELETED)
 580             return false;
 581
 582 # ifdef HAVE_DIRCACHE
 583         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 584             || is_dircache_intact())
 585 #endif
 586         {
 587             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 588             return true;
 589         }
 590     }
 591 #else
 592     (void)use_ram;
 593 #endif
 594
 595     if (masterfd < 0)
 596     {
 597         struct master_header tcmh;
 598
 599         localfd = true;
 600         masterfd = open_master_fd(&tcmh, false);
 601         if (masterfd < 0)
 602             return false;
 603     }
 604
 605     lseek(masterfd, idxid * sizeof(struct index_entry)
 606           + sizeof(struct master_header), SEEK_SET);
 607     if (ecread_index_entry(masterfd, idx)
 608         != sizeof(struct index_entry))
 609     {
 610         logf("read error #3");
 611         if (localfd)
 612             close(masterfd);
 613
 614         return false;
 615     }
 616
 617     if (localfd)
 618         close(masterfd);
 619
 620     if (idx->flag & FLAG_DELETED)
 621         return false;
 622
 623     return true;
 624 }
 625
 626 #ifndef __PCTOOL__
 627
 628 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 629 {
 630     /* We need to exclude all memory only flags & tags when writing to disk. */
 631     if (idx->flag & FLAG_DIRCACHE)
 632     {
 633         logf("memory only flags!");
 634         return false;
 635     }
 636
 637 #ifdef HAVE_TC_RAMCACHE
 638     /* Only update numeric data. Writing the whole index to RAM by memcpy
 639      * destroys dircache pointers!
 640      */
 641     if (tc_stat.ramcache)
 642     {
 643         int tag;
 644         struct index_entry *idx_ram = &hdr->indices[idxid];
 645
 646         for (tag = 0; tag < TAG_COUNT; tag++)
 647         {
 648             if (TAGCACHE_IS_NUMERIC(tag))
 649             {
 650                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 651             }
 652         }
 653
 654         /* Don't touch the dircache flag or attributes. */
 655         idx_ram->flag = (idx->flag & 0x0000ffff)
 656             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 657     }
 658 #endif
 659
 660     lseek(masterfd, idxid * sizeof(struct index_entry)
 661           + sizeof(struct master_header), SEEK_SET);
 662     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 663     {
 664         logf("write error #3");
 665         logf("idxid: %d", idxid);
 666         return false;
 667     }
 668
 669     return true;
 670 }
 671
 672 #endif /* !__PCTOOL__ */
 673
 674 static bool open_files(struct tagcache_search *tcs, int tag)
 675 {
 676     if (tcs->idxfd[tag] < 0)
 677     {
 678         char fn[MAX_PATH], path[MAX_PATH];
 679         const char *file;
 680
 681         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 682         file = get_user_file_path(fn, IS_FILE | NEED_WRITE, path, sizeof(path));
 683         tcs->idxfd[tag] = open(file, O_RDONLY);
 684     }
 685
 686     if (tcs->idxfd[tag] < 0)
 687     {
 688         logf("File not open!");
 689         return false;
 690     }
 691
 692     return true;
 693 }
 694
 695 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 696                      int tag, char *buf, long size)
 697 {
 698     struct tagfile_entry tfe;
 699     long seek;
 700
 701     *buf = '\0';
 702
 703     if (TAGCACHE_IS_NUMERIC(tag))
 704         return false;
 705
 706     seek = idx->tag_seek[tag];
 707     if (seek < 0)
 708     {
 709         logf("Retrieve failed");
 710         return false;
 711     }
 712
 713 #ifdef HAVE_TC_RAMCACHE
 714     if (tcs->ramsearch)
 715     {
 716         struct tagfile_entry *ep;
 717
 718 # ifdef HAVE_DIRCACHE
 719         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 720             && is_dircache_intact())
 721         {
 722             dircache_copy_path((struct dircache_entry *)seek,
 723                                buf, size);
 724             return true;
 725         }
 726         else
 727 # endif
 728         if (tag != tag_filename)
 729         {
 730             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 731             strlcpy(buf, ep->tag_data, size);
 732
 733             return true;
 734         }
 735     }
 736 #endif
 737
 738     if (!open_files(tcs, tag))
 739         return false;
 740
 741     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 742     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 743         != sizeof(struct tagfile_entry))
 744     {
 745         logf("read error #5");
 746         return false;
 747     }
 748
 749     if (tfe.tag_length >= size)
 750     {
 751         logf("too small buffer");
 752         return false;
 753     }
 754
 755     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 756         tfe.tag_length)
 757     {
 758         logf("read error #6");
 759         return false;
 760     }
 761
 762     buf[tfe.tag_length] = '\0';
 763
 764     return true;
 765 }
 766
 767 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 768
 769 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 770 {
 771     if (! COMMAND_QUEUE_IS_EMPTY)
 772     {
 773         /* Attempt to find tag data through store-to-load forwarding in
 774            command queue */
 775         long result = -1;
 776
 777         mutex_lock(&command_queue_mutex);
 778
 779         int ridx = command_queue_widx;
 780
 781         while (ridx != command_queue_ridx)
 782         {
 783             if (--ridx < 0)
 784                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 785
 786             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 787                 && command_queue[ridx].idx_id == idx_id
 788                 && command_queue[ridx].tag == tag)
 789             {
 790                 result = command_queue[ridx].data;
 791                 break;
 792             }
 793         }
 794
 795         mutex_unlock(&command_queue_mutex);
 796
 797         if (result >= 0)
 798         {
 799             logf("read_numeric_tag: "
 800                  "Recovered tag %d value %lX from write queue",
 801                  tag, result);
 802             return result;
 803         }
 804     }
 805
 806     return idx->tag_seek[tag];
 807 }
 808
 809
 810 static long check_virtual_tags(int tag, int idx_id,
 811                                const struct index_entry *idx)
 812 {
 813     long data = 0;
 814
 815     switch (tag)
 816     {
 817         case tag_virt_length_sec:
 818             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 819             break;
 820
 821         case tag_virt_length_min:
 822             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 823             break;
 824
 825         case tag_virt_playtime_sec:
 826             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 827             break;
 828
 829         case tag_virt_playtime_min:
 830             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 831             break;
 832
 833         case tag_virt_autoscore:
 834             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 835                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 836             {
 837                 data = 0;
 838             }
 839             else
 840             {
 841                 /* A straight calculus gives:
 842                      autoscore = 100 * playtime / length / playcout (1)
 843                    Now, consider the euclidian division of playtime by length:
 844                      playtime = alpha * length + beta
 845                    With:
 846                      0 <= beta < length
 847                    Now, (1) becomes:
 848                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 849                    Both terms should be small enough to avoid any overflow
 850                 */
 851                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 852                               / read_numeric_tag(tag_length, idx_id, idx))
 853                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 854                                  % read_numeric_tag(tag_length, idx_id, idx)))
 855                          / read_numeric_tag(tag_length, idx_id, idx);
 856                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 857             }
 858             break;
 859
 860         /* How many commits before the file has been added to the DB. */
 861         case tag_virt_entryage:
 862             data = current_tcmh.commitid
 863                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 864             break;
 865
 866         default:
 867             data = read_numeric_tag(tag, idx_id, idx);
 868     }
 869
 870     return data;
 871 }
 872
 873 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 874 {
 875     struct index_entry idx;
 876
 877     if (!tc_stat.ready)
 878         return false;
 879
 880     if (!TAGCACHE_IS_NUMERIC(tag))
 881         return -1;
 882
 883     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 884         return -2;
 885
 886     return check_virtual_tags(tag, tcs->idx_id, &idx);
 887 }
 888
 889 inline static bool str_ends_with(const char *str1, const char *str2)
 890 {
 891     int str_len = strlen(str1);
 892     int clause_len = strlen(str2);
 893
 894     if (clause_len > str_len)
 895         return false;
 896
 897     return !strcasecmp(&str1[str_len - clause_len], str2);
 898 }
 899
 900 inline static bool str_oneof(const char *str, const char *list)
 901 {
 902     const char *sep;
 903     int l, len = strlen(str);
 904
 905     while (*list)
 906     {
 907         sep = strchr(list, '|');
 908         l = sep ? (long)sep - (long)list : (int)strlen(list);
 909         if ((l==len) && !strncasecmp(str, list, len))
 910             return true;
 911         list += sep ? l + 1 : l;
 912     }
 913
 914     return false;
 915 }
 916
 917 static bool check_against_clause(long numeric, const char *str,
 918                                  const struct tagcache_search_clause *clause)
 919 {
 920     if (clause->numeric)
 921     {
 922         switch (clause->type)
 923         {
 924             case clause_is:
 925                 return numeric == clause->numeric_data;
 926             case clause_is_not:
 927                 return numeric != clause->numeric_data;
 928             case clause_gt:
 929                 return numeric > clause->numeric_data;
 930             case clause_gteq:
 931                 return numeric >= clause->numeric_data;
 932             case clause_lt:
 933                 return numeric < clause->numeric_data;
 934             case clause_lteq:
 935                 return numeric <= clause->numeric_data;
 936             default:
 937                 logf("Incorrect numeric tag: %d", clause->type);
 938         }
 939     }
 940     else
 941     {
 942         switch (clause->type)
 943         {
 944             case clause_is:
 945                 return !strcasecmp(clause->str, str);
 946             case clause_is_not:
 947                 return strcasecmp(clause->str, str);
 948             case clause_gt:
 949                 return 0>strcasecmp(clause->str, str);
 950             case clause_gteq:
 951                 return 0>=strcasecmp(clause->str, str);
 952             case clause_lt:
 953                 return 0<strcasecmp(clause->str, str);
 954             case clause_lteq:
 955                 return 0<=strcasecmp(clause->str, str);
 956             case clause_contains:
 957                 return (strcasestr(str, clause->str) != NULL);
 958             case clause_not_contains:
 959                 return (strcasestr(str, clause->str) == NULL);
 960             case clause_begins_with:
 961                 return (strcasestr(str, clause->str) == str);
 962             case clause_not_begins_with:
 963                 return (strcasestr(str, clause->str) != str);
 964             case clause_ends_with:
 965                 return str_ends_with(str, clause->str);
 966             case clause_not_ends_with:
 967                 return !str_ends_with(str, clause->str);
 968             case clause_oneof:
 969                 return str_oneof(str, clause->str);
 970
 971             default:
 972                 logf("Incorrect tag: %d", clause->type);
 973         }
 974     }
 975
 976     return false;
 977 }
 978
 979 static bool check_clauses(struct tagcache_search *tcs,
 980                           struct index_entry *idx,
 981                           struct tagcache_search_clause **clause, int count)
 982 {
 983     int i;
 984
 985 #ifdef HAVE_TC_RAMCACHE
 986     if (tcs->ramsearch)
 987     {
 988         /* Go through all conditional clauses. */
 989         for (i = 0; i < count; i++)
 990         {
 991             struct tagfile_entry *tfe;
 992             int seek;
 993             char buf[256];
 994             char *str = NULL;
 995
 996             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
 997
 998             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 999             {
1000                 if (clause[i]->tag == tag_filename)
1001                 {
1002                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1003                     str = buf;
1004                 }
1005                 else
1006                 {
1007                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1008                     str = tfe->tag_data;
1009                 }
1010             }
1011
1012             if (!check_against_clause(seek, str, clause[i]))
1013                 return false;
1014         }
1015     }
1016     else
1017 #endif
1018     {
1019         /* Check for conditions. */
1020         for (i = 0; i < count; i++)
1021         {
1022             struct tagfile_entry tfe;
1023             int seek;
1024             char str[256];
1025
1026             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1027
1028             memset(str, 0, sizeof str);
1029             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1030             {
1031                 int fd = tcs->idxfd[clause[i]->tag];
1032                 lseek(fd, seek, SEEK_SET);
1033                 ecread_tagfile_entry(fd, &tfe);
1034                 if (tfe.tag_length >= (int)sizeof(str))
1035                 {
1036                     logf("Too long tag read!");
1037                     break ;
1038                 }
1039
1040                 read(fd, str, tfe.tag_length);
1041
1042                 /* Check if entry has been deleted. */
1043                 if (str[0] == '\0')
1044                     break;
1045             }
1046
1047             if (!check_against_clause(seek, str, clause[i]))
1048                 return false;
1049         }
1050     }
1051
1052     return true;
1053 }
1054
1055 bool tagcache_check_clauses(struct tagcache_search *tcs,
1056                             struct tagcache_search_clause **clause, int count)
1057 {
1058     struct index_entry idx;
1059
1060     if (count == 0)
1061         return true;
1062
1063     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1064         return false;
1065
1066     return check_clauses(tcs, &idx, clause, count);
1067 }
1068
1069 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1070 {
1071     int i;
1072
1073     /* If uniq buffer is not defined we must return true for search to work. */
1074     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1075         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1076     {
1077         return true;
1078     }
1079
1080     for (i = 0; i < tcs->unique_list_count; i++)
1081     {
1082         /* Return false if entry is found. */
1083         if (tcs->unique_list[i] == id)
1084             return false;
1085     }
1086
1087     if (tcs->unique_list_count < tcs->unique_list_capacity)
1088     {
1089         tcs->unique_list[i] = id;
1090         tcs->unique_list_count++;
1091     }
1092
1093     return true;
1094 }
1095
1096 static bool build_lookup_list(struct tagcache_search *tcs)
1097 {
1098     struct index_entry entry;
1099     int i, j;
1100
1101     tcs->seek_list_count = 0;
1102
1103 #ifdef HAVE_TC_RAMCACHE
1104     if (tcs->ramsearch
1105 # ifdef HAVE_DIRCACHE
1106         && (tcs->type != tag_filename || is_dircache_intact())
1107 # endif
1108         )
1109     {
1110         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1111         {
1112             struct tagcache_seeklist_entry *seeklist;
1113             struct index_entry *idx = &hdr->indices[i];
1114             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1115                 break ;
1116
1117             /* Skip deleted files. */
1118             if (idx->flag & FLAG_DELETED)
1119                 continue;
1120
1121             /* Go through all filters.. */
1122             for (j = 0; j < tcs->filter_count; j++)
1123             {
1124                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1125                 {
1126                     break ;
1127                 }
1128             }
1129
1130             if (j < tcs->filter_count)
1131                 continue ;
1132
1133             /* Check for conditions. */
1134             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1135                 continue;
1136
1137             /* Add to the seek list if not already in uniq buffer. */
1138             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1139                 continue;
1140
1141             /* Lets add it. */
1142             seeklist = &tcs->seeklist[tcs->seek_list_count];
1143             seeklist->seek = idx->tag_seek[tcs->type];
1144             seeklist->flag = idx->flag;
1145             seeklist->idx_id = i;
1146             tcs->seek_list_count++;
1147         }
1148
1149         tcs->seek_pos = i;
1150
1151         return tcs->seek_list_count > 0;
1152     }
1153 #endif
1154
1155     if (tcs->masterfd < 0)
1156     {
1157         struct master_header tcmh;
1158         tcs->masterfd = open_master_fd(&tcmh, false);
1159     }
1160
1161     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1162             sizeof(struct master_header), SEEK_SET);
1163
1164     while (ecread_index_entry(tcs->masterfd, &entry)
1165            == sizeof(struct index_entry))
1166     {
1167         struct tagcache_seeklist_entry *seeklist;
1168
1169         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1170             break ;
1171
1172         i = tcs->seek_pos;
1173         tcs->seek_pos++;
1174
1175         /* Check if entry has been deleted. */
1176         if (entry.flag & FLAG_DELETED)
1177             continue;
1178
1179         /* Go through all filters.. */
1180         for (j = 0; j < tcs->filter_count; j++)
1181         {
1182             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1183                 break ;
1184         }
1185
1186         if (j < tcs->filter_count)
1187             continue ;
1188
1189         /* Check for conditions. */
1190         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1191             continue;
1192
1193         /* Add to the seek list if not already in uniq buffer. */
1194         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1195             continue;
1196
1197         /* Lets add it. */
1198         seeklist = &tcs->seeklist[tcs->seek_list_count];
1199         seeklist->seek = entry.tag_seek[tcs->type];
1200         seeklist->flag = entry.flag;
1201         seeklist->idx_id = i;
1202         tcs->seek_list_count++;
1203
1204         yield();
1205     }
1206
1207     return tcs->seek_list_count > 0;
1208 }
1209
1210
1211 static void remove_files(void)
1212 {
1213     int i;
1214     char buf[MAX_PATH];
1215
1216     tc_stat.ready = false;
1217     tc_stat.ramcache = false;
1218     tc_stat.econ = false;
1219     remove(get_user_file_path(TAGCACHE_FILE_MASTER, NEED_WRITE|IS_FILE,
1220                               buf, sizeof(buf)));
1221     for (i = 0; i < TAG_COUNT; i++)
1222     {
1223         char buf2[MAX_PATH];
1224         if (TAGCACHE_IS_NUMERIC(i))
1225             continue;
1226
1227         /* database_%d.tcd -> database_0.tcd */
1228         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1229         remove(get_user_file_path(buf, NEED_WRITE | IS_FILE, buf2, sizeof(buf2)));
1230     }
1231 }
1232
1233
1234 static bool check_all_headers(void)
1235 {
1236     struct master_header myhdr;
1237     struct tagcache_header tch;
1238     int tag;
1239     int fd;
1240
1241     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1242         return false;
1243
1244     close(fd);
1245     if (myhdr.dirty)
1246     {
1247         logf("tagcache is dirty!");
1248         return false;
1249     }
1250
1251     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1252
1253     for (tag = 0; tag < TAG_COUNT; tag++)
1254     {
1255         if (TAGCACHE_IS_NUMERIC(tag))
1256             continue;
1257
1258         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1259             return false;
1260
1261         close(fd);
1262     }
1263
1264     return true;
1265 }
1266
1267 bool tagcache_is_busy(void)
1268 {
1269     return read_lock || write_lock;
1270 }
1271
1272 bool tagcache_search(struct tagcache_search *tcs, int tag)
1273 {
1274     struct tagcache_header tag_hdr;
1275     struct master_header   master_hdr;
1276     int i;
1277
1278     while (read_lock)
1279         sleep(1);
1280
1281     memset(tcs, 0, sizeof(struct tagcache_search));
1282     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1283         return false;
1284
1285     tcs->position = sizeof(struct tagcache_header);
1286     tcs->type = tag;
1287     tcs->seek_pos = 0;
1288     tcs->list_position = 0;
1289     tcs->seek_list_count = 0;
1290     tcs->filter_count = 0;
1291     tcs->masterfd = -1;
1292
1293     for (i = 0; i < TAG_COUNT; i++)
1294         tcs->idxfd[i] = -1;
1295
1296 #ifndef HAVE_TC_RAMCACHE
1297     tcs->ramsearch = false;
1298 #else
1299     tcs->ramsearch = tc_stat.ramcache;
1300     if (tcs->ramsearch)
1301     {
1302         tcs->entry_count = hdr->entry_count[tcs->type];
1303     }
1304     else
1305 #endif
1306     {
1307         /* Always open as R/W so we can pass tcs to functions that modify data also
1308          * without failing. */
1309         tcs->masterfd = open_master_fd(&master_hdr, true);
1310         if (tcs->masterfd < 0)
1311             return false;
1312
1313         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1314         {
1315             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1316             if (tcs->idxfd[tcs->type] < 0)
1317                 return false;
1318
1319             tcs->entry_count = tag_hdr.entry_count;
1320         }
1321         else
1322         {
1323             tcs->entry_count = master_hdr.tch.entry_count;
1324         }
1325     }
1326
1327     tcs->valid = true;
1328     tcs->initialized = true;
1329     write_lock++;
1330
1331     return true;
1332 }
1333
1334 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1335                                  void *buffer, long length)
1336 {
1337     tcs->unique_list = (unsigned long *)buffer;
1338     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1339     tcs->unique_list_count = 0;
1340 }
1341
1342 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1343                                 int tag, int seek)
1344 {
1345     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1346         return false;
1347
1348     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1349         return false;
1350
1351     tcs->filter_tag[tcs->filter_count] = tag;
1352     tcs->filter_seek[tcs->filter_count] = seek;
1353     tcs->filter_count++;
1354
1355     return true;
1356 }
1357
1358 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1359                                 struct tagcache_search_clause *clause)
1360 {
1361     int i;
1362
1363     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1364     {
1365         logf("Too many clauses");
1366         return false;
1367     }
1368
1369     /* Check if there is already a similar filter in present (filters are
1370      * much faster than clauses).
1371      */
1372     for (i = 0; i < tcs->filter_count; i++)
1373     {
1374         if (tcs->filter_tag[i] == clause->tag)
1375             return true;
1376     }
1377
1378     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1379     {
1380         char buf[MAX_PATH], path[MAX_PATH];
1381         const char *file;
1382         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1383         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
1384         tcs->idxfd[clause->tag] = open(file, O_RDONLY);
1385     }
1386
1387     tcs->clause[tcs->clause_count] = clause;
1388     tcs->clause_count++;
1389
1390     return true;
1391 }
1392
1393 static bool get_next(struct tagcache_search *tcs)
1394 {
1395     static char buf[TAG_MAXLEN+32];
1396     struct tagfile_entry entry;
1397     long flag = 0;
1398
1399     if (!tcs->valid || !tc_stat.ready)
1400         return false;
1401
1402     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1403 #ifdef HAVE_TC_RAMCACHE
1404         && !tcs->ramsearch
1405 #endif
1406         )
1407         return false;
1408
1409     /* Relative fetch. */
1410     if (tcs->filter_count > 0 || tcs->clause_count > 0
1411         || TAGCACHE_IS_NUMERIC(tcs->type)
1412 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1413         /* We need to retrieve flag status for dircache. */
1414         || (tcs->ramsearch && tcs->type == tag_filename)
1415 #endif
1416         )
1417     {
1418         struct tagcache_seeklist_entry *seeklist;
1419
1420         /* Check for end of list. */
1421         if (tcs->list_position == tcs->seek_list_count)
1422         {
1423             tcs->list_position = 0;
1424
1425             /* Try to fetch more. */
1426             if (!build_lookup_list(tcs))
1427             {
1428                 tcs->valid = false;
1429                 return false;
1430             }
1431         }
1432
1433         seeklist = &tcs->seeklist[tcs->list_position];
1434         flag = seeklist->flag;
1435         tcs->position = seeklist->seek;
1436         tcs->idx_id = seeklist->idx_id;
1437         tcs->list_position++;
1438     }
1439     else
1440     {
1441         if (tcs->entry_count == 0)
1442         {
1443             tcs->valid = false;
1444             return false;
1445         }
1446
1447         tcs->entry_count--;
1448     }
1449
1450     tcs->result_seek = tcs->position;
1451
1452     if (TAGCACHE_IS_NUMERIC(tcs->type))
1453     {
1454         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1455         tcs->result = buf;
1456         tcs->result_len = strlen(buf) + 1;
1457         return true;
1458     }
1459
1460     /* Direct fetch. */
1461 #ifdef HAVE_TC_RAMCACHE
1462     if (tcs->ramsearch)
1463     {
1464
1465 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1466         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1467             && is_dircache_intact())
1468         {
1469             dircache_copy_path((struct dircache_entry *)tcs->position,
1470                                buf, sizeof buf);
1471             tcs->result = buf;
1472             tcs->result_len = strlen(buf) + 1;
1473             tcs->ramresult = false;
1474
1475             return true;
1476         }
1477         else
1478 #endif
1479         if (tcs->type != tag_filename)
1480         {
1481             struct tagfile_entry *ep;
1482
1483             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1484             tcs->result = ep->tag_data;
1485             tcs->result_len = strlen(tcs->result) + 1;
1486             tcs->idx_id = ep->idx_id;
1487             tcs->ramresult = true;
1488
1489             /* Increase position for the next run. This may get overwritten. */
1490             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1491
1492             return true;
1493         }
1494     }
1495 #endif
1496
1497     if (!open_files(tcs, tcs->type))
1498     {
1499         tcs->valid = false;
1500         return false;
1501     }
1502
1503     /* Seek stream to the correct position and continue to direct fetch. */
1504     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1505
1506     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1507     {
1508         logf("read error #5");
1509         tcs->valid = false;
1510         return false;
1511     }
1512
1513     if (entry.tag_length > (long)sizeof(buf))
1514     {
1515         tcs->valid = false;
1516         logf("too long tag #2");
1517         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1518         return false;
1519     }
1520
1521     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1522     {
1523         tcs->valid = false;
1524         logf("read error #4");
1525         return false;
1526     }
1527
1528     /**
1529      Update the position for the next read (this may be overridden
1530      if filters or clauses are being used).
1531      */
1532     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1533     tcs->result = buf;
1534     tcs->result_len = strlen(tcs->result) + 1;
1535     tcs->idx_id = entry.idx_id;
1536     tcs->ramresult = false;
1537
1538     return true;
1539 }
1540
1541 bool tagcache_get_next(struct tagcache_search *tcs)
1542 {
1543     while (get_next(tcs))
1544     {
1545         if (tcs->result_len > 1)
1546             return true;
1547     }
1548
1549     return false;
1550 }
1551
1552 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1553                        int tag, char *buf, long size)
1554 {
1555     struct index_entry idx;
1556
1557     *buf = '\0';
1558     if (!get_index(tcs->masterfd, idxid, &idx, true))
1559         return false;
1560
1561     return retrieve(tcs, &idx, tag, buf, size);
1562 }
1563
1564 static bool update_master_header(void)
1565 {
1566     struct master_header myhdr;
1567     int fd;
1568
1569     if (!tc_stat.ready)
1570         return false;
1571
1572     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1573         return false;
1574
1575     myhdr.serial = current_tcmh.serial;
1576     myhdr.commitid = current_tcmh.commitid;
1577     myhdr.dirty = current_tcmh.dirty;
1578
1579     /* Write it back */
1580     lseek(fd, 0, SEEK_SET);
1581     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1582     close(fd);
1583
1584 #ifdef HAVE_TC_RAMCACHE
1585     if (hdr)
1586     {
1587         hdr->h.serial = current_tcmh.serial;
1588         hdr->h.commitid = current_tcmh.commitid;
1589         hdr->h.dirty = current_tcmh.dirty;
1590     }
1591 #endif
1592
1593     return true;
1594 }
1595
1596 #if 0
1597
1598 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1599 {
1600     struct tagentry *entry;
1601
1602     if (tcs->type != tag_title)
1603         return ;
1604
1605     /* We will need reserve buffer for this. */
1606     if (tcs->ramcache)
1607     {
1608         struct tagfile_entry *ep;
1609
1610         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1611         tcs->seek_list[tcs->seek_list_count];
1612     }
1613
1614     entry = find_entry_ram();
1615
1616 }
1617 #endif
1618
1619 void tagcache_search_finish(struct tagcache_search *tcs)
1620 {
1621     int i;
1622
1623     if (!tcs->initialized)
1624         return;
1625
1626     if (tcs->masterfd >= 0)
1627     {
1628         close(tcs->masterfd);
1629         tcs->masterfd = -1;
1630     }
1631
1632     for (i = 0; i < TAG_COUNT; i++)
1633     {
1634         if (tcs->idxfd[i] >= 0)
1635         {
1636             close(tcs->idxfd[i]);
1637             tcs->idxfd[i] = -1;
1638         }
1639     }
1640
1641     tcs->ramsearch = false;
1642     tcs->valid = false;
1643     tcs->initialized = 0;
1644     if (write_lock > 0)
1645         write_lock--;
1646 }
1647
1648 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1649 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1650 {
1651     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1652 }
1653
1654 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1655 {
1656     return check_virtual_tags(tag, idx_id, entry);
1657 }
1658
1659 static char* get_tag_string(const struct index_entry *entry, int tag)
1660 {
1661     char* s = get_tag(entry, tag)->tag_data;
1662     return strcmp(s, UNTAGGED) ? s : NULL;
1663 }
1664
1665 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1666 {
1667     struct index_entry *entry;
1668     int idx_id;
1669
1670     if (!tc_stat.ready || !tc_stat.ramcache)
1671         return false;
1672
1673     /* Find the corresponding entry in tagcache. */
1674     idx_id = find_entry_ram(filename, NULL);
1675     if (idx_id < 0)
1676         return false;
1677
1678     entry = &hdr->indices[idx_id];
1679
1680     memset(id3, 0, sizeof(struct mp3entry));
1681
1682     id3->title        = get_tag_string(entry, tag_title);
1683     id3->artist       = get_tag_string(entry, tag_artist);
1684     id3->album        = get_tag_string(entry, tag_album);
1685     id3->genre_string = get_tag_string(entry, tag_genre);
1686     id3->composer     = get_tag_string(entry, tag_composer);
1687     id3->comment      = get_tag_string(entry, tag_comment);
1688     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1689     id3->grouping     = get_tag_string(entry, tag_grouping);
1690
1691     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1692     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1693     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1694     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1695     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1696     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1697
1698     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1699     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1700     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1701     if (id3->bitrate == 0)
1702         id3->bitrate = 1;
1703
1704     return true;
1705 }
1706 #endif
1707
1708 static inline void write_item(const char *item)
1709 {
1710     int len = strlen(item) + 1;
1711
1712     data_size += len;
1713     write(cachefd, item, len);
1714 }
1715
1716 static int check_if_empty(char **tag)
1717 {
1718     int length;
1719
1720     if (*tag == NULL || **tag == '\0')
1721     {
1722         *tag = UNTAGGED;
1723         return sizeof(UNTAGGED); /* Tag length */
1724     }
1725
1726     length = strlen(*tag);
1727     if (length > TAG_MAXLEN)
1728     {
1729         logf("over length tag: %s", *tag);
1730         length = TAG_MAXLEN;
1731         (*tag)[length] = '\0';
1732     }
1733
1734     return length + 1;
1735 }
1736
1737 #define ADD_TAG(entry,tag,data) \
1738     /* Adding tag */ \
1739     entry.tag_offset[tag] = offset; \
1740     entry.tag_length[tag] = check_if_empty(data); \
1741     offset += entry.tag_length[tag]
1742 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1743  * idea, as it uses lots of stack and is called from a recursive function
1744  * (check_dir).
1745  */
1746 static void __attribute__ ((noinline)) add_tagcache(char *path,
1747                                                     unsigned long mtime
1748 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1749                                                     ,const struct dircache_entry *dc
1750 #endif
1751                                                    )
1752 {
1753     struct mp3entry id3;
1754     struct temp_file_entry entry;
1755     bool ret;
1756     int fd;
1757     int idx_id = -1;
1758     char tracknumfix[3];
1759     int offset = 0;
1760     int path_length = strlen(path);
1761     bool has_albumartist;
1762     bool has_grouping;
1763
1764 #ifdef SIMULATOR
1765     /* Crude logging for the sim - to aid in debugging */
1766     int logfd = open(ROCKBOX_DIR "/database.log",
1767                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1768     if (logfd >= 0) {
1769         write(logfd, path, strlen(path));
1770         write(logfd, "\n", 1);
1771         close(logfd);
1772     }
1773 #endif
1774
1775     if (cachefd < 0)
1776         return ;
1777
1778     /* Check for overlength file path. */
1779     if (path_length > TAG_MAXLEN)
1780     {
1781         /* Path can't be shortened. */
1782         logf("Too long path: %s", path);
1783         return ;
1784     }
1785
1786     /* Check if the file is supported. */
1787     if (probe_file_format(path) == AFMT_UNKNOWN)
1788         return ;
1789
1790     /* Check if the file is already cached. */
1791 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1792     if (tc_stat.ramcache && is_dircache_intact())
1793     {
1794         idx_id = find_entry_ram(path, dc);
1795     }
1796 #endif
1797
1798     /* Be sure the entry doesn't exist. */
1799     if (filenametag_fd >= 0 && idx_id < 0)
1800         idx_id = find_entry_disk(path, false);
1801
1802     /* Check if file has been modified. */
1803     if (idx_id >= 0)
1804     {
1805         struct index_entry idx;
1806
1807         /* TODO: Mark that the index exists (for fast reverse scan) */
1808         //found_idx[idx_id/8] |= idx_id%8;
1809
1810         if (!get_index(-1, idx_id, &idx, true))
1811         {
1812             logf("failed to retrieve index entry");
1813             return ;
1814         }
1815
1816         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1817         {
1818             /* No changes to file. */
1819             return ;
1820         }
1821
1822         /* Metadata might have been changed. Delete the entry. */
1823         logf("Re-adding: %s", path);
1824         if (!delete_entry(idx_id))
1825         {
1826             logf("delete_entry failed: %d", idx_id);
1827             return ;
1828         }
1829     }
1830
1831     fd = open(path, O_RDONLY);
1832     if (fd < 0)
1833     {
1834         logf("open fail: %s", path);
1835         return ;
1836     }
1837
1838     memset(&id3, 0, sizeof(struct mp3entry));
1839     memset(&entry, 0, sizeof(struct temp_file_entry));
1840     memset(&tracknumfix, 0, sizeof(tracknumfix));
1841     ret = get_metadata(&id3, fd, path);
1842     close(fd);
1843
1844     if (!ret)
1845         return ;
1846
1847     logf("-> %s", path);
1848
1849     /* Generate track number if missing. */
1850     if (id3.tracknum <= 0)
1851     {
1852         const char *p = strrchr(path, '.');
1853
1854         if (p == NULL)
1855             p = &path[strlen(path)-1];
1856
1857         while (*p != '/')
1858         {
1859             if (isdigit(*p) && isdigit(*(p-1)))
1860             {
1861                 tracknumfix[1] = *p--;
1862                 tracknumfix[0] = *p;
1863                 break;
1864             }
1865             p--;
1866         }
1867
1868         if (tracknumfix[0] != '\0')
1869         {
1870             id3.tracknum = atoi(tracknumfix);
1871             /* Set a flag to indicate track number has been generated. */
1872             entry.flag |= FLAG_TRKNUMGEN;
1873         }
1874         else
1875         {
1876             /* Unable to generate track number. */
1877             id3.tracknum = -1;
1878         }
1879     }
1880
1881     /* Numeric tags */
1882     entry.tag_offset[tag_year] = id3.year;
1883     entry.tag_offset[tag_discnumber] = id3.discnum;
1884     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1885     entry.tag_offset[tag_length] = id3.length;
1886     entry.tag_offset[tag_bitrate] = id3.bitrate;
1887     entry.tag_offset[tag_mtime] = mtime;
1888
1889     /* String tags. */
1890     has_albumartist = id3.albumartist != NULL
1891         && strlen(id3.albumartist) > 0;
1892     has_grouping = id3.grouping != NULL
1893         && strlen(id3.grouping) > 0;
1894
1895     ADD_TAG(entry, tag_filename, &path);
1896     ADD_TAG(entry, tag_title, &id3.title);
1897     ADD_TAG(entry, tag_artist, &id3.artist);
1898     ADD_TAG(entry, tag_album, &id3.album);
1899     ADD_TAG(entry, tag_genre, &id3.genre_string);
1900     ADD_TAG(entry, tag_composer, &id3.composer);
1901     ADD_TAG(entry, tag_comment, &id3.comment);
1902     if (has_albumartist)
1903     {
1904         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1905     }
1906     else
1907     {
1908         ADD_TAG(entry, tag_albumartist, &id3.artist);
1909     }
1910     if (has_grouping)
1911     {
1912         ADD_TAG(entry, tag_grouping, &id3.grouping);
1913     }
1914     else
1915     {
1916         ADD_TAG(entry, tag_grouping, &id3.title);
1917     }
1918     entry.data_length = offset;
1919
1920     /* Write the header */
1921     write(cachefd, &entry, sizeof(struct temp_file_entry));
1922
1923     /* And tags also... Correct order is critical */
1924     write_item(path);
1925     write_item(id3.title);
1926     write_item(id3.artist);
1927     write_item(id3.album);
1928     write_item(id3.genre_string);
1929     write_item(id3.composer);
1930     write_item(id3.comment);
1931     if (has_albumartist)
1932     {
1933         write_item(id3.albumartist);
1934     }
1935     else
1936     {
1937         write_item(id3.artist);
1938     }
1939     if (has_grouping)
1940     {
1941         write_item(id3.grouping);
1942     }
1943     else
1944     {
1945         write_item(id3.title);
1946     }
1947     total_entry_count++;
1948 }
1949
1950 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1951 {
1952     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1953     int len = strlen(str)+1;
1954     int i;
1955     unsigned crc32;
1956     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1957     char buf[TAG_MAXLEN+32];
1958
1959     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1960         buf[i] = tolower(str[i]);
1961     buf[i] = '\0';
1962
1963     crc32 = crc_32(buf, i, 0xffffffff);
1964
1965     if (unique)
1966     {
1967         /* Check if the crc does not exist -> entry does not exist for sure. */
1968         for (i = 0; i < tempbufidx; i++)
1969         {
1970             if (crcbuf[-i] != crc32)
1971                 continue;
1972
1973             if (!strcasecmp(str, index[i].str))
1974             {
1975                 if (id < 0 || id >= lookup_buffer_depth)
1976                 {
1977                     logf("lookup buf overf.: %d", id);
1978                     return false;
1979                 }
1980
1981                 lookup[id] = &index[i];
1982                 return true;
1983             }
1984         }
1985     }
1986
1987     /* Insert to CRC buffer. */
1988     crcbuf[-tempbufidx] = crc32;
1989     tempbuf_left -= 4;
1990
1991     /* Insert it to the buffer. */
1992     tempbuf_left -= len;
1993     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1994         return false;
1995
1996     if (id >= lookup_buffer_depth)
1997     {
1998         logf("lookup buf overf. #2: %d", id);
1999         return false;
2000     }
2001
2002     if (id >= 0)
2003     {
2004         lookup[id] = &index[tempbufidx];
2005         index[tempbufidx].idlist.id = id;
2006     }
2007     else
2008         index[tempbufidx].idlist.id = -1;
2009
2010     index[tempbufidx].idlist.next = NULL;
2011     index[tempbufidx].idx_id = idx_id;
2012     index[tempbufidx].seek = -1;
2013     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2014     memcpy(index[tempbufidx].str, str, len);
2015     tempbuf_pos += len;
2016     tempbufidx++;
2017
2018     return true;
2019 }
2020
2021 static int compare(const void *p1, const void *p2)
2022 {
2023     do_timed_yield();
2024
2025     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2026     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2027
2028     if (strcmp(e1->str, UNTAGGED) == 0)
2029     {
2030         if (strcmp(e2->str, UNTAGGED) == 0)
2031             return 0;
2032         return -1;
2033     }
2034     else if (strcmp(e2->str, UNTAGGED) == 0)
2035         return 1;
2036
2037     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2038 }
2039
2040 static int tempbuf_sort(int fd)
2041 {
2042     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2043     struct tagfile_entry fe;
2044     int i;
2045     int length;
2046
2047     /* Generate reverse lookup entries. */
2048     for (i = 0; i < lookup_buffer_depth; i++)
2049     {
2050         struct tempbuf_id_list *idlist;
2051
2052         if (!lookup[i])
2053             continue;
2054
2055         if (lookup[i]->idlist.id == i)
2056             continue;
2057
2058         idlist = &lookup[i]->idlist;
2059         while (idlist->next != NULL)
2060             idlist = idlist->next;
2061
2062         tempbuf_left -= sizeof(struct tempbuf_id_list);
2063         if (tempbuf_left - 4 < 0)
2064             return -1;
2065
2066         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2067         if (tempbuf_pos & 0x03)
2068         {
2069             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2070             tempbuf_left -= 3;
2071             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2072         }
2073         tempbuf_pos += sizeof(struct tempbuf_id_list);
2074
2075         idlist = idlist->next;
2076         idlist->id = i;
2077         idlist->next = NULL;
2078
2079         do_timed_yield();
2080     }
2081
2082     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2083     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2084
2085     for (i = 0; i < tempbufidx; i++)
2086     {
2087         struct tempbuf_id_list *idlist = &index[i].idlist;
2088
2089         /* Fix the lookup list. */
2090         while (idlist != NULL)
2091         {
2092             if (idlist->id >= 0)
2093                 lookup[idlist->id] = &index[i];
2094             idlist = idlist->next;
2095         }
2096
2097         index[i].seek = lseek(fd, 0, SEEK_CUR);
2098         length = strlen(index[i].str) + 1;
2099         fe.tag_length = length;
2100         fe.idx_id = index[i].idx_id;
2101
2102         /* Check the chunk alignment. */
2103         if ((fe.tag_length + sizeof(struct tagfile_entry))
2104             % TAGFILE_ENTRY_CHUNK_LENGTH)
2105         {
2106             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2107                 ((fe.tag_length + sizeof(struct tagfile_entry))
2108                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2109         }
2110
2111 #ifdef TAGCACHE_STRICT_ALIGN
2112         /* Make sure the entry is long aligned. */
2113         if (index[i].seek & 0x03)
2114         {
2115             logf("tempbuf_sort: alignment error!");
2116             return -3;
2117         }
2118 #endif
2119
2120         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2121             sizeof(struct tagfile_entry))
2122         {
2123             logf("tempbuf_sort: write error #1");
2124             return -1;
2125         }
2126
2127         if (write(fd, index[i].str, length) != length)
2128         {
2129             logf("tempbuf_sort: write error #2");
2130             return -2;
2131         }
2132
2133         /* Write some padding. */
2134         if (fe.tag_length - length > 0)
2135             write(fd, "XXXXXXXX", fe.tag_length - length);
2136     }
2137
2138     return i;
2139 }
2140
2141 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2142 {
2143     if (id < 0 || id >= lookup_buffer_depth)
2144         return NULL;
2145
2146     return lookup[id];
2147 }
2148
2149
2150 inline static int tempbuf_find_location(int id)
2151 {
2152     struct tempbuf_searchidx *entry;
2153
2154     entry = tempbuf_locate(id);
2155     if (entry == NULL)
2156         return -1;
2157
2158     return entry->seek;
2159 }
2160
2161 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2162 {
2163     struct master_header tcmh;
2164     struct index_entry idx;
2165     int masterfd;
2166     int masterfd_pos;
2167     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2168     int max_entries;
2169     int entries_processed = 0;
2170     int i, j;
2171     char buf[TAG_MAXLEN];
2172
2173     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2174
2175     logf("Building numeric indices...");
2176     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2177
2178     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2179         return false;
2180
2181     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2182                          SEEK_CUR);
2183     if (masterfd_pos == filesize(masterfd))
2184     {
2185         logf("we can't append!");
2186         close(masterfd);
2187         return false;
2188     }
2189
2190     while (entries_processed < h->entry_count)
2191     {
2192         int count = MIN(h->entry_count - entries_processed, max_entries);
2193
2194         /* Read in as many entries as possible. */
2195         for (i = 0; i < count; i++)
2196         {
2197             struct temp_file_entry *tfe = &entrybuf[i];
2198             int datastart;
2199
2200             /* Read in numeric data. */
2201             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2202                 sizeof(struct temp_file_entry))
2203             {
2204                 logf("read fail #1");
2205                 close(masterfd);
2206                 return false;
2207             }
2208
2209             datastart = lseek(tmpfd, 0, SEEK_CUR);
2210
2211             /**
2212              * Read string data from the following tags:
2213              * - tag_filename
2214              * - tag_artist
2215              * - tag_album
2216              * - tag_title
2217              *
2218              * A crc32 hash is calculated from the read data
2219              * and stored back to the data offset field kept in memory.
2220              */
2221 #define tmpdb_read_string_tag(tag) \
2222     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2223     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2224     { \
2225         logf("read fail: buffer overflow"); \
2226         close(masterfd); \
2227         return false; \
2228     } \
2229     \
2230     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2231         tfe->tag_length[tag]) \
2232     { \
2233         logf("read fail #2"); \
2234         close(masterfd); \
2235         return false; \
2236     } \
2237     \
2238     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2239     lseek(tmpfd, datastart, SEEK_SET)
2240
2241             tmpdb_read_string_tag(tag_filename);
2242             tmpdb_read_string_tag(tag_artist);
2243             tmpdb_read_string_tag(tag_album);
2244             tmpdb_read_string_tag(tag_title);
2245
2246             /* Seek to the end of the string data. */
2247             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2248         }
2249
2250         /* Backup the master index position. */
2251         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2252         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2253
2254         /* Check if we can resurrect some deleted runtime statistics data. */
2255         for (i = 0; i < tcmh.tch.entry_count; i++)
2256         {
2257             /* Read the index entry. */
2258             if (ecread_index_entry(masterfd, &idx)
2259                 != sizeof(struct index_entry))
2260             {
2261                 logf("read fail #3");
2262                 close(masterfd);
2263                 return false;
2264             }
2265
2266             /**
2267              * Skip unless the entry is marked as being deleted
2268              * or the data has already been resurrected.
2269              */
2270             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2271                 continue;
2272
2273             /* Now try to match the entry. */
2274             /**
2275              * To succesfully match a song, the following conditions
2276              * must apply:
2277              *
2278              * For numeric fields: tag_length
2279              * - Full identical match is required
2280              *
2281              * If tag_filename matches, no further checking necessary.
2282              *
2283              * For string hashes: tag_artist, tag_album, tag_title
2284              * - Two of these must match
2285              */
2286             for (j = 0; j < count; j++)
2287             {
2288                 struct temp_file_entry *tfe = &entrybuf[j];
2289
2290                 /* Try to match numeric fields first. */
2291                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2292                     continue;
2293
2294                 /* Now it's time to do the hash matching. */
2295                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2296                 {
2297                     int match_count = 0;
2298
2299                     /* No filename match, check if we can match two other tags. */
2300 #define tmpdb_match(tag) \
2301     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2302         match_count++
2303
2304                     tmpdb_match(tag_artist);
2305                     tmpdb_match(tag_album);
2306                     tmpdb_match(tag_title);
2307
2308                     if (match_count < 2)
2309                     {
2310                         /* Still no match found, give up. */
2311                         continue;
2312                     }
2313                 }
2314
2315                 /* A match found, now copy & resurrect the statistical data. */
2316 #define tmpdb_copy_tag(tag) \
2317     tfe->tag_offset[tag] = idx.tag_seek[tag]
2318
2319                 tmpdb_copy_tag(tag_playcount);
2320                 tmpdb_copy_tag(tag_rating);
2321                 tmpdb_copy_tag(tag_playtime);
2322                 tmpdb_copy_tag(tag_lastplayed);
2323                 tmpdb_copy_tag(tag_commitid);
2324
2325                 /* Avoid processing this entry again. */
2326                 idx.flag |= FLAG_RESURRECTED;
2327
2328                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2329                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2330                 {
2331                     logf("masterfd writeback fail #1");
2332                     close(masterfd);
2333                     return false;
2334                 }
2335
2336                 logf("Entry resurrected");
2337             }
2338         }
2339
2340
2341         /* Restore the master index position. */
2342         lseek(masterfd, masterfd_pos, SEEK_SET);
2343
2344         /* Commit the data to the index. */
2345         for (i = 0; i < count; i++)
2346         {
2347             int loc = lseek(masterfd, 0, SEEK_CUR);
2348
2349             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2350             {
2351                 logf("read fail #3");
2352                 close(masterfd);
2353                 return false;
2354             }
2355
2356             for (j = 0; j < TAG_COUNT; j++)
2357             {
2358                 if (!TAGCACHE_IS_NUMERIC(j))
2359                     continue;
2360
2361                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2362             }
2363             idx.flag = entrybuf[i].flag;
2364
2365             if (idx.tag_seek[tag_commitid])
2366             {
2367                 /* Data has been resurrected. */
2368                 idx.flag |= FLAG_DIRTYNUM;
2369             }
2370             else if (tc_stat.ready && current_tcmh.commitid > 0)
2371             {
2372                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2373                 idx.flag |= FLAG_DIRTYNUM;
2374             }
2375
2376             /* Write back the updated index. */
2377             lseek(masterfd, loc, SEEK_SET);
2378             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2379             {
2380                 logf("write fail");
2381                 close(masterfd);
2382                 return false;
2383             }
2384         }
2385
2386         entries_processed += count;
2387         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2388     }
2389
2390     close(masterfd);
2391
2392     return true;
2393 }
2394
2395 /**
2396  * Return values:
2397  *     > 0   success
2398  *    == 0   temporary failure
2399  *     < 0   fatal error
2400  */
2401 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2402 {
2403     int i;
2404     struct tagcache_header tch;
2405     struct master_header   tcmh;
2406     struct index_entry idxbuf[IDX_BUF_DEPTH];
2407     int idxbuf_pos;
2408     char buf[TAG_MAXLEN+32], path[MAX_PATH];
2409     const char *file;
2410     int fd = -1, masterfd;
2411     bool error = false;
2412     int init;
2413     int masterfd_pos;
2414
2415     logf("Building index: %d", index_type);
2416
2417     /* Check the number of entries we need to allocate ram for. */
2418     commit_entry_count = h->entry_count + 1;
2419
2420     masterfd = open_master_fd(&tcmh, false);
2421     if (masterfd >= 0)
2422     {
2423         commit_entry_count += tcmh.tch.entry_count;
2424         close(masterfd);
2425     }
2426     else
2427         remove_files(); /* Just to be sure we are clean. */
2428
2429     /* Open the index file, which contains the tag names. */
2430     fd = open_tag_fd(&tch, index_type, true);
2431     if (fd >= 0)
2432     {
2433         logf("tch.datasize=%ld", tch.datasize);
2434         lookup_buffer_depth = 1 +
2435         /* First part */ commit_entry_count +
2436         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2437     }
2438     else
2439     {
2440         lookup_buffer_depth = 1 +
2441         /* First part */ commit_entry_count +
2442         /* Second part */ 0;
2443     }
2444
2445     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2446     logf("commit_entry_count=%ld", commit_entry_count);
2447
2448     /* Allocate buffer for all index entries from both old and new
2449      * tag files. */
2450     tempbufidx = 0;
2451     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2452
2453     /* Allocate lookup buffer. The first portion of commit_entry_count
2454      * contains the new tags in the temporary file and the second
2455      * part for locating entries already in the db.
2456      *
2457      *  New tags  Old tags
2458      * +---------+---------------------------+
2459      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2460      * +---------+---------------------------+
2461      *
2462      * Old tags are inserted to a temporary buffer with position:
2463      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2464      * And new tags with index:
2465      *     tempbuf_insert(idx, ...);
2466      *
2467      * The buffer is sorted and written into tag file:
2468      *     tempbuf_sort(...);
2469      * leaving master index locations messed up.
2470      *
2471      * That is fixed using the lookup buffer for old tags:
2472      *     new_seek = tempbuf_find_location(old_seek, ...);
2473      * and for new tags:
2474      *     new_seek = tempbuf_find_location(idx);
2475      */
2476     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2477     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2478     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2479
2480     /* And calculate the remaining data space used mainly for storing
2481      * tag data (strings). */
2482     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2483     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2484     {
2485         logf("Buffer way too small!");
2486         return 0;
2487     }
2488
2489     if (fd >= 0)
2490     {
2491         /**
2492          * If tag file contains unique tags (sorted index), we will load
2493          * it entirely into memory so we can resort it later for use with
2494          * chunked browsing.
2495          */
2496         if (TAGCACHE_IS_SORTED(index_type))
2497         {
2498             logf("loading tags...");
2499             for (i = 0; i < tch.entry_count; i++)
2500             {
2501                 struct tagfile_entry entry;
2502                 int loc = lseek(fd, 0, SEEK_CUR);
2503                 bool ret;
2504
2505                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2506                 {
2507                     logf("read error #7");
2508                     close(fd);
2509                     return -2;
2510                 }
2511
2512                 if (entry.tag_length >= (int)sizeof(buf))
2513                 {
2514                     logf("too long tag #3");
2515                     close(fd);
2516                     return -2;
2517                 }
2518
2519                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2520                 {
2521                     logf("read error #8");
2522                     close(fd);
2523                     return -2;
2524                 }
2525
2526                 /* Skip deleted entries. */
2527                 if (buf[0] == '\0')
2528                     continue;
2529
2530                 /**
2531                  * Save the tag and tag id in the memory buffer. Tag id
2532                  * is saved so we can later reindex the master lookup
2533                  * table when the index gets resorted.
2534                  */
2535                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2536                                      + commit_entry_count, entry.idx_id,
2537                                      TAGCACHE_IS_UNIQUE(index_type));
2538                 if (!ret)
2539                 {
2540                     close(fd);
2541                     return -3;
2542                 }
2543                 do_timed_yield();
2544             }
2545             logf("done");
2546         }
2547         else
2548             tempbufidx = tch.entry_count;
2549     }
2550     else
2551     {
2552         /**
2553          * Creating new index file to store the tags. No need to preload
2554          * anything whether the index type is sorted or not.
2555          */
2556         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2557         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
2558         fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2559         if (fd < 0)
2560         {
2561             logf("%s open fail", buf);
2562             return -2;
2563         }
2564
2565         tch.magic = TAGCACHE_MAGIC;
2566         tch.entry_count = 0;
2567         tch.datasize = 0;
2568
2569         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2570             != sizeof(struct tagcache_header))
2571         {
2572             logf("header write failed");
2573             close(fd);
2574             return -2;
2575         }
2576     }
2577
2578     file = get_user_file_path(TAGCACHE_FILE_MASTER,
2579                                             IS_FILE|NEED_WRITE,
2580                                             buf, sizeof(buf));
2581     /* Loading the tag lookup file as "master file". */
2582     logf("Loading index file");
2583     masterfd = open(file, O_RDWR);
2584
2585     if (masterfd < 0)
2586     {
2587         logf("Creating new DB");
2588         masterfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2589
2590         if (masterfd < 0)
2591         {
2592             logf("Failure to create index file (%s)", file);
2593             close(fd);
2594             return -2;
2595         }
2596
2597         /* Write the header (write real values later). */
2598         memset(&tcmh, 0, sizeof(struct master_header));
2599         tcmh.tch = *h;
2600         tcmh.tch.entry_count = 0;
2601         tcmh.tch.datasize = 0;
2602         tcmh.dirty = true;
2603         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2604         init = true;
2605         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2606     }
2607     else
2608     {
2609         /**
2610          * Master file already exists so we need to process the current
2611          * file first.
2612          */
2613         init = false;
2614
2615         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2616             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2617         {
2618             logf("header error");
2619             close(fd);
2620             close(masterfd);
2621             return -2;
2622         }
2623
2624         /**
2625          * If we reach end of the master file, we need to expand it to
2626          * hold new tags. If the current index is not sorted, we can
2627          * simply append new data to end of the file.
2628          * However, if the index is sorted, we need to update all tag
2629          * pointers in the master file for the current index.
2630          */
2631         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2632             SEEK_CUR);
2633         if (masterfd_pos == filesize(masterfd))
2634         {
2635             logf("appending...");
2636             init = true;
2637         }
2638     }
2639
2640     /**
2641      * Load new unique tags in memory to be sorted later and added
2642      * to the master lookup file.
2643      */
2644     if (TAGCACHE_IS_SORTED(index_type))
2645     {
2646         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2647         /* h is the header of the temporary file containing new tags. */
2648         logf("inserting new tags...");
2649         for (i = 0; i < h->entry_count; i++)
2650         {
2651             struct temp_file_entry entry;
2652
2653             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2654                 sizeof(struct temp_file_entry))
2655             {
2656                 logf("read fail #3");
2657                 error = true;
2658                 goto error_exit;
2659             }
2660
2661             /* Read data. */
2662             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2663             {
2664                 logf("too long entry!");
2665                 error = true;
2666                 goto error_exit;
2667             }
2668
2669             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2670             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2671                 entry.tag_length[index_type])
2672             {
2673                 logf("read fail #4");
2674                 error = true;
2675                 goto error_exit;
2676             }
2677
2678             if (TAGCACHE_IS_UNIQUE(index_type))
2679                 error = !tempbuf_insert(buf, i, -1, true);
2680             else
2681                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2682
2683             if (error)
2684             {
2685                 logf("insert error");
2686                 goto error_exit;
2687             }
2688
2689             /* Skip to next. */
2690             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2691                     entry.tag_length[index_type], SEEK_CUR);
2692             do_timed_yield();
2693         }
2694         logf("done");
2695
2696         /* Sort the buffer data and write it to the index file. */
2697         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2698         /**
2699          * We need to truncate the index file now. There can be junk left
2700          * at the end of file (however, we _should_ always follow the
2701          * entry_count and don't crash with that).
2702          */
2703         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2704
2705         i = tempbuf_sort(fd);
2706         if (i < 0)
2707             goto error_exit;
2708         logf("sorted %d tags", i);
2709
2710         /**
2711          * Now update all indexes in the master lookup file.
2712          */
2713         logf("updating indices...");
2714         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2715         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2716         {
2717             int j;
2718             int loc = lseek(masterfd, 0, SEEK_CUR);
2719
2720             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2721
2722             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2723                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2724             {
2725                 logf("read fail #5");
2726                 error = true;
2727                 goto error_exit ;
2728             }
2729             lseek(masterfd, loc, SEEK_SET);
2730
2731             for (j = 0; j < idxbuf_pos; j++)
2732             {
2733                 if (idxbuf[j].flag & FLAG_DELETED)
2734                 {
2735                     /* We can just ignore deleted entries. */
2736                     // idxbuf[j].tag_seek[index_type] = 0;
2737                     continue;
2738                 }
2739
2740                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2741                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2742                     + commit_entry_count);
2743
2744                 if (idxbuf[j].tag_seek[index_type] < 0)
2745                 {
2746                     logf("update error: %ld/%d/%ld",
2747                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2748                     error = true;
2749                     goto error_exit;
2750                 }
2751
2752                 do_timed_yield();
2753             }
2754
2755             /* Write back the updated index. */
2756             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2757                         index_entry_ec, tc_stat.econ) !=
2758                 (int)sizeof(struct index_entry)*idxbuf_pos)
2759             {
2760                 logf("write fail");
2761                 error = true;
2762                 goto error_exit;
2763             }
2764         }
2765         logf("done");
2766     }
2767
2768     /**
2769      * Walk through the temporary file containing the new tags.
2770      */
2771     // build_normal_index(h, tmpfd, masterfd, idx);
2772     logf("updating new indices...");
2773     lseek(masterfd, masterfd_pos, SEEK_SET);
2774     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2775     lseek(fd, 0, SEEK_END);
2776     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2777     {
2778         int j;
2779
2780         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2781         if (init)
2782         {
2783             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2784         }
2785         else
2786         {
2787             int loc = lseek(masterfd, 0, SEEK_CUR);
2788
2789             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2790                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2791             {
2792                 logf("read fail #6");
2793                 error = true;
2794                 break ;
2795             }
2796             lseek(masterfd, loc, SEEK_SET);
2797         }
2798
2799         /* Read entry headers. */
2800         for (j = 0; j < idxbuf_pos; j++)
2801         {
2802             if (!TAGCACHE_IS_SORTED(index_type))
2803             {
2804                 struct temp_file_entry entry;
2805                 struct tagfile_entry fe;
2806
2807                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2808                     sizeof(struct temp_file_entry))
2809                 {
2810                     logf("read fail #7");
2811                     error = true;
2812                     break ;
2813                 }
2814
2815                 /* Read data. */
2816                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2817                 {
2818                     logf("too long entry!");
2819                     logf("length=%d", entry.tag_length[index_type]);
2820                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2821                     error = true;
2822                     break ;
2823                 }
2824
2825                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2826                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2827                     entry.tag_length[index_type])
2828                 {
2829                     logf("read fail #8");
2830                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2831                     logf("length=0x%02x", entry.tag_length[index_type]);
2832                     error = true;
2833                     break ;
2834                 }
2835
2836                 /* Write to index file. */
2837                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2838                 fe.tag_length = entry.tag_length[index_type];
2839                 fe.idx_id = tcmh.tch.entry_count + i + j;
2840                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2841                 write(fd, buf, fe.tag_length);
2842                 tempbufidx++;
2843
2844                 /* Skip to next. */
2845                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2846                       entry.tag_length[index_type], SEEK_CUR);
2847             }
2848             else
2849             {
2850                 /* Locate the correct entry from the sorted array. */
2851                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2852                 if (idxbuf[j].tag_seek[index_type] < 0)
2853                 {
2854                     logf("entry not found (%d)", j);
2855                     error = true;
2856                     break ;
2857                 }
2858             }
2859         }
2860
2861         /* Write index. */
2862         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2863                   index_entry_ec, tc_stat.econ) !=
2864             (int)sizeof(struct index_entry)*idxbuf_pos)
2865         {
2866             logf("tagcache: write fail #4");
2867             error = true;
2868             break ;
2869         }
2870
2871         do_timed_yield();
2872     }
2873     logf("done");
2874
2875     /* Finally write the header. */
2876     tch.magic = TAGCACHE_MAGIC;
2877     tch.entry_count = tempbufidx;
2878     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2879     lseek(fd, 0, SEEK_SET);
2880     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2881
2882     if (index_type != tag_filename)
2883         h->datasize += tch.datasize;
2884     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2885     error_exit:
2886
2887     close(fd);
2888     close(masterfd);
2889
2890     if (error)
2891         return -2;
2892
2893     return 1;
2894 }
2895
2896 static bool commit(void)
2897 {
2898     struct tagcache_header tch;
2899     struct master_header   tcmh;
2900     char path[MAX_PATH];
2901     const char *file;
2902     int i, len, rc;
2903     int tmpfd;
2904     int masterfd;
2905 #ifdef HAVE_DIRCACHE
2906     bool dircache_buffer_stolen = false;
2907 #endif
2908     bool local_allocation = false;
2909
2910     logf("committing tagcache");
2911
2912     while (write_lock)
2913         sleep(1);
2914
2915     file = get_user_file_path(TAGCACHE_FILE_TEMP,
2916                             IS_FILE|NEED_WRITE, path, sizeof(path));
2917
2918     tmpfd = open(file, O_RDONLY);
2919     if (tmpfd < 0)
2920     {
2921         logf("nothing to commit");
2922         return true;
2923     }
2924
2925
2926     /* Load the header. */
2927     len = sizeof(struct tagcache_header);
2928     rc = read(tmpfd, &tch, len);
2929
2930     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2931     {
2932         logf("incorrect tmpheader");
2933         close(tmpfd);
2934         remove(file);
2935         return false;
2936     }
2937
2938     if (tch.entry_count == 0)
2939     {
2940         logf("nothing to commit");
2941         close(tmpfd);
2942         remove(file);
2943         return true;
2944     }
2945
2946     /* Fully initialize existing headers (if any) before going further. */
2947     tc_stat.ready = check_all_headers();
2948
2949 #ifdef HAVE_EEPROM_SETTINGS
2950     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
2951                 path, sizeof(path)));
2952 #endif
2953
2954     /* At first be sure to unload the ramcache! */
2955 #ifdef HAVE_TC_RAMCACHE
2956     tc_stat.ramcache = false;
2957 #endif
2958
2959     read_lock++;
2960
2961     /* Try to steal every buffer we can :) */
2962     if (tempbuf_size == 0)
2963         local_allocation = true;
2964
2965 #ifdef HAVE_DIRCACHE
2966     if (tempbuf_size == 0)
2967     {
2968         /* Try to steal the dircache buffer. */
2969         tempbuf = dircache_steal_buffer(&tempbuf_size);
2970         tempbuf_size &= ~0x03;
2971
2972         if (tempbuf_size > 0)
2973         {
2974             dircache_buffer_stolen = true;
2975         }
2976     }
2977 #endif
2978
2979 #ifdef HAVE_TC_RAMCACHE
2980     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2981     {
2982         tempbuf = (char *)(hdr + 1);
2983         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2984         tempbuf_size &= ~0x03;
2985     }
2986 #endif
2987
2988     /* And finally fail if there are no buffers available. */
2989     if (tempbuf_size == 0)
2990     {
2991         logf("delaying commit until next boot");
2992         tc_stat.commit_delayed = true;
2993         close(tmpfd);
2994         read_lock--;
2995         return false;
2996     }
2997
2998     logf("commit %ld entries...", tch.entry_count);
2999
3000     /* Mark DB dirty so it will stay disabled if commit fails. */
3001     current_tcmh.dirty = true;
3002     update_master_header();
3003
3004     /* Now create the index files. */
3005     tc_stat.commit_step = 0;
3006     tch.datasize = 0;
3007     tc_stat.commit_delayed = false;
3008
3009     for (i = 0; i < TAG_COUNT; i++)
3010     {
3011         int ret;
3012
3013         if (TAGCACHE_IS_NUMERIC(i))
3014             continue;
3015
3016         tc_stat.commit_step++;
3017         ret = build_index(i, &tch, tmpfd);
3018         if (ret <= 0)
3019         {
3020             close(tmpfd);
3021             logf("tagcache failed init");
3022             if (ret == 0)
3023                 tc_stat.commit_delayed = true;
3024
3025             tc_stat.commit_step = 0;
3026             read_lock--;
3027             return false;
3028         }
3029     }
3030
3031     if (!build_numeric_indices(&tch, tmpfd))
3032     {
3033         logf("Failure to commit numeric indices");
3034         close(tmpfd);
3035         tc_stat.commit_step = 0;
3036         read_lock--;
3037         return false;
3038     }
3039
3040     close(tmpfd);
3041     remove(file);
3042
3043     tc_stat.commit_step = 0;
3044
3045     /* Update the master index headers. */
3046     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3047     {
3048         read_lock--;
3049         return false;
3050     }
3051
3052     tcmh.tch.entry_count += tch.entry_count;
3053     tcmh.tch.datasize = sizeof(struct master_header)
3054         + sizeof(struct index_entry) * tcmh.tch.entry_count
3055         + tch.datasize;
3056     tcmh.dirty = false;
3057     tcmh.commitid++;
3058
3059     lseek(masterfd, 0, SEEK_SET);
3060     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3061     close(masterfd);
3062
3063     logf("tagcache committed");
3064     tc_stat.ready = check_all_headers();
3065     tc_stat.readyvalid = true;
3066
3067     if (local_allocation)
3068     {
3069         tempbuf = NULL;
3070         tempbuf_size = 0;
3071     }
3072
3073 #ifdef HAVE_DIRCACHE
3074     /* Rebuild the dircache, if we stole the buffer. */
3075     if (dircache_buffer_stolen)
3076         dircache_build(0);
3077 #endif
3078
3079 #ifdef HAVE_TC_RAMCACHE
3080     /* Reload tagcache. */
3081     if (tc_stat.ramcache_allocated > 0)
3082         tagcache_start_scan();
3083 #endif
3084
3085     read_lock--;
3086
3087     return true;
3088 }
3089
3090 static void allocate_tempbuf(void)
3091 {
3092     /* Yeah, malloc would be really nice now :) */
3093 #ifdef __PCTOOL__
3094     tempbuf_size = 32*1024*1024;
3095     tempbuf = malloc(tempbuf_size);
3096 #else
3097     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3098     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3099     audiobuf += tempbuf_size;
3100 #endif
3101 }
3102
3103 static void free_tempbuf(void)
3104 {
3105     if (tempbuf_size == 0)
3106         return ;
3107
3108 #ifdef __PCTOOL__
3109     free(tempbuf);
3110 #else
3111     audiobuf -= tempbuf_size;
3112 #endif
3113     tempbuf = NULL;
3114     tempbuf_size = 0;
3115 }
3116
3117 #ifndef __PCTOOL__
3118
3119 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3120 {
3121     struct index_entry idx;
3122
3123     if (!tc_stat.ready)
3124         return false;
3125
3126     if (!TAGCACHE_IS_NUMERIC(tag))
3127         return false;
3128
3129     if (!get_index(masterfd, idx_id, &idx, false))
3130         return false;
3131
3132     idx.tag_seek[tag] = data;
3133     idx.flag |= FLAG_DIRTYNUM;
3134
3135     return write_index(masterfd, idx_id, &idx);
3136 }
3137
3138 #if 0
3139 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3140                                    int tag, long data)
3141 {
3142     struct master_header myhdr;
3143
3144     if (tcs->masterfd < 0)
3145     {
3146         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3147             return false;
3148     }
3149
3150     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3151 }
3152 #endif
3153
3154 static bool command_queue_is_full(void)
3155 {
3156     int next;
3157
3158     next = command_queue_widx + 1;
3159     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3160         next = 0;
3161
3162     return (next == command_queue_ridx);
3163 }
3164
3165 static void command_queue_sync_callback(void *data)
3166 {
3167     (void)data;
3168     struct master_header myhdr;
3169     int masterfd;
3170
3171     mutex_lock(&command_queue_mutex);
3172
3173     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3174         return;
3175
3176     while (command_queue_ridx != command_queue_widx)
3177     {
3178         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3179
3180         switch (ce->command)
3181         {
3182             case CMD_UPDATE_MASTER_HEADER:
3183             {
3184                 close(masterfd);
3185                 update_master_header();
3186
3187                 /* Re-open the masterfd. */
3188                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3189                     return;
3190
3191                 break;
3192             }
3193             case CMD_UPDATE_NUMERIC:
3194             {
3195                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3196                 break;
3197             }
3198         }
3199
3200         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3201             command_queue_ridx = 0;
3202     }
3203
3204     close(masterfd);
3205
3206     tc_stat.queue_length = 0;
3207     mutex_unlock(&command_queue_mutex);
3208 }
3209
3210 static void run_command_queue(bool force)
3211 {
3212     if (COMMAND_QUEUE_IS_EMPTY)
3213         return;
3214
3215     if (force || command_queue_is_full())
3216         command_queue_sync_callback(NULL);
3217     else
3218         register_storage_idle_func(command_queue_sync_callback);
3219 }
3220
3221 static void queue_command(int cmd, long idx_id, int tag, long data)
3222 {
3223     while (1)
3224     {
3225         int next;
3226
3227         mutex_lock(&command_queue_mutex);
3228         next = command_queue_widx + 1;
3229         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3230             next = 0;
3231
3232         /* Make sure queue is not full. */
3233         if (next != command_queue_ridx)
3234         {
3235             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3236
3237             ce->command = cmd;
3238             ce->idx_id = idx_id;
3239             ce->tag = tag;
3240             ce->data = data;
3241
3242             command_queue_widx = next;
3243
3244             tc_stat.queue_length++;
3245
3246             mutex_unlock(&command_queue_mutex);
3247             break;
3248         }
3249
3250         /* Queue is full, try again later... */
3251         mutex_unlock(&command_queue_mutex);
3252         sleep(1);
3253     }
3254 }
3255
3256 long tagcache_increase_serial(void)
3257 {
3258     long old;
3259
3260     if (!tc_stat.ready)
3261         return -2;
3262
3263     while (read_lock)
3264         sleep(1);
3265
3266     old = current_tcmh.serial++;
3267     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3268
3269     return old;
3270 }
3271
3272 void tagcache_update_numeric(int idx_id, int tag, long data)
3273 {
3274     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3275 }
3276 #endif /* !__PCTOOL__ */
3277
3278 long tagcache_get_serial(void)
3279 {
3280     return current_tcmh.serial;
3281 }
3282
3283 long tagcache_get_commitid(void)
3284 {
3285     return current_tcmh.commitid;
3286 }
3287
3288 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3289 {
3290     char buf[512];
3291     int i;
3292
3293     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3294     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3295     {
3296         if (*datastr == '\0')
3297             break;
3298
3299         if (*datastr == '"' || *datastr == '\\')
3300             buf[i++] = '\\';
3301
3302         buf[i] = *(datastr++);
3303     }
3304
3305     strcpy(&buf[i], "\" ");
3306
3307     write(fd, buf, i + 2);
3308
3309     return true;
3310 }
3311
3312 #ifndef __PCTOOL__
3313
3314 static bool read_tag(char *dest, long size,
3315                      const char *src, const char *tagstr)
3316 {
3317     int pos;
3318     char current_tag[32];
3319
3320     while (*src != '\0')
3321     {
3322         /* Skip all whitespace */
3323         while (*src == ' ')
3324             src++;
3325
3326         if (*src == '\0')
3327             break;
3328
3329         pos = 0;
3330         /* Read in tag name */
3331         while (*src != '=' && *src != ' ')
3332         {
3333             current_tag[pos] = *src;
3334             src++;
3335             pos++;
3336
3337             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3338                 return false;
3339         }
3340         current_tag[pos] = '\0';
3341
3342         /* Read in tag data */
3343
3344         /* Find the start. */
3345         while (*src != '"' && *src != '\0')
3346             src++;
3347
3348         if (*src == '\0' || *(++src) == '\0')
3349             return false;
3350
3351         /* Read the data. */
3352         for (pos = 0; pos < size; pos++)
3353         {
3354             if (*src == '\0')
3355                 break;
3356
3357             if (*src == '\\')
3358             {
3359                 dest[pos] = *(src+1);
3360                 src += 2;
3361                 continue;
3362             }
3363
3364             dest[pos] = *src;
3365
3366             if (*src == '"')
3367             {
3368                 src++;
3369                 break;
3370             }
3371
3372             if (*src == '\0')
3373                 break;
3374
3375             src++;
3376         }
3377         dest[pos] = '\0';
3378
3379         if (!strcasecmp(tagstr, current_tag))
3380             return true;
3381     }
3382
3383     return false;
3384 }
3385
3386 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3387 {
3388     struct index_entry idx;
3389     char tag_data[TAG_MAXLEN+32];
3390     int idx_id;
3391     long masterfd = (long)parameters;
3392     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3393         tag_commitid };
3394     int i;
3395     (void)line_n;
3396
3397     if (*buf == '#')
3398         return 0;
3399
3400     logf("%d/%s", line_n, buf);
3401     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3402     {
3403         logf("filename missing");
3404         logf("-> %s", buf);
3405         return 0;
3406     }
3407
3408     idx_id = find_index(tag_data);
3409     if (idx_id < 0)
3410     {
3411         logf("entry not found");
3412         return 0;
3413     }
3414
3415     if (!get_index(masterfd, idx_id, &idx, false))
3416     {
3417         logf("failed to retrieve index entry");
3418         return 0;
3419     }
3420
3421     /* Stop if tag has already been modified. */
3422     if (idx.flag & FLAG_DIRTYNUM)
3423         return 0;
3424
3425     logf("import: %s", tag_data);
3426
3427     idx.flag |= FLAG_DIRTYNUM;
3428     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3429     {
3430         int data;
3431
3432         if (!read_tag(tag_data, sizeof tag_data, buf,
3433                       tagcache_tag_to_str(import_tags[i])))
3434         {
3435             continue;
3436         }
3437
3438         data = atoi(tag_data);
3439         if (data < 0)
3440             continue;
3441
3442         idx.tag_seek[import_tags[i]] = data;
3443
3444         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3445             current_tcmh.serial = data + 1;
3446         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3447             current_tcmh.commitid = data + 1;
3448     }
3449
3450     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3451 }
3452
3453 bool tagcache_import_changelog(void)
3454 {
3455     struct master_header myhdr;
3456     struct tagcache_header tch;
3457     int clfd;
3458     long masterfd;
3459     char buf[MAX(MAX_PATH, 2048)];
3460     const char *file;
3461
3462     if (!tc_stat.ready)
3463         return false;
3464
3465     while (read_lock)
3466         sleep(1);
3467
3468     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG,
3469                                    IS_FILE|NEED_WRITE, buf, sizeof(buf));
3470     clfd = open(file, O_RDONLY);
3471     if (clfd < 0)
3472     {
3473         logf("failure to open changelog");
3474         return false;
3475     }
3476
3477     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3478     {
3479         close(clfd);
3480         return false;
3481     }
3482
3483     write_lock++;
3484
3485     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3486
3487     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3488                   parse_changelog_line);
3489
3490     close(clfd);
3491     close(masterfd);
3492
3493     if (filenametag_fd >= 0)
3494     {
3495         close(filenametag_fd);
3496         filenametag_fd = -1;
3497     }
3498
3499     write_lock--;
3500
3501     update_master_header();
3502
3503     return true;
3504 }
3505
3506 #endif /* !__PCTOOL__ */
3507
3508 bool tagcache_create_changelog(struct tagcache_search *tcs)
3509 {
3510     struct master_header myhdr;
3511     struct index_entry idx;
3512     const char *file;
3513     char buf[MAX(TAG_MAXLEN+32, MAX_PATH)];
3514     char temp[32];
3515     int clfd;
3516     int i, j;
3517
3518     if (!tc_stat.ready)
3519         return false;
3520
3521     if (!tagcache_search(tcs, tag_filename))
3522         return false;
3523
3524     /* Initialize the changelog */
3525     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG, IS_FILE | NEED_WRITE,
3526                               buf, sizeof(buf));
3527     clfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3528     if (clfd < 0)
3529     {
3530         logf("failure to open changelog");
3531         return false;
3532     }
3533
3534     if (tcs->masterfd < 0)
3535     {
3536         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3537             return false;
3538     }
3539     else
3540     {
3541         lseek(tcs->masterfd, 0, SEEK_SET);
3542         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3543     }
3544
3545     write(clfd, "## Changelog version 1\n", 23);
3546
3547     for (i = 0; i < myhdr.tch.entry_count; i++)
3548     {
3549         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3550         {
3551             logf("read error #9");
3552             tagcache_search_finish(tcs);
3553             close(clfd);
3554             return false;
3555         }
3556
3557         /* Skip until the entry found has been modified. */
3558         if (! (idx.flag & FLAG_DIRTYNUM) )
3559             continue;
3560
3561         /* Skip deleted entries too. */
3562         if (idx.flag & FLAG_DELETED)
3563             continue;
3564
3565         /* Now retrieve all tags. */
3566         for (j = 0; j < TAG_COUNT; j++)
3567         {
3568             if (TAGCACHE_IS_NUMERIC(j))
3569             {
3570                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3571                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3572                 continue;
3573             }
3574
3575             tcs->type = j;
3576             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3577             write_tag(clfd, tagcache_tag_to_str(j), buf);
3578         }
3579
3580         write(clfd, "\n", 1);
3581         do_timed_yield();
3582     }
3583
3584     close(clfd);
3585
3586     tagcache_search_finish(tcs);
3587
3588     return true;
3589 }
3590
3591 static bool delete_entry(long idx_id)
3592 {
3593     int fd = -1;
3594     int masterfd = -1;
3595     int tag, i;
3596     struct index_entry idx, myidx;
3597     struct master_header myhdr;
3598     char buf[TAG_MAXLEN+32];
3599     int in_use[TAG_COUNT];
3600
3601     logf("delete_entry(): %ld", idx_id);
3602
3603 #ifdef HAVE_TC_RAMCACHE
3604     /* At first mark the entry removed from ram cache. */
3605     if (tc_stat.ramcache)
3606         hdr->indices[idx_id].flag |= FLAG_DELETED;
3607 #endif
3608
3609     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3610         return false;
3611
3612     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3613     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3614     {
3615         logf("delete_entry(): read error");
3616         goto cleanup;
3617     }
3618
3619     if (myidx.flag & FLAG_DELETED)
3620     {
3621         logf("delete_entry(): already deleted!");
3622         goto cleanup;
3623     }
3624
3625     myidx.flag |= FLAG_DELETED;
3626     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3627     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3628     {
3629         logf("delete_entry(): write_error #1");
3630         goto cleanup;
3631     }
3632
3633     /* Now check which tags are no longer in use (if any) */
3634     for (tag = 0; tag < TAG_COUNT; tag++)
3635         in_use[tag] = 0;
3636
3637     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3638     for (i = 0; i < myhdr.tch.entry_count; i++)
3639     {
3640         struct index_entry *idxp;
3641
3642 #ifdef HAVE_TC_RAMCACHE
3643         /* Use RAM DB if available for greater speed */
3644         if (tc_stat.ramcache)
3645             idxp = &hdr->indices[i];
3646         else
3647 #endif
3648         {
3649             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3650             {
3651                 logf("delete_entry(): read error #2");
3652                 goto cleanup;
3653             }
3654             idxp = &idx;
3655         }
3656
3657         if (idxp->flag & FLAG_DELETED)
3658             continue;
3659
3660         for (tag = 0; tag < TAG_COUNT; tag++)
3661         {
3662             if (TAGCACHE_IS_NUMERIC(tag))
3663                 continue;
3664
3665             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3666                 in_use[tag]++;
3667         }
3668     }
3669
3670     /* Now delete all tags no longer in use. */
3671     for (tag = 0; tag < TAG_COUNT; tag++)
3672     {
3673         struct tagcache_header tch;
3674         int oldseek = myidx.tag_seek[tag];
3675
3676         if (TAGCACHE_IS_NUMERIC(tag))
3677             continue;
3678
3679         /**
3680          * Replace tag seek with a hash value of the field string data.
3681          * That way runtime statistics of moved or altered files can be
3682          * resurrected.
3683          */
3684 #ifdef HAVE_TC_RAMCACHE
3685         if (tc_stat.ramcache && tag != tag_filename)
3686         {
3687             struct tagfile_entry *tfe;
3688             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3689
3690             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3691             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3692             myidx.tag_seek[tag] = *seek;
3693         }
3694         else
3695 #endif
3696         {
3697             struct tagfile_entry tfe;
3698
3699             /* Open the index file, which contains the tag names. */
3700             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3701                 goto cleanup;
3702
3703             /* Skip the header block */
3704             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3705             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3706             {
3707                 logf("delete_entry(): read error #3");
3708                 goto cleanup;
3709             }
3710
3711             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3712             {
3713                 logf("delete_entry(): read error #4");
3714                 goto cleanup;
3715             }
3716
3717             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3718         }
3719
3720         if (in_use[tag])
3721         {
3722             logf("in use: %d/%d", tag, in_use[tag]);
3723             if (fd >= 0)
3724             {
3725                 close(fd);
3726                 fd = -1;
3727             }
3728             continue;
3729         }
3730
3731 #ifdef HAVE_TC_RAMCACHE
3732         /* Delete from ram. */
3733         if (tc_stat.ramcache && tag != tag_filename)
3734         {
3735             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3736             tagentry->tag_data[0] = '\0';
3737         }
3738 #endif
3739
3740         /* Open the index file, which contains the tag names. */
3741         if (fd < 0)
3742         {
3743             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3744                 goto cleanup;
3745         }
3746
3747         /* Skip the header block */
3748         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3749
3750         /* Debug, print 10 first characters of the tag
3751         read(fd, buf, 10);
3752         buf[10]='\0';
3753         logf("TAG:%s", buf);
3754         lseek(fd, -10, SEEK_CUR);
3755         */
3756
3757         /* Write first data byte in tag as \0 */
3758         write(fd, "", 1);
3759
3760         /* Now tag data has been removed */
3761         close(fd);
3762         fd = -1;
3763     }
3764
3765     /* Write index entry back into master index. */
3766     lseek(masterfd, sizeof(struct master_header) +
3767           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3768     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3769     {
3770         logf("delete_entry(): write_error #2");
3771         goto cleanup;
3772     }
3773
3774     close(masterfd);
3775
3776     return true;
3777
3778     cleanup:
3779     if (fd >= 0)
3780         close(fd);
3781     if (masterfd >= 0)
3782         close(masterfd);
3783
3784     return false;
3785 }
3786
3787 #ifndef __PCTOOL__
3788 /**
3789  * Returns true if there is an event waiting in the queue
3790  * that requires the current operation to be aborted.
3791  */
3792 static bool check_event_queue(void)
3793 {
3794     struct queue_event ev;
3795
3796     if(!queue_peek(&tagcache_queue, &ev))
3797         return false;
3798
3799     switch (ev.id)
3800     {
3801         case Q_STOP_SCAN:
3802         case SYS_POWEROFF:
3803         case SYS_USB_CONNECTED:
3804             return true;
3805     }
3806
3807     return false;
3808 }
3809 #endif
3810
3811 #ifdef HAVE_TC_RAMCACHE
3812 static bool allocate_tagcache(void)
3813 {
3814     struct master_header tcmh;
3815     int fd;
3816
3817     /* Load the header. */
3818     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3819     {
3820         hdr = NULL;
3821         return false;
3822     }
3823
3824     close(fd);
3825
3826     /**
3827      * Now calculate the required cache size plus
3828      * some extra space for alignment fixes.
3829      */
3830     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3831         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3832     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3833     memset(hdr, 0, sizeof(struct ramcache_header));
3834     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3835     hdr->indices = (struct index_entry *)(hdr + 1);
3836     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3837
3838     return true;
3839 }
3840
3841 # ifdef HAVE_EEPROM_SETTINGS
3842 static bool tagcache_dumpload(void)
3843 {
3844     struct statefile_header shdr;
3845     char path[MAX_PATH];
3846     const char *file;
3847     int fd, rc;
3848     long offpos;
3849     int i;
3850
3851     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3852                                     path, sizeof(path));
3853     fd = open(file, O_RDONLY);
3854     if (fd < 0)
3855     {
3856         logf("no tagcache statedump");
3857         return false;
3858     }
3859
3860     /* Check the statefile memory placement */
3861     hdr = buffer_alloc(0);
3862     rc = read(fd, &shdr, sizeof(struct statefile_header));
3863     if (rc != sizeof(struct statefile_header)
3864         /* || (long)hdr != (long)shdr.hdr */)
3865     {
3866         logf("incorrect statefile");
3867         hdr = NULL;
3868         close(fd);
3869         return false;
3870     }
3871
3872     offpos = (long)hdr - (long)shdr.hdr;
3873
3874     /* Lets allocate real memory and load it */
3875     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3876     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3877     close(fd);
3878
3879     if (rc != shdr.tc_stat.ramcache_allocated)
3880     {
3881         logf("read failure!");
3882         hdr = NULL;
3883         return false;
3884     }
3885
3886     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3887
3888     /* Now fix the pointers */
3889     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3890     for (i = 0; i < TAG_COUNT; i++)
3891         hdr->tags[i] += offpos;
3892
3893     return true;
3894 }
3895
3896 static bool tagcache_dumpsave(void)
3897 {
3898     struct statefile_header shdr;
3899     char path[MAX_PATH];
3900     const char *file;
3901     int fd;
3902
3903     if (!tc_stat.ramcache)
3904         return false;
3905
3906     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3907                                     path, sizeof(path));
3908     fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3909     if (fd < 0)
3910     {
3911         logf("failed to create a statedump");
3912         return false;
3913     }
3914
3915     /* Create the header */
3916     shdr.hdr = hdr;
3917     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3918     write(fd, &shdr, sizeof(struct statefile_header));
3919
3920     /* And dump the data too */
3921     write(fd, hdr, tc_stat.ramcache_allocated);
3922     close(fd);
3923
3924     return true;
3925 }
3926 # endif
3927
3928 static bool load_tagcache(void)
3929 {
3930     struct tagcache_header *tch;
3931     long bytesleft = tc_stat.ramcache_allocated;
3932     struct index_entry *idx;
3933     int rc, fd;
3934     char *p, path[MAX_PATH];
3935     const char *file;
3936     int i, tag;
3937
3938 # ifdef HAVE_DIRCACHE
3939     while (dircache_is_initializing())
3940         sleep(1);
3941
3942     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3943 # endif
3944
3945     logf("loading tagcache to ram...");
3946
3947     file = get_user_file_path(TAGCACHE_FILE_MASTER,
3948                               IS_FILE|NEED_WRITE,
3949                               path, sizeof(path));
3950     fd = open(file, O_RDONLY);
3951     if (fd < 0)
3952     {
3953         logf("tagcache open failed");
3954         return false;
3955     }
3956
3957     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3958         != sizeof(struct master_header)
3959         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3960     {
3961         logf("incorrect header");
3962         return false;
3963     }
3964
3965     idx = hdr->indices;
3966
3967     /* Load the master index table. */
3968     for (i = 0; i < hdr->h.tch.entry_count; i++)
3969     {
3970         rc = ecread_index_entry(fd, idx);
3971         if (rc != sizeof(struct index_entry))
3972         {
3973             logf("read error #10");
3974             close(fd);
3975             return false;
3976         }
3977
3978         bytesleft -= sizeof(struct index_entry);
3979         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3980         {
3981             logf("too big tagcache.");
3982             close(fd);
3983             return false;
3984         }
3985
3986         idx++;
3987     }
3988
3989     close(fd);
3990
3991     /* Load the tags. */
3992     p = (char *)idx;
3993     for (tag = 0; tag < TAG_COUNT; tag++)
3994     {
3995         struct tagfile_entry *fe;
3996         char buf[TAG_MAXLEN+32];
3997
3998         if (TAGCACHE_IS_NUMERIC(tag))
3999             continue ;
4000
4001         //p = ((void *)p+1);
4002         p = (char *)((long)p & ~0x03) + 0x04;
4003         hdr->tags[tag] = p;
4004
4005         /* Check the header. */
4006         tch = (struct tagcache_header *)p;
4007         p += sizeof(struct tagcache_header);
4008
4009         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
4010             return false;
4011
4012         for (hdr->entry_count[tag] = 0;
4013              hdr->entry_count[tag] < tch->entry_count;
4014              hdr->entry_count[tag]++)
4015         {
4016             long pos;
4017
4018             if (do_timed_yield())
4019             {
4020                 /* Abort if we got a critical event in queue */
4021                 if (check_event_queue())
4022                     return false;
4023             }
4024
4025             fe = (struct tagfile_entry *)p;
4026             pos = lseek(fd, 0, SEEK_CUR);
4027             rc = ecread_tagfile_entry(fd, fe);
4028             if (rc != sizeof(struct tagfile_entry))
4029             {
4030                 /* End of lookup table. */
4031                 logf("read error #11");
4032                 close(fd);
4033                 return false;
4034             }
4035
4036             /* We have a special handling for the filename tags. */
4037             if (tag == tag_filename)
4038             {
4039 # ifdef HAVE_DIRCACHE
4040                 const struct dircache_entry *dc;
4041 # endif
4042
4043                 // FIXME: This is wrong!
4044                 // idx = &hdr->indices[hdr->entry_count[i]];
4045                 idx = &hdr->indices[fe->idx_id];
4046
4047                 if (fe->tag_length >= (long)sizeof(buf)-1)
4048                 {
4049                     read(fd, buf, 10);
4050                     buf[10] = '\0';
4051                     logf("TAG:%s", buf);
4052                     logf("too long filename");
4053                     close(fd);
4054                     return false;
4055                 }
4056
4057                 rc = read(fd, buf, fe->tag_length);
4058                 if (rc != fe->tag_length)
4059                 {
4060                     logf("read error #12");
4061                     close(fd);
4062                     return false;
4063                 }
4064
4065                 /* Check if the entry has already been removed */
4066                 if (idx->flag & FLAG_DELETED)
4067                     continue;
4068
4069                 /* This flag must not be used yet. */
4070                 if (idx->flag & FLAG_DIRCACHE)
4071                 {
4072                     logf("internal error!");
4073                     close(fd);
4074                     return false;
4075                 }
4076
4077                 if (idx->tag_seek[tag] != pos)
4078                 {
4079                     logf("corrupt data structures!");
4080                     close(fd);
4081                     return false;
4082                 }
4083
4084 # ifdef HAVE_DIRCACHE
4085                 if (dircache_is_enabled())
4086                 {
4087                     dc = dircache_get_entry_ptr(buf);
4088                     if (dc == NULL)
4089                     {
4090                         logf("Entry no longer valid.");
4091                         logf("-> %s", buf);
4092                         if (global_settings.tagcache_autoupdate)
4093                             delete_entry(fe->idx_id);
4094                         continue ;
4095                     }
4096
4097                     idx->flag |= FLAG_DIRCACHE;
4098                     idx->tag_seek[tag_filename] = (long)dc;
4099                 }
4100                 else
4101 # endif
4102                 {
4103                     /* This will be very slow unless dircache is enabled
4104                        or target is flash based, but do it anyway for
4105                        consistency. */
4106                     /* Check if entry has been removed. */
4107                     if (global_settings.tagcache_autoupdate)
4108                     {
4109                         if (!file_exists(buf))
4110                         {
4111                             logf("Entry no longer valid.");
4112                             logf("-> %s", buf);
4113                             delete_entry(fe->idx_id);
4114                             continue;
4115                         }
4116                     }
4117                 }
4118
4119                 continue ;
4120             }
4121
4122             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4123             if (bytesleft < 0)
4124             {
4125                 logf("too big tagcache #2");
4126                 logf("tl: %ld", fe->tag_length);
4127                 logf("bl: %ld", bytesleft);
4128                 close(fd);
4129                 return false;
4130             }
4131
4132             p = fe->tag_data;
4133             rc = read(fd, fe->tag_data, fe->tag_length);
4134             p += rc;
4135
4136             if (rc != fe->tag_length)
4137             {
4138                 logf("read error #13");
4139                 logf("rc=0x%04x", rc); // 0x431
4140                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4141                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4142                 logf("tag=0x%02x", tag); // 0x00
4143                 close(fd);
4144                 return false;
4145             }
4146         }
4147         close(fd);
4148     }
4149
4150     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4151     logf("tagcache loaded into ram!");
4152
4153     return true;
4154 }
4155 #endif /* HAVE_TC_RAMCACHE */
4156
4157 static bool check_deleted_files(void)
4158 {
4159     int fd;
4160     char buf[TAG_MAXLEN+32], path[MAX_PATH];
4161     const char *file;
4162     struct tagfile_entry tfe;
4163
4164     logf("reverse scan...");
4165     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4166     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
4167     fd = open(file, O_RDONLY);
4168
4169     if (fd < 0)
4170     {
4171         logf("%s open fail", buf);
4172         return false;
4173     }
4174
4175     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4176     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4177 #ifndef __PCTOOL__
4178            && !check_event_queue()
4179 #endif
4180            )
4181     {
4182         if (tfe.tag_length >= (long)sizeof(buf)-1)
4183         {
4184             logf("too long tag");
4185             close(fd);
4186             return false;
4187         }
4188
4189         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4190         {
4191             logf("read error #14");
4192             close(fd);
4193             return false;
4194         }
4195
4196         /* Check if the file has already deleted from the db. */
4197         if (*buf == '\0')
4198             continue;
4199
4200         /* Now check if the file exists. */
4201         if (!file_exists(buf))
4202         {
4203             logf("Entry no longer valid.");
4204             logf("-> %s / %ld", buf, tfe.tag_length);
4205             delete_entry(tfe.idx_id);
4206         }
4207     }
4208
4209     close(fd);
4210
4211     logf("done");
4212
4213     return true;
4214 }
4215
4216
4217 /* Note that this function must not be inlined, otherwise the whole point
4218  * of having the code in a separate function is lost.
4219  */
4220 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4221     int *ignore, int *unignore)
4222 {
4223     char newpath[MAX_PATH];
4224
4225     /* check for a database.ignore file */
4226     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4227     *ignore = file_exists(newpath);
4228     /* check for a database.unignore file */
4229     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4230     *unignore = file_exists(newpath);
4231 }
4232
4233
4234 static bool check_dir(const char *dirname, int add_files)
4235 {
4236     DIR *dir;
4237     int len;
4238     int success = false;
4239     int ignore, unignore;
4240
4241     dir = opendir(dirname);
4242     if (!dir)
4243     {
4244         logf("tagcache: opendir(%s) failed", dirname);
4245         return false;
4246     }
4247
4248     /* check for a database.ignore and database.unignore */
4249     check_ignore(dirname, &ignore, &unignore);
4250
4251     /* don't do anything if both ignore and unignore are there */
4252     if (ignore != unignore)
4253         add_files = unignore;
4254
4255     /* Recursively scan the dir. */
4256 #ifdef __PCTOOL__
4257     while (1)
4258 #else
4259     while (!check_event_queue())
4260 #endif
4261     {
4262         struct dirent *entry;
4263
4264         entry = readdir(dir);
4265
4266         if (entry == NULL)
4267         {
4268             success = true;
4269             break ;
4270         }
4271
4272         struct dirinfo info = dir_get_info(dir, entry);
4273
4274         if (!strcmp((char *)entry->d_name, ".") ||
4275             !strcmp((char *)entry->d_name, ".."))
4276             continue;
4277
4278         yield();
4279
4280         len = strlen(curpath);
4281         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4282                  entry->d_name);
4283
4284         processed_dir_count++;
4285         if (info.attribute & ATTR_DIRECTORY)
4286             check_dir(curpath, add_files);
4287         else if (add_files)
4288         {
4289             tc_stat.curentry = curpath;
4290
4291             /* Add a new entry to the temporary db file. */
4292             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4293 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4294                          , dir->internal_entry
4295 #endif
4296                          );
4297
4298             /* Wait until current path for debug screen is read and unset. */
4299             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4300                 yield();
4301
4302             tc_stat.curentry = NULL;
4303         }
4304
4305         curpath[len] = '\0';
4306     }
4307
4308     closedir(dir);
4309
4310     return success;
4311 }
4312
4313 void tagcache_screensync_event(void)
4314 {
4315     tc_stat.curentry = NULL;
4316 }
4317
4318 void tagcache_screensync_enable(bool state)
4319 {
4320     tc_stat.syncscreen = state;
4321 }
4322
4323 void tagcache_build(const char *path)
4324 {
4325     struct tagcache_header header;
4326     bool ret;
4327     char buf[MAX_PATH];
4328     const char *file;
4329
4330     curpath[0] = '\0';
4331     data_size = 0;
4332     total_entry_count = 0;
4333     processed_dir_count = 0;
4334
4335 #ifdef HAVE_DIRCACHE
4336     while (dircache_is_initializing())
4337         sleep(1);
4338 #endif
4339
4340     logf("updating tagcache");
4341
4342     file = get_user_file_path(TAGCACHE_FILE_TEMP,
4343                                     IS_FILE|NEED_WRITE, buf, sizeof(buf));
4344
4345
4346     if (file_exists(file))
4347     {
4348         logf("skipping, cache already waiting for commit");
4349         return ;
4350     }
4351
4352     cachefd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0666);
4353     if (cachefd < 0)
4354     {
4355         logf("master file open failed: %s", file);
4356         return ;
4357     }
4358
4359     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4360
4361     cpu_boost(true);
4362
4363     logf("Scanning files...");
4364     /* Scan for new files. */
4365     memset(&header, 0, sizeof(struct tagcache_header));
4366     write(cachefd, &header, sizeof(struct tagcache_header));
4367
4368     if (strcmp("/", path) != 0)
4369         strcpy(curpath, path);
4370     ret = check_dir(path, true);
4371
4372     /* Write the header. */
4373     header.magic = TAGCACHE_MAGIC;
4374     header.datasize = data_size;
4375     header.entry_count = total_entry_count;
4376     lseek(cachefd, 0, SEEK_SET);
4377     write(cachefd, &header, sizeof(struct tagcache_header));
4378     close(cachefd);
4379
4380     if (filenametag_fd >= 0)
4381     {
4382         close(filenametag_fd);
4383         filenametag_fd = -1;
4384     }
4385
4386     if (!ret)
4387     {
4388         logf("Aborted.");
4389         cpu_boost(false);
4390         return ;
4391     }
4392
4393     /* Commit changes to the database. */
4394 #ifdef __PCTOOL__
4395     allocate_tempbuf();
4396 #endif
4397     if (commit())
4398     {
4399         remove(file);
4400         logf("tagcache built!");
4401     }
4402 #ifdef __PCTOOL__
4403     free_tempbuf();
4404 #endif
4405
4406 #ifdef HAVE_TC_RAMCACHE
4407     if (hdr)
4408     {
4409         /* Import runtime statistics if we just initialized the db. */
4410         if (hdr->h.serial == 0)
4411             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4412     }
4413 #endif
4414
4415     cpu_boost(false);
4416 }
4417
4418 #ifdef HAVE_TC_RAMCACHE
4419 static void load_ramcache(void)
4420 {
4421     if (!hdr)
4422         return ;
4423
4424     cpu_boost(true);
4425
4426     /* At first we should load the cache (if exists). */
4427     tc_stat.ramcache = load_tagcache();
4428
4429     if (!tc_stat.ramcache)
4430     {
4431         /* If loading failed, it must indicate some problem with the db
4432          * so disable it entirely to prevent further issues. */
4433         tc_stat.ready = false;
4434         hdr = NULL;
4435     }
4436
4437     cpu_boost(false);
4438 }
4439
4440 void tagcache_unload_ramcache(void)
4441 {
4442     tc_stat.ramcache = false;
4443     /* Just to make sure there is no statefile present. */
4444
4445 #if 0
4446     char path[MAX_PATH];
4447     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4448                 path, sizeof(path)));
4449 #endif
4450 }
4451 #endif
4452
4453 #ifndef __PCTOOL__
4454 static void tagcache_thread(void)
4455 {
4456     struct queue_event ev;
4457     bool check_done = false;
4458     char path[MAX_PATH];
4459
4460     /* If the previous cache build/update was interrupted, commit
4461      * the changes first in foreground. */
4462     cpu_boost(true);
4463     allocate_tempbuf();
4464     commit();
4465     free_tempbuf();
4466
4467 #ifdef HAVE_TC_RAMCACHE
4468 # ifdef HAVE_EEPROM_SETTINGS
4469     if (firmware_settings.initialized && firmware_settings.disk_clean
4470         && global_settings.tagcache_ram)
4471     {
4472         check_done = tagcache_dumpload();
4473     }
4474
4475     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4476                 path, sizeof(path)));
4477 # endif
4478
4479     /* Allocate space for the tagcache if found on disk. */
4480     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4481         allocate_tagcache();
4482 #endif
4483
4484     cpu_boost(false);
4485     tc_stat.initialized = true;
4486
4487     /* Don't delay bootup with the header check but do it on background. */
4488     if (!tc_stat.ready)
4489     {
4490         sleep(HZ);
4491         tc_stat.ready = check_all_headers();
4492         tc_stat.readyvalid = true;
4493     }
4494
4495     while (1)
4496     {
4497         run_command_queue(false);
4498
4499         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4500
4501         switch (ev.id)
4502         {
4503             case Q_IMPORT_CHANGELOG:
4504                 tagcache_import_changelog();
4505                 break;
4506
4507             case Q_REBUILD:
4508                 remove_files();
4509                 remove(get_user_file_path(TAGCACHE_FILE_TEMP,
4510                                 IS_FILE|NEED_WRITE, path, sizeof(path)));
4511                 tagcache_build("/");
4512                 break;
4513
4514             case Q_UPDATE:
4515                 tagcache_build("/");
4516 #ifdef HAVE_TC_RAMCACHE
4517                 load_ramcache();
4518 #endif
4519                 check_deleted_files();
4520                 break ;
4521
4522             case Q_START_SCAN:
4523                 check_done = false;
4524             case SYS_TIMEOUT:
4525                 if (check_done || !tc_stat.ready)
4526                     break ;
4527
4528 #ifdef HAVE_TC_RAMCACHE
4529                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4530                 {
4531                     load_ramcache();
4532                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4533                         tagcache_build("/");
4534                 }
4535                 else
4536 #endif
4537                 if (global_settings.tagcache_autoupdate)
4538                 {
4539                     tagcache_build("/");
4540
4541                     /* This will be very slow unless dircache is enabled
4542                        or target is flash based, but do it anyway for
4543                        consistency. */
4544                     check_deleted_files();
4545                 }
4546
4547                 logf("tagcache check done");
4548
4549                 check_done = true;
4550                 break ;
4551
4552             case Q_STOP_SCAN:
4553                 break ;
4554
4555             case SYS_POWEROFF:
4556                 break ;
4557
4558 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4559             case SYS_USB_CONNECTED:
4560                 logf("USB: TagCache");
4561                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4562                 usb_wait_for_disconnect(&tagcache_queue);
4563                 break ;
4564 #endif
4565         }
4566     }
4567 }
4568
4569 bool tagcache_prepare_shutdown(void)
4570 {
4571     if (tagcache_get_commit_step() > 0)
4572         return false;
4573
4574     tagcache_stop_scan();
4575     while (read_lock || write_lock)
4576         sleep(1);
4577
4578     return true;
4579 }
4580
4581 void tagcache_shutdown(void)
4582 {
4583     /* Flush the command queue. */
4584     run_command_queue(true);
4585
4586 #ifdef HAVE_EEPROM_SETTINGS
4587     if (tc_stat.ramcache)
4588         tagcache_dumpsave();
4589 #endif
4590 }
4591
4592 static int get_progress(void)
4593 {
4594     int total_count = -1;
4595
4596 #ifdef HAVE_DIRCACHE
4597     if (dircache_is_enabled())
4598     {
4599         total_count = dircache_get_entry_count();
4600     }
4601     else
4602 #endif
4603 #ifdef HAVE_TC_RAMCACHE
4604     {
4605         if (hdr && tc_stat.ramcache)
4606             total_count = hdr->h.tch.entry_count;
4607     }
4608 #endif
4609
4610     if (total_count < 0)
4611         return -1;
4612
4613     return processed_dir_count * 100 / total_count;
4614 }
4615
4616 struct tagcache_stat* tagcache_get_stat(void)
4617 {
4618     tc_stat.progress = get_progress();
4619     tc_stat.processed_entries = processed_dir_count;
4620
4621     return &tc_stat;
4622 }
4623
4624 void tagcache_start_scan(void)
4625 {
4626     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4627 }
4628
4629 bool tagcache_update(void)
4630 {
4631     if (!tc_stat.ready)
4632         return false;
4633
4634     queue_post(&tagcache_queue, Q_UPDATE, 0);
4635     return false;
4636 }
4637
4638 bool tagcache_rebuild()
4639 {
4640     queue_post(&tagcache_queue, Q_REBUILD, 0);
4641     return false;
4642 }
4643
4644 void tagcache_stop_scan(void)
4645 {
4646     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4647 }
4648
4649 #ifdef HAVE_TC_RAMCACHE
4650 bool tagcache_is_ramcache(void)
4651 {
4652     return tc_stat.ramcache;
4653 }
4654 #endif
4655
4656 #endif /* !__PCTOOL__ */
4657
4658
4659 void tagcache_init(void)
4660 {
4661     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4662     memset(&current_tcmh, 0, sizeof(struct master_header));
4663     filenametag_fd = -1;
4664     write_lock = read_lock = 0;
4665
4666 #ifndef __PCTOOL__
4667     mutex_init(&command_queue_mutex);
4668     queue_init(&tagcache_queue, true);
4669     create_thread(tagcache_thread, tagcache_stack,
4670                   sizeof(tagcache_stack), 0, tagcache_thread_name
4671                   IF_PRIO(, PRIORITY_BACKGROUND)
4672                   IF_COP(, CPU));
4673 #else
4674     tc_stat.initialized = true;
4675     allocate_tempbuf();
4676     commit();
4677     free_tempbuf();
4678     tc_stat.ready = check_all_headers();
4679 #endif
4680 }
4681
4682 #ifdef __PCTOOL__
4683 void tagcache_reverse_scan(void)
4684 {
4685     logf("Checking for deleted files");
4686     check_deleted_files();
4687 }
4688 #endif
4689
4690 bool tagcache_is_initialized(void)
4691 {
4692     return tc_stat.initialized;
4693 }
4694 bool tagcache_is_usable(void)
4695 {
4696     return tc_stat.initialized && tc_stat.ready;
4697 }
4698 int tagcache_get_commit_step(void)
4699 {
4700     return tc_stat.commit_step;
4701 }
4702 int tagcache_get_max_commit_step(void)
4703 {
4704     return (int)(SORTED_TAGS_COUNT)+1;
4705 }
4706