apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "filefuncs.h"
  77 #include "settings.h"
  78 #include "dir.h"
  79 #include "structec.h"
  80
  81 #ifndef __PCTOOL__
  82 #include "lang.h"
  83 #include "eeprom_settings.h"
  84 #endif
  85
  86 #ifdef __PCTOOL__
  87 #define yield() do { } while(0)
  88 #define sim_sleep(timeout) do { } while(0)
  89 #define do_timed_yield() do { } while(0)
  90 #endif
  91
  92 #ifndef __PCTOOL__
  93 /* Tag Cache thread. */
  94 static struct event_queue tagcache_queue;
  95 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  96 static const char tagcache_thread_name[] = "tagcache";
  97 #endif
  98
  99 /* Previous path when scanning directory tree recursively. */
 100 static char curpath[TAG_MAXLEN+32];
 101
 102 /* Used when removing duplicates. */
 103 static char *tempbuf;     /* Allocated when needed. */
 104 static long tempbufidx;   /* Current location in buffer. */
 105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 106 static long tempbuf_left; /* Buffer space left. */
 107 static long tempbuf_pos;
 108
 109 #define SORTED_TAGS_COUNT 8
 110 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 111 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 112 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 113     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 114 /* Tags we want to get sorted (loaded to the tempbuf). */
 115 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 116     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 117     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 118
 119 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 120 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping))
 123
 124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 125 static const char *tags_str[] = { "artist", "album", "genre", "title",
 126     "filename", "composer", "comment", "albumartist", "grouping", "year",
 127     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 128     "playtime", "lastplayed", "commitid", "mtime" };
 129
 130 /* Status information of the tagcache. */
 131 static struct tagcache_stat tc_stat;
 132
 133 /* Queue commands. */
 134 enum tagcache_queue {
 135     Q_STOP_SCAN = 0,
 136     Q_START_SCAN,
 137     Q_IMPORT_CHANGELOG,
 138     Q_UPDATE,
 139     Q_REBUILD,
 140
 141     /* Internal tagcache command queue. */
 142     CMD_UPDATE_MASTER_HEADER,
 143     CMD_UPDATE_NUMERIC,
 144 };
 145
 146 struct tagcache_command_entry {
 147     int32_t command;
 148     int32_t idx_id;
 149     int32_t tag;
 150     int32_t data;
 151 };
 152
 153 #ifndef __PCTOOL__
 154 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 155 static volatile int command_queue_widx = 0;
 156 static volatile int command_queue_ridx = 0;
 157 static struct mutex command_queue_mutex;
 158 #endif
 159
 160 /* Tag database structures. */
 161
 162 /* Variable-length tag entry in tag files. */
 163 struct tagfile_entry {
 164     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 165     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 166     char tag_data[0];  /* Begin of the tag data */
 167 };
 168
 169 /* Fixed-size tag entry in master db index. */
 170 struct index_entry {
 171     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 172     int32_t flag;                /* Status flags */
 173 };
 174
 175 /* Header is the same in every file. */
 176 struct tagcache_header {
 177     int32_t magic;       /* Header version number */
 178     int32_t datasize;    /* Data size in bytes */
 179     int32_t entry_count; /* Number of entries in this file */
 180 };
 181
 182 struct master_header {
 183     struct tagcache_header tch;
 184     int32_t serial; /* Increasing counting number */
 185     int32_t commitid; /* Number of commits so far */
 186     int32_t dirty;
 187 };
 188
 189 /* For the endianess correction */
 190 static const char *tagfile_entry_ec   = "ll";
 191 /**
 192  Note: This should be (1 + TAG_COUNT) amount of l's.
 193  */
 194 static const char *index_entry_ec     = "lllllllllllllllllllll";
 195
 196 static const char *tagcache_header_ec = "lll";
 197 static const char *master_header_ec   = "llllll";
 198
 199 static struct master_header current_tcmh;
 200
 201 #ifdef HAVE_TC_RAMCACHE
 202 /* Header is created when loading database to ram. */
 203 struct ramcache_header {
 204     struct master_header h;      /* Header from the master index */
 205     struct index_entry *indices; /* Master index file content */
 206     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 207     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 208 };
 209
 210 # ifdef HAVE_EEPROM_SETTINGS
 211 struct statefile_header {
 212     struct ramcache_header *hdr;
 213     struct tagcache_stat tc_stat;
 214 };
 215 # endif
 216
 217 /* Pointer to allocated ramcache_header */
 218 static struct ramcache_header *hdr;
 219 #endif
 220
 221 /**
 222  * Full tag entries stored in a temporary file waiting
 223  * for commit to the cache. */
 224 struct temp_file_entry {
 225     long tag_offset[TAG_COUNT];
 226     short tag_length[TAG_COUNT];
 227     long flag;
 228
 229     long data_length;
 230 };
 231
 232 struct tempbuf_id_list {
 233     long id;
 234     struct tempbuf_id_list *next;
 235 };
 236
 237 struct tempbuf_searchidx {
 238     long idx_id;
 239     char *str;
 240     int seek;
 241     struct tempbuf_id_list idlist;
 242 };
 243
 244 /* Lookup buffer for fixing messed up index while after sorting. */
 245 static long commit_entry_count;
 246 static long lookup_buffer_depth;
 247 static struct tempbuf_searchidx **lookup;
 248
 249 /* Used when building the temporary file. */
 250 static int cachefd = -1, filenametag_fd;
 251 static int total_entry_count = 0;
 252 static int data_size = 0;
 253 static int processed_dir_count;
 254
 255 /* Thread safe locking */
 256 static volatile int write_lock;
 257 static volatile int read_lock;
 258
 259 static bool delete_entry(long idx_id);
 260
 261 const char* tagcache_tag_to_str(int tag)
 262 {
 263     return tags_str[tag];
 264 }
 265
 266 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 267 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 268 {
 269     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 270 }
 271
 272 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 #ifdef HAVE_DIRCACHE
 283 /**
 284  * Returns true if specified flag is still present, i.e., dircache
 285  * has not been reloaded.
 286  */
 287 static bool is_dircache_intact(void)
 288 {
 289     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 290 }
 291 #endif
 292
 293 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 294 {
 295     int fd;
 296     char buf[MAX_PATH], path[MAX_PATH];
 297     const char * file;
 298     int rc;
 299
 300     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 301         return -1;
 302
 303     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 304     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
 305
 306     fd = open(file, write ? O_RDWR : O_RDONLY);
 307     if (fd < 0)
 308     {
 309         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 310         tc_stat.ready = false;
 311         return fd;
 312     }
 313
 314     /* Check the header. */
 315     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 316     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 317     {
 318         logf("header error");
 319         tc_stat.ready = false;
 320         close(fd);
 321         return -2;
 322     }
 323
 324     return fd;
 325 }
 326
 327 static int open_master_fd(struct master_header *hdr, bool write)
 328 {
 329     int fd;
 330     int rc;
 331     char path[MAX_PATH];
 332
 333     fd = open(get_user_file_path(TAGCACHE_FILE_MASTER,
 334                                  IS_FILE|NEED_WRITE,
 335                                  path, sizeof(path)),
 336                     write ? O_RDWR : O_RDONLY);
 337     if (fd < 0)
 338     {
 339         logf("master file open failed for R/W");
 340         tc_stat.ready = false;
 341         return fd;
 342     }
 343
 344     tc_stat.econ = false;
 345
 346     /* Check the header. */
 347     rc = read(fd, hdr, sizeof(struct master_header));
 348     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 349     {
 350         /* Success. */
 351         return fd;
 352     }
 353
 354     /* Trying to read again, this time with endianess correction enabled. */
 355     lseek(fd, 0, SEEK_SET);
 356
 357     rc = ecread(fd, hdr, 1, master_header_ec, true);
 358     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 359     {
 360         logf("header error");
 361         tc_stat.ready = false;
 362         close(fd);
 363         return -2;
 364     }
 365
 366     tc_stat.econ = true;
 367
 368     return fd;
 369 }
 370
 371 #ifndef __PCTOOL__
 372 static bool do_timed_yield(void)
 373 {
 374     /* Sorting can lock up for quite a while, so yield occasionally */
 375     static long wakeup_tick = 0;
 376     if (TIME_AFTER(current_tick, wakeup_tick))
 377     {
 378         wakeup_tick = current_tick + (HZ/4);
 379         yield();
 380         return true;
 381     }
 382     return false;
 383 }
 384 #endif
 385
 386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 387 static long find_entry_ram(const char *filename,
 388                            const struct dircache_entry *dc)
 389 {
 390     static long last_pos = 0;
 391     int i;
 392
 393     /* Check if we tagcache is loaded into ram. */
 394     if (!tc_stat.ramcache)
 395         return -1;
 396
 397     if (dc == NULL)
 398         dc = dircache_get_entry_ptr(filename);
 399
 400     if (dc == NULL)
 401     {
 402         logf("tagcache: file not found.");
 403         return -1;
 404     }
 405
 406     try_again:
 407
 408     if (last_pos > 0)
 409         i = last_pos;
 410     else
 411         i = 0;
 412
 413     for (; i < hdr->h.tch.entry_count; i++)
 414     {
 415         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 416         {
 417             last_pos = MAX(0, i - 3);
 418             return i;
 419         }
 420
 421         do_timed_yield();
 422     }
 423
 424     if (last_pos > 0)
 425     {
 426         last_pos = 0;
 427         goto try_again;
 428     }
 429
 430     return -1;
 431 }
 432 #endif
 433
 434 static long find_entry_disk(const char *filename, bool localfd)
 435 {
 436     struct tagcache_header tch;
 437     static long last_pos = -1;
 438     long pos_history[POS_HISTORY_COUNT];
 439     long pos_history_idx = 0;
 440     bool found = false;
 441     struct tagfile_entry tfe;
 442     int fd;
 443     char buf[TAG_MAXLEN+32];
 444     int i;
 445     int pos = -1;
 446
 447     if (!tc_stat.ready)
 448         return -2;
 449
 450     fd = filenametag_fd;
 451     if (fd < 0 || localfd)
 452     {
 453         last_pos = -1;
 454         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 455             return -1;
 456     }
 457
 458     check_again:
 459
 460     if (last_pos > 0)
 461         lseek(fd, last_pos, SEEK_SET);
 462     else
 463         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 464
 465     while (true)
 466     {
 467         pos = lseek(fd, 0, SEEK_CUR);
 468         for (i = pos_history_idx-1; i >= 0; i--)
 469             pos_history[i+1] = pos_history[i];
 470         pos_history[0] = pos;
 471
 472         if (ecread_tagfile_entry(fd, &tfe)
 473             != sizeof(struct tagfile_entry))
 474         {
 475             break ;
 476         }
 477
 478         if (tfe.tag_length >= (long)sizeof(buf))
 479         {
 480             logf("too long tag #1");
 481             close(fd);
 482             if (!localfd)
 483                 filenametag_fd = -1;
 484             last_pos = -1;
 485             return -2;
 486         }
 487
 488         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 489         {
 490             logf("read error #2");
 491             close(fd);
 492             if (!localfd)
 493                 filenametag_fd = -1;
 494             last_pos = -1;
 495             return -3;
 496         }
 497
 498         if (!strcasecmp(filename, buf))
 499         {
 500             last_pos = pos_history[pos_history_idx];
 501             found = true;
 502             break ;
 503         }
 504
 505         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 506             pos_history_idx++;
 507     }
 508
 509     /* Not found? */
 510     if (!found)
 511     {
 512         if (last_pos > 0)
 513         {
 514             last_pos = -1;
 515             logf("seek again");
 516             goto check_again;
 517         }
 518
 519         if (fd != filenametag_fd || localfd)
 520             close(fd);
 521         return -4;
 522     }
 523
 524     if (fd != filenametag_fd || localfd)
 525         close(fd);
 526
 527     return tfe.idx_id;
 528 }
 529
 530 static int find_index(const char *filename)
 531 {
 532     long idx_id = -1;
 533
 534 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 535     if (tc_stat.ramcache && is_dircache_intact())
 536         idx_id = find_entry_ram(filename, NULL);
 537 #endif
 538
 539     if (idx_id < 0)
 540         idx_id = find_entry_disk(filename, true);
 541
 542     return idx_id;
 543 }
 544
 545 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 546 {
 547     int idx_id;
 548
 549     if (!tc_stat.ready)
 550         return false;
 551
 552     idx_id = find_index(filename);
 553     if (idx_id < 0)
 554         return false;
 555
 556     if (!tagcache_search(tcs, tag_filename))
 557         return false;
 558
 559     tcs->entry_count = 0;
 560     tcs->idx_id = idx_id;
 561
 562     return true;
 563 }
 564
 565 static bool get_index(int masterfd, int idxid,
 566                       struct index_entry *idx, bool use_ram)
 567 {
 568     bool localfd = false;
 569
 570     if (idxid < 0)
 571     {
 572         logf("Incorrect idxid: %d", idxid);
 573         return false;
 574     }
 575
 576 #ifdef HAVE_TC_RAMCACHE
 577     if (tc_stat.ramcache && use_ram)
 578     {
 579         if (hdr->indices[idxid].flag & FLAG_DELETED)
 580             return false;
 581
 582 # ifdef HAVE_DIRCACHE
 583         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 584             || is_dircache_intact())
 585 #endif
 586         {
 587             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 588             return true;
 589         }
 590     }
 591 #else
 592     (void)use_ram;
 593 #endif
 594
 595     if (masterfd < 0)
 596     {
 597         struct master_header tcmh;
 598
 599         localfd = true;
 600         masterfd = open_master_fd(&tcmh, false);
 601         if (masterfd < 0)
 602             return false;
 603     }
 604
 605     lseek(masterfd, idxid * sizeof(struct index_entry)
 606           + sizeof(struct master_header), SEEK_SET);
 607     if (ecread_index_entry(masterfd, idx)
 608         != sizeof(struct index_entry))
 609     {
 610         logf("read error #3");
 611         if (localfd)
 612             close(masterfd);
 613
 614         return false;
 615     }
 616
 617     if (localfd)
 618         close(masterfd);
 619
 620     if (idx->flag & FLAG_DELETED)
 621         return false;
 622
 623     return true;
 624 }
 625
 626 #ifndef __PCTOOL__
 627
 628 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 629 {
 630     /* We need to exclude all memory only flags & tags when writing to disk. */
 631     if (idx->flag & FLAG_DIRCACHE)
 632     {
 633         logf("memory only flags!");
 634         return false;
 635     }
 636
 637 #ifdef HAVE_TC_RAMCACHE
 638     /* Only update numeric data. Writing the whole index to RAM by memcpy
 639      * destroys dircache pointers!
 640      */
 641     if (tc_stat.ramcache)
 642     {
 643         int tag;
 644         struct index_entry *idx_ram = &hdr->indices[idxid];
 645
 646         for (tag = 0; tag < TAG_COUNT; tag++)
 647         {
 648             if (TAGCACHE_IS_NUMERIC(tag))
 649             {
 650                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 651             }
 652         }
 653
 654         /* Don't touch the dircache flag or attributes. */
 655         idx_ram->flag = (idx->flag & 0x0000ffff)
 656             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 657     }
 658 #endif
 659
 660     lseek(masterfd, idxid * sizeof(struct index_entry)
 661           + sizeof(struct master_header), SEEK_SET);
 662     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 663     {
 664         logf("write error #3");
 665         logf("idxid: %d", idxid);
 666         return false;
 667     }
 668
 669     return true;
 670 }
 671
 672 #endif /* !__PCTOOL__ */
 673
 674 static bool open_files(struct tagcache_search *tcs, int tag)
 675 {
 676     if (tcs->idxfd[tag] < 0)
 677     {
 678         char fn[MAX_PATH], path[MAX_PATH];
 679         const char *file;
 680
 681         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 682         file = get_user_file_path(fn, IS_FILE | NEED_WRITE, path, sizeof(path));
 683         tcs->idxfd[tag] = open(file, O_RDONLY);
 684     }
 685
 686     if (tcs->idxfd[tag] < 0)
 687     {
 688         logf("File not open!");
 689         return false;
 690     }
 691
 692     return true;
 693 }
 694
 695 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 696                      int tag, char *buf, long size)
 697 {
 698     struct tagfile_entry tfe;
 699     long seek;
 700
 701     *buf = '\0';
 702
 703     if (TAGCACHE_IS_NUMERIC(tag))
 704         return false;
 705
 706     seek = idx->tag_seek[tag];
 707     if (seek < 0)
 708     {
 709         logf("Retrieve failed");
 710         return false;
 711     }
 712
 713 #ifdef HAVE_TC_RAMCACHE
 714     if (tcs->ramsearch)
 715     {
 716         struct tagfile_entry *ep;
 717
 718 # ifdef HAVE_DIRCACHE
 719         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 720             && is_dircache_intact())
 721         {
 722             dircache_copy_path((struct dircache_entry *)seek,
 723                                buf, size);
 724             return true;
 725         }
 726         else
 727 # endif
 728         if (tag != tag_filename)
 729         {
 730             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 731             strlcpy(buf, ep->tag_data, size);
 732
 733             return true;
 734         }
 735     }
 736 #endif
 737
 738     if (!open_files(tcs, tag))
 739         return false;
 740
 741     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 742     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 743         != sizeof(struct tagfile_entry))
 744     {
 745         logf("read error #5");
 746         return false;
 747     }
 748
 749     if (tfe.tag_length >= size)
 750     {
 751         logf("too small buffer");
 752         return false;
 753     }
 754
 755     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 756         tfe.tag_length)
 757     {
 758         logf("read error #6");
 759         return false;
 760     }
 761
 762     buf[tfe.tag_length] = '\0';
 763
 764     return true;
 765 }
 766
 767 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 768
 769 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 770 {
 771 #ifndef __PCTOOL__
 772     if (! COMMAND_QUEUE_IS_EMPTY)
 773     {
 774         /* Attempt to find tag data through store-to-load forwarding in
 775            command queue */
 776         long result = -1;
 777
 778         mutex_lock(&command_queue_mutex);
 779
 780         int ridx = command_queue_widx;
 781
 782         while (ridx != command_queue_ridx)
 783         {
 784             if (--ridx < 0)
 785                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 786
 787             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 788                 && command_queue[ridx].idx_id == idx_id
 789                 && command_queue[ridx].tag == tag)
 790             {
 791                 result = command_queue[ridx].data;
 792                 break;
 793             }
 794         }
 795
 796         mutex_unlock(&command_queue_mutex);
 797
 798         if (result >= 0)
 799         {
 800             logf("read_numeric_tag: "
 801                  "Recovered tag %d value %lX from write queue",
 802                  tag, result);
 803             return result;
 804         }
 805     }
 806 #endif
 807
 808     return idx->tag_seek[tag];
 809 }
 810
 811
 812 static long check_virtual_tags(int tag, int idx_id,
 813                                const struct index_entry *idx)
 814 {
 815     long data = 0;
 816
 817     switch (tag)
 818     {
 819         case tag_virt_length_sec:
 820             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 821             break;
 822
 823         case tag_virt_length_min:
 824             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 825             break;
 826
 827         case tag_virt_playtime_sec:
 828             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 829             break;
 830
 831         case tag_virt_playtime_min:
 832             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 833             break;
 834
 835         case tag_virt_autoscore:
 836             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 837                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 838             {
 839                 data = 0;
 840             }
 841             else
 842             {
 843                 /* A straight calculus gives:
 844                      autoscore = 100 * playtime / length / playcout (1)
 845                    Now, consider the euclidian division of playtime by length:
 846                      playtime = alpha * length + beta
 847                    With:
 848                      0 <= beta < length
 849                    Now, (1) becomes:
 850                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 851                    Both terms should be small enough to avoid any overflow
 852                 */
 853                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 854                               / read_numeric_tag(tag_length, idx_id, idx))
 855                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 856                                  % read_numeric_tag(tag_length, idx_id, idx)))
 857                          / read_numeric_tag(tag_length, idx_id, idx);
 858                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 859             }
 860             break;
 861
 862         /* How many commits before the file has been added to the DB. */
 863         case tag_virt_entryage:
 864             data = current_tcmh.commitid
 865                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 866             break;
 867
 868         default:
 869             data = read_numeric_tag(tag, idx_id, idx);
 870     }
 871
 872     return data;
 873 }
 874
 875 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 876 {
 877     struct index_entry idx;
 878
 879     if (!tc_stat.ready)
 880         return false;
 881
 882     if (!TAGCACHE_IS_NUMERIC(tag))
 883         return -1;
 884
 885     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 886         return -2;
 887
 888     return check_virtual_tags(tag, tcs->idx_id, &idx);
 889 }
 890
 891 inline static bool str_ends_with(const char *str1, const char *str2)
 892 {
 893     int str_len = strlen(str1);
 894     int clause_len = strlen(str2);
 895
 896     if (clause_len > str_len)
 897         return false;
 898
 899     return !strcasecmp(&str1[str_len - clause_len], str2);
 900 }
 901
 902 inline static bool str_oneof(const char *str, const char *list)
 903 {
 904     const char *sep;
 905     int l, len = strlen(str);
 906
 907     while (*list)
 908     {
 909         sep = strchr(list, '|');
 910         l = sep ? (long)sep - (long)list : (int)strlen(list);
 911         if ((l==len) && !strncasecmp(str, list, len))
 912             return true;
 913         list += sep ? l + 1 : l;
 914     }
 915
 916     return false;
 917 }
 918
 919 static bool check_against_clause(long numeric, const char *str,
 920                                  const struct tagcache_search_clause *clause)
 921 {
 922     if (clause->numeric)
 923     {
 924         switch (clause->type)
 925         {
 926             case clause_is:
 927                 return numeric == clause->numeric_data;
 928             case clause_is_not:
 929                 return numeric != clause->numeric_data;
 930             case clause_gt:
 931                 return numeric > clause->numeric_data;
 932             case clause_gteq:
 933                 return numeric >= clause->numeric_data;
 934             case clause_lt:
 935                 return numeric < clause->numeric_data;
 936             case clause_lteq:
 937                 return numeric <= clause->numeric_data;
 938             default:
 939                 logf("Incorrect numeric tag: %d", clause->type);
 940         }
 941     }
 942     else
 943     {
 944         switch (clause->type)
 945         {
 946             case clause_is:
 947                 return !strcasecmp(clause->str, str);
 948             case clause_is_not:
 949                 return strcasecmp(clause->str, str);
 950             case clause_gt:
 951                 return 0>strcasecmp(clause->str, str);
 952             case clause_gteq:
 953                 return 0>=strcasecmp(clause->str, str);
 954             case clause_lt:
 955                 return 0<strcasecmp(clause->str, str);
 956             case clause_lteq:
 957                 return 0<=strcasecmp(clause->str, str);
 958             case clause_contains:
 959                 return (strcasestr(str, clause->str) != NULL);
 960             case clause_not_contains:
 961                 return (strcasestr(str, clause->str) == NULL);
 962             case clause_begins_with:
 963                 return (strcasestr(str, clause->str) == str);
 964             case clause_not_begins_with:
 965                 return (strcasestr(str, clause->str) != str);
 966             case clause_ends_with:
 967                 return str_ends_with(str, clause->str);
 968             case clause_not_ends_with:
 969                 return !str_ends_with(str, clause->str);
 970             case clause_oneof:
 971                 return str_oneof(str, clause->str);
 972
 973             default:
 974                 logf("Incorrect tag: %d", clause->type);
 975         }
 976     }
 977
 978     return false;
 979 }
 980
 981 static bool check_clauses(struct tagcache_search *tcs,
 982                           struct index_entry *idx,
 983                           struct tagcache_search_clause **clause, int count)
 984 {
 985     int i;
 986
 987 #ifdef HAVE_TC_RAMCACHE
 988     if (tcs->ramsearch)
 989     {
 990         /* Go through all conditional clauses. */
 991         for (i = 0; i < count; i++)
 992         {
 993             struct tagfile_entry *tfe;
 994             int seek;
 995             char buf[256];
 996             char *str = NULL;
 997
 998             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
 999
1000             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1001             {
1002                 if (clause[i]->tag == tag_filename)
1003                 {
1004                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1005                     str = buf;
1006                 }
1007                 else
1008                 {
1009                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1010                     str = tfe->tag_data;
1011                 }
1012             }
1013
1014             if (!check_against_clause(seek, str, clause[i]))
1015                 return false;
1016         }
1017     }
1018     else
1019 #endif
1020     {
1021         /* Check for conditions. */
1022         for (i = 0; i < count; i++)
1023         {
1024             struct tagfile_entry tfe;
1025             int seek;
1026             char str[256];
1027
1028             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1029
1030             memset(str, 0, sizeof str);
1031             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1032             {
1033                 int fd = tcs->idxfd[clause[i]->tag];
1034                 lseek(fd, seek, SEEK_SET);
1035                 ecread_tagfile_entry(fd, &tfe);
1036                 if (tfe.tag_length >= (int)sizeof(str))
1037                 {
1038                     logf("Too long tag read!");
1039                     break ;
1040                 }
1041
1042                 read(fd, str, tfe.tag_length);
1043
1044                 /* Check if entry has been deleted. */
1045                 if (str[0] == '\0')
1046                     break;
1047             }
1048
1049             if (!check_against_clause(seek, str, clause[i]))
1050                 return false;
1051         }
1052     }
1053
1054     return true;
1055 }
1056
1057 bool tagcache_check_clauses(struct tagcache_search *tcs,
1058                             struct tagcache_search_clause **clause, int count)
1059 {
1060     struct index_entry idx;
1061
1062     if (count == 0)
1063         return true;
1064
1065     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1066         return false;
1067
1068     return check_clauses(tcs, &idx, clause, count);
1069 }
1070
1071 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1072 {
1073     int i;
1074
1075     /* If uniq buffer is not defined we must return true for search to work. */
1076     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1077         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1078     {
1079         return true;
1080     }
1081
1082     for (i = 0; i < tcs->unique_list_count; i++)
1083     {
1084         /* Return false if entry is found. */
1085         if (tcs->unique_list[i] == id)
1086             return false;
1087     }
1088
1089     if (tcs->unique_list_count < tcs->unique_list_capacity)
1090     {
1091         tcs->unique_list[i] = id;
1092         tcs->unique_list_count++;
1093     }
1094
1095     return true;
1096 }
1097
1098 static bool build_lookup_list(struct tagcache_search *tcs)
1099 {
1100     struct index_entry entry;
1101     int i, j;
1102
1103     tcs->seek_list_count = 0;
1104
1105 #ifdef HAVE_TC_RAMCACHE
1106     if (tcs->ramsearch
1107 # ifdef HAVE_DIRCACHE
1108         && (tcs->type != tag_filename || is_dircache_intact())
1109 # endif
1110         )
1111     {
1112         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1113         {
1114             struct tagcache_seeklist_entry *seeklist;
1115             struct index_entry *idx = &hdr->indices[i];
1116             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1117                 break ;
1118
1119             /* Skip deleted files. */
1120             if (idx->flag & FLAG_DELETED)
1121                 continue;
1122
1123             /* Go through all filters.. */
1124             for (j = 0; j < tcs->filter_count; j++)
1125             {
1126                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1127                 {
1128                     break ;
1129                 }
1130             }
1131
1132             if (j < tcs->filter_count)
1133                 continue ;
1134
1135             /* Check for conditions. */
1136             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1137                 continue;
1138
1139             /* Add to the seek list if not already in uniq buffer. */
1140             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1141                 continue;
1142
1143             /* Lets add it. */
1144             seeklist = &tcs->seeklist[tcs->seek_list_count];
1145             seeklist->seek = idx->tag_seek[tcs->type];
1146             seeklist->flag = idx->flag;
1147             seeklist->idx_id = i;
1148             tcs->seek_list_count++;
1149         }
1150
1151         tcs->seek_pos = i;
1152
1153         return tcs->seek_list_count > 0;
1154     }
1155 #endif
1156
1157     if (tcs->masterfd < 0)
1158     {
1159         struct master_header tcmh;
1160         tcs->masterfd = open_master_fd(&tcmh, false);
1161     }
1162
1163     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1164             sizeof(struct master_header), SEEK_SET);
1165
1166     while (ecread_index_entry(tcs->masterfd, &entry)
1167            == sizeof(struct index_entry))
1168     {
1169         struct tagcache_seeklist_entry *seeklist;
1170
1171         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1172             break ;
1173
1174         i = tcs->seek_pos;
1175         tcs->seek_pos++;
1176
1177         /* Check if entry has been deleted. */
1178         if (entry.flag & FLAG_DELETED)
1179             continue;
1180
1181         /* Go through all filters.. */
1182         for (j = 0; j < tcs->filter_count; j++)
1183         {
1184             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1185                 break ;
1186         }
1187
1188         if (j < tcs->filter_count)
1189             continue ;
1190
1191         /* Check for conditions. */
1192         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1193             continue;
1194
1195         /* Add to the seek list if not already in uniq buffer. */
1196         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1197             continue;
1198
1199         /* Lets add it. */
1200         seeklist = &tcs->seeklist[tcs->seek_list_count];
1201         seeklist->seek = entry.tag_seek[tcs->type];
1202         seeklist->flag = entry.flag;
1203         seeklist->idx_id = i;
1204         tcs->seek_list_count++;
1205
1206         yield();
1207     }
1208
1209     return tcs->seek_list_count > 0;
1210 }
1211
1212
1213 static void remove_files(void)
1214 {
1215     int i;
1216     char buf[MAX_PATH];
1217
1218     tc_stat.ready = false;
1219     tc_stat.ramcache = false;
1220     tc_stat.econ = false;
1221     remove(get_user_file_path(TAGCACHE_FILE_MASTER, NEED_WRITE|IS_FILE,
1222                               buf, sizeof(buf)));
1223     for (i = 0; i < TAG_COUNT; i++)
1224     {
1225         char buf2[MAX_PATH];
1226         if (TAGCACHE_IS_NUMERIC(i))
1227             continue;
1228
1229         /* database_%d.tcd -> database_0.tcd */
1230         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1231         remove(get_user_file_path(buf, NEED_WRITE | IS_FILE, buf2, sizeof(buf2)));
1232     }
1233 }
1234
1235
1236 static bool check_all_headers(void)
1237 {
1238     struct master_header myhdr;
1239     struct tagcache_header tch;
1240     int tag;
1241     int fd;
1242
1243     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1244         return false;
1245
1246     close(fd);
1247     if (myhdr.dirty)
1248     {
1249         logf("tagcache is dirty!");
1250         return false;
1251     }
1252
1253     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1254
1255     for (tag = 0; tag < TAG_COUNT; tag++)
1256     {
1257         if (TAGCACHE_IS_NUMERIC(tag))
1258             continue;
1259
1260         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1261             return false;
1262
1263         close(fd);
1264     }
1265
1266     return true;
1267 }
1268
1269 bool tagcache_is_busy(void)
1270 {
1271     return read_lock || write_lock;
1272 }
1273
1274 bool tagcache_search(struct tagcache_search *tcs, int tag)
1275 {
1276     struct tagcache_header tag_hdr;
1277     struct master_header   master_hdr;
1278     int i;
1279
1280     while (read_lock)
1281         sleep(1);
1282
1283     memset(tcs, 0, sizeof(struct tagcache_search));
1284     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1285         return false;
1286
1287     tcs->position = sizeof(struct tagcache_header);
1288     tcs->type = tag;
1289     tcs->seek_pos = 0;
1290     tcs->list_position = 0;
1291     tcs->seek_list_count = 0;
1292     tcs->filter_count = 0;
1293     tcs->masterfd = -1;
1294
1295     for (i = 0; i < TAG_COUNT; i++)
1296         tcs->idxfd[i] = -1;
1297
1298 #ifndef HAVE_TC_RAMCACHE
1299     tcs->ramsearch = false;
1300 #else
1301     tcs->ramsearch = tc_stat.ramcache;
1302     if (tcs->ramsearch)
1303     {
1304         tcs->entry_count = hdr->entry_count[tcs->type];
1305     }
1306     else
1307 #endif
1308     {
1309         /* Always open as R/W so we can pass tcs to functions that modify data also
1310          * without failing. */
1311         tcs->masterfd = open_master_fd(&master_hdr, true);
1312         if (tcs->masterfd < 0)
1313             return false;
1314
1315         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1316         {
1317             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1318             if (tcs->idxfd[tcs->type] < 0)
1319                 return false;
1320
1321             tcs->entry_count = tag_hdr.entry_count;
1322         }
1323         else
1324         {
1325             tcs->entry_count = master_hdr.tch.entry_count;
1326         }
1327     }
1328
1329     tcs->valid = true;
1330     tcs->initialized = true;
1331     write_lock++;
1332
1333     return true;
1334 }
1335
1336 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1337                                  void *buffer, long length)
1338 {
1339     tcs->unique_list = (unsigned long *)buffer;
1340     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1341     tcs->unique_list_count = 0;
1342 }
1343
1344 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1345                                 int tag, int seek)
1346 {
1347     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1348         return false;
1349
1350     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1351         return false;
1352
1353     tcs->filter_tag[tcs->filter_count] = tag;
1354     tcs->filter_seek[tcs->filter_count] = seek;
1355     tcs->filter_count++;
1356
1357     return true;
1358 }
1359
1360 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1361                                 struct tagcache_search_clause *clause)
1362 {
1363     int i;
1364
1365     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1366     {
1367         logf("Too many clauses");
1368         return false;
1369     }
1370
1371     /* Check if there is already a similar filter in present (filters are
1372      * much faster than clauses).
1373      */
1374     for (i = 0; i < tcs->filter_count; i++)
1375     {
1376         if (tcs->filter_tag[i] == clause->tag)
1377             return true;
1378     }
1379
1380     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1381     {
1382         char buf[MAX_PATH], path[MAX_PATH];
1383         const char *file;
1384         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1385         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
1386         tcs->idxfd[clause->tag] = open(file, O_RDONLY);
1387     }
1388
1389     tcs->clause[tcs->clause_count] = clause;
1390     tcs->clause_count++;
1391
1392     return true;
1393 }
1394
1395 static bool get_next(struct tagcache_search *tcs)
1396 {
1397     static char buf[TAG_MAXLEN+32];
1398     struct tagfile_entry entry;
1399     long flag = 0;
1400
1401     if (!tcs->valid || !tc_stat.ready)
1402         return false;
1403
1404     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1405 #ifdef HAVE_TC_RAMCACHE
1406         && !tcs->ramsearch
1407 #endif
1408         )
1409         return false;
1410
1411     /* Relative fetch. */
1412     if (tcs->filter_count > 0 || tcs->clause_count > 0
1413         || TAGCACHE_IS_NUMERIC(tcs->type)
1414 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1415         /* We need to retrieve flag status for dircache. */
1416         || (tcs->ramsearch && tcs->type == tag_filename)
1417 #endif
1418         )
1419     {
1420         struct tagcache_seeklist_entry *seeklist;
1421
1422         /* Check for end of list. */
1423         if (tcs->list_position == tcs->seek_list_count)
1424         {
1425             tcs->list_position = 0;
1426
1427             /* Try to fetch more. */
1428             if (!build_lookup_list(tcs))
1429             {
1430                 tcs->valid = false;
1431                 return false;
1432             }
1433         }
1434
1435         seeklist = &tcs->seeklist[tcs->list_position];
1436         flag = seeklist->flag;
1437         tcs->position = seeklist->seek;
1438         tcs->idx_id = seeklist->idx_id;
1439         tcs->list_position++;
1440     }
1441     else
1442     {
1443         if (tcs->entry_count == 0)
1444         {
1445             tcs->valid = false;
1446             return false;
1447         }
1448
1449         tcs->entry_count--;
1450     }
1451
1452     tcs->result_seek = tcs->position;
1453
1454     if (TAGCACHE_IS_NUMERIC(tcs->type))
1455     {
1456         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1457         tcs->result = buf;
1458         tcs->result_len = strlen(buf) + 1;
1459         return true;
1460     }
1461
1462     /* Direct fetch. */
1463 #ifdef HAVE_TC_RAMCACHE
1464     if (tcs->ramsearch)
1465     {
1466
1467 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1468         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1469             && is_dircache_intact())
1470         {
1471             dircache_copy_path((struct dircache_entry *)tcs->position,
1472                                buf, sizeof buf);
1473             tcs->result = buf;
1474             tcs->result_len = strlen(buf) + 1;
1475             tcs->ramresult = false;
1476
1477             return true;
1478         }
1479         else
1480 #endif
1481         if (tcs->type != tag_filename)
1482         {
1483             struct tagfile_entry *ep;
1484
1485             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1486             tcs->result = ep->tag_data;
1487             tcs->result_len = strlen(tcs->result) + 1;
1488             tcs->idx_id = ep->idx_id;
1489             tcs->ramresult = true;
1490
1491             /* Increase position for the next run. This may get overwritten. */
1492             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1493
1494             return true;
1495         }
1496     }
1497 #endif
1498
1499     if (!open_files(tcs, tcs->type))
1500     {
1501         tcs->valid = false;
1502         return false;
1503     }
1504
1505     /* Seek stream to the correct position and continue to direct fetch. */
1506     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1507
1508     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1509     {
1510         logf("read error #5");
1511         tcs->valid = false;
1512         return false;
1513     }
1514
1515     if (entry.tag_length > (long)sizeof(buf))
1516     {
1517         tcs->valid = false;
1518         logf("too long tag #2");
1519         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1520         return false;
1521     }
1522
1523     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1524     {
1525         tcs->valid = false;
1526         logf("read error #4");
1527         return false;
1528     }
1529
1530     /**
1531      Update the position for the next read (this may be overridden
1532      if filters or clauses are being used).
1533      */
1534     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1535     tcs->result = buf;
1536     tcs->result_len = strlen(tcs->result) + 1;
1537     tcs->idx_id = entry.idx_id;
1538     tcs->ramresult = false;
1539
1540     return true;
1541 }
1542
1543 bool tagcache_get_next(struct tagcache_search *tcs)
1544 {
1545     while (get_next(tcs))
1546     {
1547         if (tcs->result_len > 1)
1548             return true;
1549     }
1550
1551     return false;
1552 }
1553
1554 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1555                        int tag, char *buf, long size)
1556 {
1557     struct index_entry idx;
1558
1559     *buf = '\0';
1560     if (!get_index(tcs->masterfd, idxid, &idx, true))
1561         return false;
1562
1563     return retrieve(tcs, &idx, tag, buf, size);
1564 }
1565
1566 static bool update_master_header(void)
1567 {
1568     struct master_header myhdr;
1569     int fd;
1570
1571     if (!tc_stat.ready)
1572         return false;
1573
1574     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1575         return false;
1576
1577     myhdr.serial = current_tcmh.serial;
1578     myhdr.commitid = current_tcmh.commitid;
1579     myhdr.dirty = current_tcmh.dirty;
1580
1581     /* Write it back */
1582     lseek(fd, 0, SEEK_SET);
1583     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1584     close(fd);
1585
1586 #ifdef HAVE_TC_RAMCACHE
1587     if (hdr)
1588     {
1589         hdr->h.serial = current_tcmh.serial;
1590         hdr->h.commitid = current_tcmh.commitid;
1591         hdr->h.dirty = current_tcmh.dirty;
1592     }
1593 #endif
1594
1595     return true;
1596 }
1597
1598 #if 0
1599
1600 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1601 {
1602     struct tagentry *entry;
1603
1604     if (tcs->type != tag_title)
1605         return ;
1606
1607     /* We will need reserve buffer for this. */
1608     if (tcs->ramcache)
1609     {
1610         struct tagfile_entry *ep;
1611
1612         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1613         tcs->seek_list[tcs->seek_list_count];
1614     }
1615
1616     entry = find_entry_ram();
1617
1618 }
1619 #endif
1620
1621 void tagcache_search_finish(struct tagcache_search *tcs)
1622 {
1623     int i;
1624
1625     if (!tcs->initialized)
1626         return;
1627
1628     if (tcs->masterfd >= 0)
1629     {
1630         close(tcs->masterfd);
1631         tcs->masterfd = -1;
1632     }
1633
1634     for (i = 0; i < TAG_COUNT; i++)
1635     {
1636         if (tcs->idxfd[i] >= 0)
1637         {
1638             close(tcs->idxfd[i]);
1639             tcs->idxfd[i] = -1;
1640         }
1641     }
1642
1643     tcs->ramsearch = false;
1644     tcs->valid = false;
1645     tcs->initialized = 0;
1646     if (write_lock > 0)
1647         write_lock--;
1648 }
1649
1650 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1651 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1652 {
1653     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1654 }
1655
1656 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1657 {
1658     return check_virtual_tags(tag, idx_id, entry);
1659 }
1660
1661 static char* get_tag_string(const struct index_entry *entry, int tag)
1662 {
1663     char* s = get_tag(entry, tag)->tag_data;
1664     return strcmp(s, UNTAGGED) ? s : NULL;
1665 }
1666
1667 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1668 {
1669     struct index_entry *entry;
1670     int idx_id;
1671
1672     if (!tc_stat.ready || !tc_stat.ramcache)
1673         return false;
1674
1675     /* Find the corresponding entry in tagcache. */
1676     idx_id = find_entry_ram(filename, NULL);
1677     if (idx_id < 0)
1678         return false;
1679
1680     entry = &hdr->indices[idx_id];
1681
1682     memset(id3, 0, sizeof(struct mp3entry));
1683
1684     id3->title        = get_tag_string(entry, tag_title);
1685     id3->artist       = get_tag_string(entry, tag_artist);
1686     id3->album        = get_tag_string(entry, tag_album);
1687     id3->genre_string = get_tag_string(entry, tag_genre);
1688     id3->composer     = get_tag_string(entry, tag_composer);
1689     id3->comment      = get_tag_string(entry, tag_comment);
1690     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1691     id3->grouping     = get_tag_string(entry, tag_grouping);
1692
1693     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1694     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1695     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1696     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1697     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1698     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1699
1700     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1701     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1702     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1703     if (id3->bitrate == 0)
1704         id3->bitrate = 1;
1705
1706     return true;
1707 }
1708 #endif
1709
1710 static inline void write_item(const char *item)
1711 {
1712     int len = strlen(item) + 1;
1713
1714     data_size += len;
1715     write(cachefd, item, len);
1716 }
1717
1718 static int check_if_empty(char **tag)
1719 {
1720     int length;
1721
1722     if (*tag == NULL || **tag == '\0')
1723     {
1724         *tag = UNTAGGED;
1725         return sizeof(UNTAGGED); /* Tag length */
1726     }
1727
1728     length = strlen(*tag);
1729     if (length > TAG_MAXLEN)
1730     {
1731         logf("over length tag: %s", *tag);
1732         length = TAG_MAXLEN;
1733         (*tag)[length] = '\0';
1734     }
1735
1736     return length + 1;
1737 }
1738
1739 #define ADD_TAG(entry,tag,data) \
1740     /* Adding tag */ \
1741     entry.tag_offset[tag] = offset; \
1742     entry.tag_length[tag] = check_if_empty(data); \
1743     offset += entry.tag_length[tag]
1744 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1745  * idea, as it uses lots of stack and is called from a recursive function
1746  * (check_dir).
1747  */
1748 static void __attribute__ ((noinline)) add_tagcache(char *path,
1749                                                     unsigned long mtime
1750 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1751                                                     ,const struct dircache_entry *dc
1752 #endif
1753                                                    )
1754 {
1755     struct mp3entry id3;
1756     struct temp_file_entry entry;
1757     bool ret;
1758     int fd;
1759     int idx_id = -1;
1760     char tracknumfix[3];
1761     int offset = 0;
1762     int path_length = strlen(path);
1763     bool has_albumartist;
1764     bool has_grouping;
1765
1766 #ifdef SIMULATOR
1767     /* Crude logging for the sim - to aid in debugging */
1768     int logfd = open(ROCKBOX_DIR "/database.log",
1769                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1770     if (logfd >= 0) {
1771         write(logfd, path, strlen(path));
1772         write(logfd, "\n", 1);
1773         close(logfd);
1774     }
1775 #endif
1776
1777     if (cachefd < 0)
1778         return ;
1779
1780     /* Check for overlength file path. */
1781     if (path_length > TAG_MAXLEN)
1782     {
1783         /* Path can't be shortened. */
1784         logf("Too long path: %s", path);
1785         return ;
1786     }
1787
1788     /* Check if the file is supported. */
1789     if (probe_file_format(path) == AFMT_UNKNOWN)
1790         return ;
1791
1792     /* Check if the file is already cached. */
1793 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1794     if (tc_stat.ramcache && is_dircache_intact())
1795     {
1796         idx_id = find_entry_ram(path, dc);
1797     }
1798 #endif
1799
1800     /* Be sure the entry doesn't exist. */
1801     if (filenametag_fd >= 0 && idx_id < 0)
1802         idx_id = find_entry_disk(path, false);
1803
1804     /* Check if file has been modified. */
1805     if (idx_id >= 0)
1806     {
1807         struct index_entry idx;
1808
1809         /* TODO: Mark that the index exists (for fast reverse scan) */
1810         //found_idx[idx_id/8] |= idx_id%8;
1811
1812         if (!get_index(-1, idx_id, &idx, true))
1813         {
1814             logf("failed to retrieve index entry");
1815             return ;
1816         }
1817
1818         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1819         {
1820             /* No changes to file. */
1821             return ;
1822         }
1823
1824         /* Metadata might have been changed. Delete the entry. */
1825         logf("Re-adding: %s", path);
1826         if (!delete_entry(idx_id))
1827         {
1828             logf("delete_entry failed: %d", idx_id);
1829             return ;
1830         }
1831     }
1832
1833     fd = open(path, O_RDONLY);
1834     if (fd < 0)
1835     {
1836         logf("open fail: %s", path);
1837         return ;
1838     }
1839
1840     memset(&id3, 0, sizeof(struct mp3entry));
1841     memset(&entry, 0, sizeof(struct temp_file_entry));
1842     memset(&tracknumfix, 0, sizeof(tracknumfix));
1843     ret = get_metadata(&id3, fd, path);
1844     close(fd);
1845
1846     if (!ret)
1847         return ;
1848
1849     logf("-> %s", path);
1850
1851     /* Generate track number if missing. */
1852     if (id3.tracknum <= 0)
1853     {
1854         const char *p = strrchr(path, '.');
1855
1856         if (p == NULL)
1857             p = &path[strlen(path)-1];
1858
1859         while (*p != '/')
1860         {
1861             if (isdigit(*p) && isdigit(*(p-1)))
1862             {
1863                 tracknumfix[1] = *p--;
1864                 tracknumfix[0] = *p;
1865                 break;
1866             }
1867             p--;
1868         }
1869
1870         if (tracknumfix[0] != '\0')
1871         {
1872             id3.tracknum = atoi(tracknumfix);
1873             /* Set a flag to indicate track number has been generated. */
1874             entry.flag |= FLAG_TRKNUMGEN;
1875         }
1876         else
1877         {
1878             /* Unable to generate track number. */
1879             id3.tracknum = -1;
1880         }
1881     }
1882
1883     /* Numeric tags */
1884     entry.tag_offset[tag_year] = id3.year;
1885     entry.tag_offset[tag_discnumber] = id3.discnum;
1886     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1887     entry.tag_offset[tag_length] = id3.length;
1888     entry.tag_offset[tag_bitrate] = id3.bitrate;
1889     entry.tag_offset[tag_mtime] = mtime;
1890
1891     /* String tags. */
1892     has_albumartist = id3.albumartist != NULL
1893         && strlen(id3.albumartist) > 0;
1894     has_grouping = id3.grouping != NULL
1895         && strlen(id3.grouping) > 0;
1896
1897     ADD_TAG(entry, tag_filename, &path);
1898     ADD_TAG(entry, tag_title, &id3.title);
1899     ADD_TAG(entry, tag_artist, &id3.artist);
1900     ADD_TAG(entry, tag_album, &id3.album);
1901     ADD_TAG(entry, tag_genre, &id3.genre_string);
1902     ADD_TAG(entry, tag_composer, &id3.composer);
1903     ADD_TAG(entry, tag_comment, &id3.comment);
1904     if (has_albumartist)
1905     {
1906         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1907     }
1908     else
1909     {
1910         ADD_TAG(entry, tag_albumartist, &id3.artist);
1911     }
1912     if (has_grouping)
1913     {
1914         ADD_TAG(entry, tag_grouping, &id3.grouping);
1915     }
1916     else
1917     {
1918         ADD_TAG(entry, tag_grouping, &id3.title);
1919     }
1920     entry.data_length = offset;
1921
1922     /* Write the header */
1923     write(cachefd, &entry, sizeof(struct temp_file_entry));
1924
1925     /* And tags also... Correct order is critical */
1926     write_item(path);
1927     write_item(id3.title);
1928     write_item(id3.artist);
1929     write_item(id3.album);
1930     write_item(id3.genre_string);
1931     write_item(id3.composer);
1932     write_item(id3.comment);
1933     if (has_albumartist)
1934     {
1935         write_item(id3.albumartist);
1936     }
1937     else
1938     {
1939         write_item(id3.artist);
1940     }
1941     if (has_grouping)
1942     {
1943         write_item(id3.grouping);
1944     }
1945     else
1946     {
1947         write_item(id3.title);
1948     }
1949     total_entry_count++;
1950 }
1951
1952 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1953 {
1954     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1955     int len = strlen(str)+1;
1956     int i;
1957     unsigned crc32;
1958     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1959     char buf[TAG_MAXLEN+32];
1960
1961     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1962         buf[i] = tolower(str[i]);
1963     buf[i] = '\0';
1964
1965     crc32 = crc_32(buf, i, 0xffffffff);
1966
1967     if (unique)
1968     {
1969         /* Check if the crc does not exist -> entry does not exist for sure. */
1970         for (i = 0; i < tempbufidx; i++)
1971         {
1972             if (crcbuf[-i] != crc32)
1973                 continue;
1974
1975             if (!strcasecmp(str, index[i].str))
1976             {
1977                 if (id < 0 || id >= lookup_buffer_depth)
1978                 {
1979                     logf("lookup buf overf.: %d", id);
1980                     return false;
1981                 }
1982
1983                 lookup[id] = &index[i];
1984                 return true;
1985             }
1986         }
1987     }
1988
1989     /* Insert to CRC buffer. */
1990     crcbuf[-tempbufidx] = crc32;
1991     tempbuf_left -= 4;
1992
1993     /* Insert it to the buffer. */
1994     tempbuf_left -= len;
1995     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1996         return false;
1997
1998     if (id >= lookup_buffer_depth)
1999     {
2000         logf("lookup buf overf. #2: %d", id);
2001         return false;
2002     }
2003
2004     if (id >= 0)
2005     {
2006         lookup[id] = &index[tempbufidx];
2007         index[tempbufidx].idlist.id = id;
2008     }
2009     else
2010         index[tempbufidx].idlist.id = -1;
2011
2012     index[tempbufidx].idlist.next = NULL;
2013     index[tempbufidx].idx_id = idx_id;
2014     index[tempbufidx].seek = -1;
2015     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2016     memcpy(index[tempbufidx].str, str, len);
2017     tempbuf_pos += len;
2018     tempbufidx++;
2019
2020     return true;
2021 }
2022
2023 static int compare(const void *p1, const void *p2)
2024 {
2025     do_timed_yield();
2026
2027     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2028     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2029
2030     if (strcmp(e1->str, UNTAGGED) == 0)
2031     {
2032         if (strcmp(e2->str, UNTAGGED) == 0)
2033             return 0;
2034         return -1;
2035     }
2036     else if (strcmp(e2->str, UNTAGGED) == 0)
2037         return 1;
2038
2039     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2040 }
2041
2042 static int tempbuf_sort(int fd)
2043 {
2044     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2045     struct tagfile_entry fe;
2046     int i;
2047     int length;
2048
2049     /* Generate reverse lookup entries. */
2050     for (i = 0; i < lookup_buffer_depth; i++)
2051     {
2052         struct tempbuf_id_list *idlist;
2053
2054         if (!lookup[i])
2055             continue;
2056
2057         if (lookup[i]->idlist.id == i)
2058             continue;
2059
2060         idlist = &lookup[i]->idlist;
2061         while (idlist->next != NULL)
2062             idlist = idlist->next;
2063
2064         tempbuf_left -= sizeof(struct tempbuf_id_list);
2065         if (tempbuf_left - 4 < 0)
2066             return -1;
2067
2068         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2069         if (tempbuf_pos & 0x03)
2070         {
2071             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2072             tempbuf_left -= 3;
2073             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2074         }
2075         tempbuf_pos += sizeof(struct tempbuf_id_list);
2076
2077         idlist = idlist->next;
2078         idlist->id = i;
2079         idlist->next = NULL;
2080
2081         do_timed_yield();
2082     }
2083
2084     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2085     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2086
2087     for (i = 0; i < tempbufidx; i++)
2088     {
2089         struct tempbuf_id_list *idlist = &index[i].idlist;
2090
2091         /* Fix the lookup list. */
2092         while (idlist != NULL)
2093         {
2094             if (idlist->id >= 0)
2095                 lookup[idlist->id] = &index[i];
2096             idlist = idlist->next;
2097         }
2098
2099         index[i].seek = lseek(fd, 0, SEEK_CUR);
2100         length = strlen(index[i].str) + 1;
2101         fe.tag_length = length;
2102         fe.idx_id = index[i].idx_id;
2103
2104         /* Check the chunk alignment. */
2105         if ((fe.tag_length + sizeof(struct tagfile_entry))
2106             % TAGFILE_ENTRY_CHUNK_LENGTH)
2107         {
2108             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2109                 ((fe.tag_length + sizeof(struct tagfile_entry))
2110                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2111         }
2112
2113 #ifdef TAGCACHE_STRICT_ALIGN
2114         /* Make sure the entry is long aligned. */
2115         if (index[i].seek & 0x03)
2116         {
2117             logf("tempbuf_sort: alignment error!");
2118             return -3;
2119         }
2120 #endif
2121
2122         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2123             sizeof(struct tagfile_entry))
2124         {
2125             logf("tempbuf_sort: write error #1");
2126             return -1;
2127         }
2128
2129         if (write(fd, index[i].str, length) != length)
2130         {
2131             logf("tempbuf_sort: write error #2");
2132             return -2;
2133         }
2134
2135         /* Write some padding. */
2136         if (fe.tag_length - length > 0)
2137             write(fd, "XXXXXXXX", fe.tag_length - length);
2138     }
2139
2140     return i;
2141 }
2142
2143 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2144 {
2145     if (id < 0 || id >= lookup_buffer_depth)
2146         return NULL;
2147
2148     return lookup[id];
2149 }
2150
2151
2152 inline static int tempbuf_find_location(int id)
2153 {
2154     struct tempbuf_searchidx *entry;
2155
2156     entry = tempbuf_locate(id);
2157     if (entry == NULL)
2158         return -1;
2159
2160     return entry->seek;
2161 }
2162
2163 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2164 {
2165     struct master_header tcmh;
2166     struct index_entry idx;
2167     int masterfd;
2168     int masterfd_pos;
2169     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2170     int max_entries;
2171     int entries_processed = 0;
2172     int i, j;
2173     char buf[TAG_MAXLEN];
2174
2175     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2176
2177     logf("Building numeric indices...");
2178     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2179
2180     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2181         return false;
2182
2183     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2184                          SEEK_CUR);
2185     if (masterfd_pos == filesize(masterfd))
2186     {
2187         logf("we can't append!");
2188         close(masterfd);
2189         return false;
2190     }
2191
2192     while (entries_processed < h->entry_count)
2193     {
2194         int count = MIN(h->entry_count - entries_processed, max_entries);
2195
2196         /* Read in as many entries as possible. */
2197         for (i = 0; i < count; i++)
2198         {
2199             struct temp_file_entry *tfe = &entrybuf[i];
2200             int datastart;
2201
2202             /* Read in numeric data. */
2203             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2204                 sizeof(struct temp_file_entry))
2205             {
2206                 logf("read fail #1");
2207                 close(masterfd);
2208                 return false;
2209             }
2210
2211             datastart = lseek(tmpfd, 0, SEEK_CUR);
2212
2213             /**
2214              * Read string data from the following tags:
2215              * - tag_filename
2216              * - tag_artist
2217              * - tag_album
2218              * - tag_title
2219              *
2220              * A crc32 hash is calculated from the read data
2221              * and stored back to the data offset field kept in memory.
2222              */
2223 #define tmpdb_read_string_tag(tag) \
2224     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2225     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2226     { \
2227         logf("read fail: buffer overflow"); \
2228         close(masterfd); \
2229         return false; \
2230     } \
2231     \
2232     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2233         tfe->tag_length[tag]) \
2234     { \
2235         logf("read fail #2"); \
2236         close(masterfd); \
2237         return false; \
2238     } \
2239     \
2240     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2241     lseek(tmpfd, datastart, SEEK_SET)
2242
2243             tmpdb_read_string_tag(tag_filename);
2244             tmpdb_read_string_tag(tag_artist);
2245             tmpdb_read_string_tag(tag_album);
2246             tmpdb_read_string_tag(tag_title);
2247
2248             /* Seek to the end of the string data. */
2249             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2250         }
2251
2252         /* Backup the master index position. */
2253         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2254         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2255
2256         /* Check if we can resurrect some deleted runtime statistics data. */
2257         for (i = 0; i < tcmh.tch.entry_count; i++)
2258         {
2259             /* Read the index entry. */
2260             if (ecread_index_entry(masterfd, &idx)
2261                 != sizeof(struct index_entry))
2262             {
2263                 logf("read fail #3");
2264                 close(masterfd);
2265                 return false;
2266             }
2267
2268             /**
2269              * Skip unless the entry is marked as being deleted
2270              * or the data has already been resurrected.
2271              */
2272             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2273                 continue;
2274
2275             /* Now try to match the entry. */
2276             /**
2277              * To succesfully match a song, the following conditions
2278              * must apply:
2279              *
2280              * For numeric fields: tag_length
2281              * - Full identical match is required
2282              *
2283              * If tag_filename matches, no further checking necessary.
2284              *
2285              * For string hashes: tag_artist, tag_album, tag_title
2286              * - Two of these must match
2287              */
2288             for (j = 0; j < count; j++)
2289             {
2290                 struct temp_file_entry *tfe = &entrybuf[j];
2291
2292                 /* Try to match numeric fields first. */
2293                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2294                     continue;
2295
2296                 /* Now it's time to do the hash matching. */
2297                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2298                 {
2299                     int match_count = 0;
2300
2301                     /* No filename match, check if we can match two other tags. */
2302 #define tmpdb_match(tag) \
2303     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2304         match_count++
2305
2306                     tmpdb_match(tag_artist);
2307                     tmpdb_match(tag_album);
2308                     tmpdb_match(tag_title);
2309
2310                     if (match_count < 2)
2311                     {
2312                         /* Still no match found, give up. */
2313                         continue;
2314                     }
2315                 }
2316
2317                 /* A match found, now copy & resurrect the statistical data. */
2318 #define tmpdb_copy_tag(tag) \
2319     tfe->tag_offset[tag] = idx.tag_seek[tag]
2320
2321                 tmpdb_copy_tag(tag_playcount);
2322                 tmpdb_copy_tag(tag_rating);
2323                 tmpdb_copy_tag(tag_playtime);
2324                 tmpdb_copy_tag(tag_lastplayed);
2325                 tmpdb_copy_tag(tag_commitid);
2326
2327                 /* Avoid processing this entry again. */
2328                 idx.flag |= FLAG_RESURRECTED;
2329
2330                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2331                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2332                 {
2333                     logf("masterfd writeback fail #1");
2334                     close(masterfd);
2335                     return false;
2336                 }
2337
2338                 logf("Entry resurrected");
2339             }
2340         }
2341
2342
2343         /* Restore the master index position. */
2344         lseek(masterfd, masterfd_pos, SEEK_SET);
2345
2346         /* Commit the data to the index. */
2347         for (i = 0; i < count; i++)
2348         {
2349             int loc = lseek(masterfd, 0, SEEK_CUR);
2350
2351             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2352             {
2353                 logf("read fail #3");
2354                 close(masterfd);
2355                 return false;
2356             }
2357
2358             for (j = 0; j < TAG_COUNT; j++)
2359             {
2360                 if (!TAGCACHE_IS_NUMERIC(j))
2361                     continue;
2362
2363                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2364             }
2365             idx.flag = entrybuf[i].flag;
2366
2367             if (idx.tag_seek[tag_commitid])
2368             {
2369                 /* Data has been resurrected. */
2370                 idx.flag |= FLAG_DIRTYNUM;
2371             }
2372             else if (tc_stat.ready && current_tcmh.commitid > 0)
2373             {
2374                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2375                 idx.flag |= FLAG_DIRTYNUM;
2376             }
2377
2378             /* Write back the updated index. */
2379             lseek(masterfd, loc, SEEK_SET);
2380             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2381             {
2382                 logf("write fail");
2383                 close(masterfd);
2384                 return false;
2385             }
2386         }
2387
2388         entries_processed += count;
2389         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2390     }
2391
2392     close(masterfd);
2393
2394     return true;
2395 }
2396
2397 /**
2398  * Return values:
2399  *     > 0   success
2400  *    == 0   temporary failure
2401  *     < 0   fatal error
2402  */
2403 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2404 {
2405     int i;
2406     struct tagcache_header tch;
2407     struct master_header   tcmh;
2408     struct index_entry idxbuf[IDX_BUF_DEPTH];
2409     int idxbuf_pos;
2410     char buf[TAG_MAXLEN+32], path[MAX_PATH];
2411     const char *file;
2412     int fd = -1, masterfd;
2413     bool error = false;
2414     int init;
2415     int masterfd_pos;
2416
2417     logf("Building index: %d", index_type);
2418
2419     /* Check the number of entries we need to allocate ram for. */
2420     commit_entry_count = h->entry_count + 1;
2421
2422     masterfd = open_master_fd(&tcmh, false);
2423     if (masterfd >= 0)
2424     {
2425         commit_entry_count += tcmh.tch.entry_count;
2426         close(masterfd);
2427     }
2428     else
2429         remove_files(); /* Just to be sure we are clean. */
2430
2431     /* Open the index file, which contains the tag names. */
2432     fd = open_tag_fd(&tch, index_type, true);
2433     if (fd >= 0)
2434     {
2435         logf("tch.datasize=%ld", tch.datasize);
2436         lookup_buffer_depth = 1 +
2437         /* First part */ commit_entry_count +
2438         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2439     }
2440     else
2441     {
2442         lookup_buffer_depth = 1 +
2443         /* First part */ commit_entry_count +
2444         /* Second part */ 0;
2445     }
2446
2447     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2448     logf("commit_entry_count=%ld", commit_entry_count);
2449
2450     /* Allocate buffer for all index entries from both old and new
2451      * tag files. */
2452     tempbufidx = 0;
2453     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2454
2455     /* Allocate lookup buffer. The first portion of commit_entry_count
2456      * contains the new tags in the temporary file and the second
2457      * part for locating entries already in the db.
2458      *
2459      *  New tags  Old tags
2460      * +---------+---------------------------+
2461      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2462      * +---------+---------------------------+
2463      *
2464      * Old tags are inserted to a temporary buffer with position:
2465      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2466      * And new tags with index:
2467      *     tempbuf_insert(idx, ...);
2468      *
2469      * The buffer is sorted and written into tag file:
2470      *     tempbuf_sort(...);
2471      * leaving master index locations messed up.
2472      *
2473      * That is fixed using the lookup buffer for old tags:
2474      *     new_seek = tempbuf_find_location(old_seek, ...);
2475      * and for new tags:
2476      *     new_seek = tempbuf_find_location(idx);
2477      */
2478     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2479     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2480     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2481
2482     /* And calculate the remaining data space used mainly for storing
2483      * tag data (strings). */
2484     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2485     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2486     {
2487         logf("Buffer way too small!");
2488         return 0;
2489     }
2490
2491     if (fd >= 0)
2492     {
2493         /**
2494          * If tag file contains unique tags (sorted index), we will load
2495          * it entirely into memory so we can resort it later for use with
2496          * chunked browsing.
2497          */
2498         if (TAGCACHE_IS_SORTED(index_type))
2499         {
2500             logf("loading tags...");
2501             for (i = 0; i < tch.entry_count; i++)
2502             {
2503                 struct tagfile_entry entry;
2504                 int loc = lseek(fd, 0, SEEK_CUR);
2505                 bool ret;
2506
2507                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2508                 {
2509                     logf("read error #7");
2510                     close(fd);
2511                     return -2;
2512                 }
2513
2514                 if (entry.tag_length >= (int)sizeof(buf))
2515                 {
2516                     logf("too long tag #3");
2517                     close(fd);
2518                     return -2;
2519                 }
2520
2521                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2522                 {
2523                     logf("read error #8");
2524                     close(fd);
2525                     return -2;
2526                 }
2527
2528                 /* Skip deleted entries. */
2529                 if (buf[0] == '\0')
2530                     continue;
2531
2532                 /**
2533                  * Save the tag and tag id in the memory buffer. Tag id
2534                  * is saved so we can later reindex the master lookup
2535                  * table when the index gets resorted.
2536                  */
2537                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2538                                      + commit_entry_count, entry.idx_id,
2539                                      TAGCACHE_IS_UNIQUE(index_type));
2540                 if (!ret)
2541                 {
2542                     close(fd);
2543                     return -3;
2544                 }
2545                 do_timed_yield();
2546             }
2547             logf("done");
2548         }
2549         else
2550             tempbufidx = tch.entry_count;
2551     }
2552     else
2553     {
2554         /**
2555          * Creating new index file to store the tags. No need to preload
2556          * anything whether the index type is sorted or not.
2557          */
2558         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2559         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
2560         fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2561         if (fd < 0)
2562         {
2563             logf("%s open fail", buf);
2564             return -2;
2565         }
2566
2567         tch.magic = TAGCACHE_MAGIC;
2568         tch.entry_count = 0;
2569         tch.datasize = 0;
2570
2571         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2572             != sizeof(struct tagcache_header))
2573         {
2574             logf("header write failed");
2575             close(fd);
2576             return -2;
2577         }
2578     }
2579
2580     file = get_user_file_path(TAGCACHE_FILE_MASTER,
2581                                             IS_FILE|NEED_WRITE,
2582                                             buf, sizeof(buf));
2583     /* Loading the tag lookup file as "master file". */
2584     logf("Loading index file");
2585     masterfd = open(file, O_RDWR);
2586
2587     if (masterfd < 0)
2588     {
2589         logf("Creating new DB");
2590         masterfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2591
2592         if (masterfd < 0)
2593         {
2594             logf("Failure to create index file (%s)", file);
2595             close(fd);
2596             return -2;
2597         }
2598
2599         /* Write the header (write real values later). */
2600         memset(&tcmh, 0, sizeof(struct master_header));
2601         tcmh.tch = *h;
2602         tcmh.tch.entry_count = 0;
2603         tcmh.tch.datasize = 0;
2604         tcmh.dirty = true;
2605         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2606         init = true;
2607         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2608     }
2609     else
2610     {
2611         /**
2612          * Master file already exists so we need to process the current
2613          * file first.
2614          */
2615         init = false;
2616
2617         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2618             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2619         {
2620             logf("header error");
2621             close(fd);
2622             close(masterfd);
2623             return -2;
2624         }
2625
2626         /**
2627          * If we reach end of the master file, we need to expand it to
2628          * hold new tags. If the current index is not sorted, we can
2629          * simply append new data to end of the file.
2630          * However, if the index is sorted, we need to update all tag
2631          * pointers in the master file for the current index.
2632          */
2633         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2634             SEEK_CUR);
2635         if (masterfd_pos == filesize(masterfd))
2636         {
2637             logf("appending...");
2638             init = true;
2639         }
2640     }
2641
2642     /**
2643      * Load new unique tags in memory to be sorted later and added
2644      * to the master lookup file.
2645      */
2646     if (TAGCACHE_IS_SORTED(index_type))
2647     {
2648         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2649         /* h is the header of the temporary file containing new tags. */
2650         logf("inserting new tags...");
2651         for (i = 0; i < h->entry_count; i++)
2652         {
2653             struct temp_file_entry entry;
2654
2655             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2656                 sizeof(struct temp_file_entry))
2657             {
2658                 logf("read fail #3");
2659                 error = true;
2660                 goto error_exit;
2661             }
2662
2663             /* Read data. */
2664             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2665             {
2666                 logf("too long entry!");
2667                 error = true;
2668                 goto error_exit;
2669             }
2670
2671             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2672             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2673                 entry.tag_length[index_type])
2674             {
2675                 logf("read fail #4");
2676                 error = true;
2677                 goto error_exit;
2678             }
2679
2680             if (TAGCACHE_IS_UNIQUE(index_type))
2681                 error = !tempbuf_insert(buf, i, -1, true);
2682             else
2683                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2684
2685             if (error)
2686             {
2687                 logf("insert error");
2688                 goto error_exit;
2689             }
2690
2691             /* Skip to next. */
2692             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2693                     entry.tag_length[index_type], SEEK_CUR);
2694             do_timed_yield();
2695         }
2696         logf("done");
2697
2698         /* Sort the buffer data and write it to the index file. */
2699         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2700         /**
2701          * We need to truncate the index file now. There can be junk left
2702          * at the end of file (however, we _should_ always follow the
2703          * entry_count and don't crash with that).
2704          */
2705         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2706
2707         i = tempbuf_sort(fd);
2708         if (i < 0)
2709             goto error_exit;
2710         logf("sorted %d tags", i);
2711
2712         /**
2713          * Now update all indexes in the master lookup file.
2714          */
2715         logf("updating indices...");
2716         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2717         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2718         {
2719             int j;
2720             int loc = lseek(masterfd, 0, SEEK_CUR);
2721
2722             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2723
2724             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2725                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2726             {
2727                 logf("read fail #5");
2728                 error = true;
2729                 goto error_exit ;
2730             }
2731             lseek(masterfd, loc, SEEK_SET);
2732
2733             for (j = 0; j < idxbuf_pos; j++)
2734             {
2735                 if (idxbuf[j].flag & FLAG_DELETED)
2736                 {
2737                     /* We can just ignore deleted entries. */
2738                     // idxbuf[j].tag_seek[index_type] = 0;
2739                     continue;
2740                 }
2741
2742                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2743                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2744                     + commit_entry_count);
2745
2746                 if (idxbuf[j].tag_seek[index_type] < 0)
2747                 {
2748                     logf("update error: %ld/%d/%ld",
2749                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2750                     error = true;
2751                     goto error_exit;
2752                 }
2753
2754                 do_timed_yield();
2755             }
2756
2757             /* Write back the updated index. */
2758             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2759                         index_entry_ec, tc_stat.econ) !=
2760                 (int)sizeof(struct index_entry)*idxbuf_pos)
2761             {
2762                 logf("write fail");
2763                 error = true;
2764                 goto error_exit;
2765             }
2766         }
2767         logf("done");
2768     }
2769
2770     /**
2771      * Walk through the temporary file containing the new tags.
2772      */
2773     // build_normal_index(h, tmpfd, masterfd, idx);
2774     logf("updating new indices...");
2775     lseek(masterfd, masterfd_pos, SEEK_SET);
2776     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2777     lseek(fd, 0, SEEK_END);
2778     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2779     {
2780         int j;
2781
2782         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2783         if (init)
2784         {
2785             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2786         }
2787         else
2788         {
2789             int loc = lseek(masterfd, 0, SEEK_CUR);
2790
2791             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2792                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2793             {
2794                 logf("read fail #6");
2795                 error = true;
2796                 break ;
2797             }
2798             lseek(masterfd, loc, SEEK_SET);
2799         }
2800
2801         /* Read entry headers. */
2802         for (j = 0; j < idxbuf_pos; j++)
2803         {
2804             if (!TAGCACHE_IS_SORTED(index_type))
2805             {
2806                 struct temp_file_entry entry;
2807                 struct tagfile_entry fe;
2808
2809                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2810                     sizeof(struct temp_file_entry))
2811                 {
2812                     logf("read fail #7");
2813                     error = true;
2814                     break ;
2815                 }
2816
2817                 /* Read data. */
2818                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2819                 {
2820                     logf("too long entry!");
2821                     logf("length=%d", entry.tag_length[index_type]);
2822                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2823                     error = true;
2824                     break ;
2825                 }
2826
2827                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2828                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2829                     entry.tag_length[index_type])
2830                 {
2831                     logf("read fail #8");
2832                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2833                     logf("length=0x%02x", entry.tag_length[index_type]);
2834                     error = true;
2835                     break ;
2836                 }
2837
2838                 /* Write to index file. */
2839                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2840                 fe.tag_length = entry.tag_length[index_type];
2841                 fe.idx_id = tcmh.tch.entry_count + i + j;
2842                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2843                 write(fd, buf, fe.tag_length);
2844                 tempbufidx++;
2845
2846                 /* Skip to next. */
2847                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2848                       entry.tag_length[index_type], SEEK_CUR);
2849             }
2850             else
2851             {
2852                 /* Locate the correct entry from the sorted array. */
2853                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2854                 if (idxbuf[j].tag_seek[index_type] < 0)
2855                 {
2856                     logf("entry not found (%d)", j);
2857                     error = true;
2858                     break ;
2859                 }
2860             }
2861         }
2862
2863         /* Write index. */
2864         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2865                   index_entry_ec, tc_stat.econ) !=
2866             (int)sizeof(struct index_entry)*idxbuf_pos)
2867         {
2868             logf("tagcache: write fail #4");
2869             error = true;
2870             break ;
2871         }
2872
2873         do_timed_yield();
2874     }
2875     logf("done");
2876
2877     /* Finally write the header. */
2878     tch.magic = TAGCACHE_MAGIC;
2879     tch.entry_count = tempbufidx;
2880     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2881     lseek(fd, 0, SEEK_SET);
2882     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2883
2884     if (index_type != tag_filename)
2885         h->datasize += tch.datasize;
2886     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2887     error_exit:
2888
2889     close(fd);
2890     close(masterfd);
2891
2892     if (error)
2893         return -2;
2894
2895     return 1;
2896 }
2897
2898 static bool commit(void)
2899 {
2900     struct tagcache_header tch;
2901     struct master_header   tcmh;
2902     char path[MAX_PATH];
2903     const char *file;
2904     int i, len, rc;
2905     int tmpfd;
2906     int masterfd;
2907 #ifdef HAVE_DIRCACHE
2908     bool dircache_buffer_stolen = false;
2909 #endif
2910     bool local_allocation = false;
2911
2912     logf("committing tagcache");
2913
2914     while (write_lock)
2915         sleep(1);
2916
2917     file = get_user_file_path(TAGCACHE_FILE_TEMP,
2918                             IS_FILE|NEED_WRITE, path, sizeof(path));
2919
2920     tmpfd = open(file, O_RDONLY);
2921     if (tmpfd < 0)
2922     {
2923         logf("nothing to commit");
2924         return true;
2925     }
2926
2927
2928     /* Load the header. */
2929     len = sizeof(struct tagcache_header);
2930     rc = read(tmpfd, &tch, len);
2931
2932     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2933     {
2934         logf("incorrect tmpheader");
2935         close(tmpfd);
2936         remove(file);
2937         return false;
2938     }
2939
2940     if (tch.entry_count == 0)
2941     {
2942         logf("nothing to commit");
2943         close(tmpfd);
2944         remove(file);
2945         return true;
2946     }
2947
2948     /* Fully initialize existing headers (if any) before going further. */
2949     tc_stat.ready = check_all_headers();
2950
2951 #ifdef HAVE_EEPROM_SETTINGS
2952     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
2953                 path, sizeof(path)));
2954 #endif
2955
2956     /* At first be sure to unload the ramcache! */
2957 #ifdef HAVE_TC_RAMCACHE
2958     tc_stat.ramcache = false;
2959 #endif
2960
2961     read_lock++;
2962
2963     /* Try to steal every buffer we can :) */
2964     if (tempbuf_size == 0)
2965         local_allocation = true;
2966
2967 #ifdef HAVE_DIRCACHE
2968     if (tempbuf_size == 0)
2969     {
2970         /* Try to steal the dircache buffer. */
2971         tempbuf = dircache_steal_buffer(&tempbuf_size);
2972         tempbuf_size &= ~0x03;
2973
2974         if (tempbuf_size > 0)
2975         {
2976             dircache_buffer_stolen = true;
2977         }
2978     }
2979 #endif
2980
2981 #ifdef HAVE_TC_RAMCACHE
2982     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2983     {
2984         tempbuf = (char *)(hdr + 1);
2985         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2986         tempbuf_size &= ~0x03;
2987     }
2988 #endif
2989
2990     /* And finally fail if there are no buffers available. */
2991     if (tempbuf_size == 0)
2992     {
2993         logf("delaying commit until next boot");
2994         tc_stat.commit_delayed = true;
2995         close(tmpfd);
2996         read_lock--;
2997         return false;
2998     }
2999
3000     logf("commit %ld entries...", tch.entry_count);
3001
3002     /* Mark DB dirty so it will stay disabled if commit fails. */
3003     current_tcmh.dirty = true;
3004     update_master_header();
3005
3006     /* Now create the index files. */
3007     tc_stat.commit_step = 0;
3008     tch.datasize = 0;
3009     tc_stat.commit_delayed = false;
3010
3011     for (i = 0; i < TAG_COUNT; i++)
3012     {
3013         int ret;
3014
3015         if (TAGCACHE_IS_NUMERIC(i))
3016             continue;
3017
3018         tc_stat.commit_step++;
3019         ret = build_index(i, &tch, tmpfd);
3020         if (ret <= 0)
3021         {
3022             close(tmpfd);
3023             logf("tagcache failed init");
3024             if (ret == 0)
3025                 tc_stat.commit_delayed = true;
3026
3027             tc_stat.commit_step = 0;
3028             read_lock--;
3029             return false;
3030         }
3031     }
3032
3033     if (!build_numeric_indices(&tch, tmpfd))
3034     {
3035         logf("Failure to commit numeric indices");
3036         close(tmpfd);
3037         tc_stat.commit_step = 0;
3038         read_lock--;
3039         return false;
3040     }
3041
3042     close(tmpfd);
3043     remove(file);
3044
3045     tc_stat.commit_step = 0;
3046
3047     /* Update the master index headers. */
3048     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3049     {
3050         read_lock--;
3051         return false;
3052     }
3053
3054     tcmh.tch.entry_count += tch.entry_count;
3055     tcmh.tch.datasize = sizeof(struct master_header)
3056         + sizeof(struct index_entry) * tcmh.tch.entry_count
3057         + tch.datasize;
3058     tcmh.dirty = false;
3059     tcmh.commitid++;
3060
3061     lseek(masterfd, 0, SEEK_SET);
3062     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3063     close(masterfd);
3064
3065     logf("tagcache committed");
3066     tc_stat.ready = check_all_headers();
3067     tc_stat.readyvalid = true;
3068
3069     if (local_allocation)
3070     {
3071         tempbuf = NULL;
3072         tempbuf_size = 0;
3073     }
3074
3075 #ifdef HAVE_DIRCACHE
3076     /* Rebuild the dircache, if we stole the buffer. */
3077     if (dircache_buffer_stolen)
3078         dircache_build(0);
3079 #endif
3080
3081 #ifdef HAVE_TC_RAMCACHE
3082     /* Reload tagcache. */
3083     if (tc_stat.ramcache_allocated > 0)
3084         tagcache_start_scan();
3085 #endif
3086
3087     read_lock--;
3088
3089     return true;
3090 }
3091
3092 static void allocate_tempbuf(void)
3093 {
3094     /* Yeah, malloc would be really nice now :) */
3095 #ifdef __PCTOOL__
3096     tempbuf_size = 32*1024*1024;
3097     tempbuf = malloc(tempbuf_size);
3098 #else
3099     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3100     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3101     audiobuf += tempbuf_size;
3102 #endif
3103 }
3104
3105 static void free_tempbuf(void)
3106 {
3107     if (tempbuf_size == 0)
3108         return ;
3109
3110 #ifdef __PCTOOL__
3111     free(tempbuf);
3112 #else
3113     audiobuf -= tempbuf_size;
3114 #endif
3115     tempbuf = NULL;
3116     tempbuf_size = 0;
3117 }
3118
3119 #ifndef __PCTOOL__
3120
3121 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3122 {
3123     struct index_entry idx;
3124
3125     if (!tc_stat.ready)
3126         return false;
3127
3128     if (!TAGCACHE_IS_NUMERIC(tag))
3129         return false;
3130
3131     if (!get_index(masterfd, idx_id, &idx, false))
3132         return false;
3133
3134     idx.tag_seek[tag] = data;
3135     idx.flag |= FLAG_DIRTYNUM;
3136
3137     return write_index(masterfd, idx_id, &idx);
3138 }
3139
3140 #if 0
3141 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3142                                    int tag, long data)
3143 {
3144     struct master_header myhdr;
3145
3146     if (tcs->masterfd < 0)
3147     {
3148         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3149             return false;
3150     }
3151
3152     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3153 }
3154 #endif
3155
3156 static bool command_queue_is_full(void)
3157 {
3158     int next;
3159
3160     next = command_queue_widx + 1;
3161     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3162         next = 0;
3163
3164     return (next == command_queue_ridx);
3165 }
3166
3167 static void command_queue_sync_callback(void *data)
3168 {
3169     (void)data;
3170     struct master_header myhdr;
3171     int masterfd;
3172
3173     mutex_lock(&command_queue_mutex);
3174
3175     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3176         return;
3177
3178     while (command_queue_ridx != command_queue_widx)
3179     {
3180         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3181
3182         switch (ce->command)
3183         {
3184             case CMD_UPDATE_MASTER_HEADER:
3185             {
3186                 close(masterfd);
3187                 update_master_header();
3188
3189                 /* Re-open the masterfd. */
3190                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3191                     return;
3192
3193                 break;
3194             }
3195             case CMD_UPDATE_NUMERIC:
3196             {
3197                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3198                 break;
3199             }
3200         }
3201
3202         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3203             command_queue_ridx = 0;
3204     }
3205
3206     close(masterfd);
3207
3208     tc_stat.queue_length = 0;
3209     mutex_unlock(&command_queue_mutex);
3210 }
3211
3212 static void run_command_queue(bool force)
3213 {
3214     if (COMMAND_QUEUE_IS_EMPTY)
3215         return;
3216
3217     if (force || command_queue_is_full())
3218         command_queue_sync_callback(NULL);
3219     else
3220         register_storage_idle_func(command_queue_sync_callback);
3221 }
3222
3223 static void queue_command(int cmd, long idx_id, int tag, long data)
3224 {
3225     while (1)
3226     {
3227         int next;
3228
3229         mutex_lock(&command_queue_mutex);
3230         next = command_queue_widx + 1;
3231         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3232             next = 0;
3233
3234         /* Make sure queue is not full. */
3235         if (next != command_queue_ridx)
3236         {
3237             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3238
3239             ce->command = cmd;
3240             ce->idx_id = idx_id;
3241             ce->tag = tag;
3242             ce->data = data;
3243
3244             command_queue_widx = next;
3245
3246             tc_stat.queue_length++;
3247
3248             mutex_unlock(&command_queue_mutex);
3249             break;
3250         }
3251
3252         /* Queue is full, try again later... */
3253         mutex_unlock(&command_queue_mutex);
3254         sleep(1);
3255     }
3256 }
3257
3258 long tagcache_increase_serial(void)
3259 {
3260     long old;
3261
3262     if (!tc_stat.ready)
3263         return -2;
3264
3265     while (read_lock)
3266         sleep(1);
3267
3268     old = current_tcmh.serial++;
3269     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3270
3271     return old;
3272 }
3273
3274 void tagcache_update_numeric(int idx_id, int tag, long data)
3275 {
3276     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3277 }
3278 #endif /* !__PCTOOL__ */
3279
3280 long tagcache_get_serial(void)
3281 {
3282     return current_tcmh.serial;
3283 }
3284
3285 long tagcache_get_commitid(void)
3286 {
3287     return current_tcmh.commitid;
3288 }
3289
3290 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3291 {
3292     char buf[512];
3293     int i;
3294
3295     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3296     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3297     {
3298         if (*datastr == '\0')
3299             break;
3300
3301         if (*datastr == '"' || *datastr == '\\')
3302             buf[i++] = '\\';
3303
3304         buf[i] = *(datastr++);
3305     }
3306
3307     strcpy(&buf[i], "\" ");
3308
3309     write(fd, buf, i + 2);
3310
3311     return true;
3312 }
3313
3314 #ifndef __PCTOOL__
3315
3316 static bool read_tag(char *dest, long size,
3317                      const char *src, const char *tagstr)
3318 {
3319     int pos;
3320     char current_tag[32];
3321
3322     while (*src != '\0')
3323     {
3324         /* Skip all whitespace */
3325         while (*src == ' ')
3326             src++;
3327
3328         if (*src == '\0')
3329             break;
3330
3331         pos = 0;
3332         /* Read in tag name */
3333         while (*src != '=' && *src != ' ')
3334         {
3335             current_tag[pos] = *src;
3336             src++;
3337             pos++;
3338
3339             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3340                 return false;
3341         }
3342         current_tag[pos] = '\0';
3343
3344         /* Read in tag data */
3345
3346         /* Find the start. */
3347         while (*src != '"' && *src != '\0')
3348             src++;
3349
3350         if (*src == '\0' || *(++src) == '\0')
3351             return false;
3352
3353         /* Read the data. */
3354         for (pos = 0; pos < size; pos++)
3355         {
3356             if (*src == '\0')
3357                 break;
3358
3359             if (*src == '\\')
3360             {
3361                 dest[pos] = *(src+1);
3362                 src += 2;
3363                 continue;
3364             }
3365
3366             dest[pos] = *src;
3367
3368             if (*src == '"')
3369             {
3370                 src++;
3371                 break;
3372             }
3373
3374             if (*src == '\0')
3375                 break;
3376
3377             src++;
3378         }
3379         dest[pos] = '\0';
3380
3381         if (!strcasecmp(tagstr, current_tag))
3382             return true;
3383     }
3384
3385     return false;
3386 }
3387
3388 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3389 {
3390     struct index_entry idx;
3391     char tag_data[TAG_MAXLEN+32];
3392     int idx_id;
3393     long masterfd = (long)parameters;
3394     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3395         tag_commitid };
3396     int i;
3397     (void)line_n;
3398
3399     if (*buf == '#')
3400         return 0;
3401
3402     logf("%d/%s", line_n, buf);
3403     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3404     {
3405         logf("filename missing");
3406         logf("-> %s", buf);
3407         return 0;
3408     }
3409
3410     idx_id = find_index(tag_data);
3411     if (idx_id < 0)
3412     {
3413         logf("entry not found");
3414         return 0;
3415     }
3416
3417     if (!get_index(masterfd, idx_id, &idx, false))
3418     {
3419         logf("failed to retrieve index entry");
3420         return 0;
3421     }
3422
3423     /* Stop if tag has already been modified. */
3424     if (idx.flag & FLAG_DIRTYNUM)
3425         return 0;
3426
3427     logf("import: %s", tag_data);
3428
3429     idx.flag |= FLAG_DIRTYNUM;
3430     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3431     {
3432         int data;
3433
3434         if (!read_tag(tag_data, sizeof tag_data, buf,
3435                       tagcache_tag_to_str(import_tags[i])))
3436         {
3437             continue;
3438         }
3439
3440         data = atoi(tag_data);
3441         if (data < 0)
3442             continue;
3443
3444         idx.tag_seek[import_tags[i]] = data;
3445
3446         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3447             current_tcmh.serial = data + 1;
3448         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3449             current_tcmh.commitid = data + 1;
3450     }
3451
3452     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3453 }
3454
3455 bool tagcache_import_changelog(void)
3456 {
3457     struct master_header myhdr;
3458     struct tagcache_header tch;
3459     int clfd;
3460     long masterfd;
3461     char buf[MAX(MAX_PATH, 2048)];
3462     const char *file;
3463
3464     if (!tc_stat.ready)
3465         return false;
3466
3467     while (read_lock)
3468         sleep(1);
3469
3470     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG,
3471                                    IS_FILE|NEED_WRITE, buf, sizeof(buf));
3472     clfd = open(file, O_RDONLY);
3473     if (clfd < 0)
3474     {
3475         logf("failure to open changelog");
3476         return false;
3477     }
3478
3479     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3480     {
3481         close(clfd);
3482         return false;
3483     }
3484
3485     write_lock++;
3486
3487     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3488
3489     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3490                   parse_changelog_line);
3491
3492     close(clfd);
3493     close(masterfd);
3494
3495     if (filenametag_fd >= 0)
3496     {
3497         close(filenametag_fd);
3498         filenametag_fd = -1;
3499     }
3500
3501     write_lock--;
3502
3503     update_master_header();
3504
3505     return true;
3506 }
3507
3508 #endif /* !__PCTOOL__ */
3509
3510 bool tagcache_create_changelog(struct tagcache_search *tcs)
3511 {
3512     struct master_header myhdr;
3513     struct index_entry idx;
3514     const char *file;
3515     char buf[MAX(TAG_MAXLEN+32, MAX_PATH)];
3516     char temp[32];
3517     int clfd;
3518     int i, j;
3519
3520     if (!tc_stat.ready)
3521         return false;
3522
3523     if (!tagcache_search(tcs, tag_filename))
3524         return false;
3525
3526     /* Initialize the changelog */
3527     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG, IS_FILE | NEED_WRITE,
3528                               buf, sizeof(buf));
3529     clfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3530     if (clfd < 0)
3531     {
3532         logf("failure to open changelog");
3533         return false;
3534     }
3535
3536     if (tcs->masterfd < 0)
3537     {
3538         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3539             return false;
3540     }
3541     else
3542     {
3543         lseek(tcs->masterfd, 0, SEEK_SET);
3544         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3545     }
3546
3547     write(clfd, "## Changelog version 1\n", 23);
3548
3549     for (i = 0; i < myhdr.tch.entry_count; i++)
3550     {
3551         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3552         {
3553             logf("read error #9");
3554             tagcache_search_finish(tcs);
3555             close(clfd);
3556             return false;
3557         }
3558
3559         /* Skip until the entry found has been modified. */
3560         if (! (idx.flag & FLAG_DIRTYNUM) )
3561             continue;
3562
3563         /* Skip deleted entries too. */
3564         if (idx.flag & FLAG_DELETED)
3565             continue;
3566
3567         /* Now retrieve all tags. */
3568         for (j = 0; j < TAG_COUNT; j++)
3569         {
3570             if (TAGCACHE_IS_NUMERIC(j))
3571             {
3572                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3573                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3574                 continue;
3575             }
3576
3577             tcs->type = j;
3578             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3579             write_tag(clfd, tagcache_tag_to_str(j), buf);
3580         }
3581
3582         write(clfd, "\n", 1);
3583         do_timed_yield();
3584     }
3585
3586     close(clfd);
3587
3588     tagcache_search_finish(tcs);
3589
3590     return true;
3591 }
3592
3593 static bool delete_entry(long idx_id)
3594 {
3595     int fd = -1;
3596     int masterfd = -1;
3597     int tag, i;
3598     struct index_entry idx, myidx;
3599     struct master_header myhdr;
3600     char buf[TAG_MAXLEN+32];
3601     int in_use[TAG_COUNT];
3602
3603     logf("delete_entry(): %ld", idx_id);
3604
3605 #ifdef HAVE_TC_RAMCACHE
3606     /* At first mark the entry removed from ram cache. */
3607     if (tc_stat.ramcache)
3608         hdr->indices[idx_id].flag |= FLAG_DELETED;
3609 #endif
3610
3611     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3612         return false;
3613
3614     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3615     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3616     {
3617         logf("delete_entry(): read error");
3618         goto cleanup;
3619     }
3620
3621     if (myidx.flag & FLAG_DELETED)
3622     {
3623         logf("delete_entry(): already deleted!");
3624         goto cleanup;
3625     }
3626
3627     myidx.flag |= FLAG_DELETED;
3628     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3629     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3630     {
3631         logf("delete_entry(): write_error #1");
3632         goto cleanup;
3633     }
3634
3635     /* Now check which tags are no longer in use (if any) */
3636     for (tag = 0; tag < TAG_COUNT; tag++)
3637         in_use[tag] = 0;
3638
3639     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3640     for (i = 0; i < myhdr.tch.entry_count; i++)
3641     {
3642         struct index_entry *idxp;
3643
3644 #ifdef HAVE_TC_RAMCACHE
3645         /* Use RAM DB if available for greater speed */
3646         if (tc_stat.ramcache)
3647             idxp = &hdr->indices[i];
3648         else
3649 #endif
3650         {
3651             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3652             {
3653                 logf("delete_entry(): read error #2");
3654                 goto cleanup;
3655             }
3656             idxp = &idx;
3657         }
3658
3659         if (idxp->flag & FLAG_DELETED)
3660             continue;
3661
3662         for (tag = 0; tag < TAG_COUNT; tag++)
3663         {
3664             if (TAGCACHE_IS_NUMERIC(tag))
3665                 continue;
3666
3667             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3668                 in_use[tag]++;
3669         }
3670     }
3671
3672     /* Now delete all tags no longer in use. */
3673     for (tag = 0; tag < TAG_COUNT; tag++)
3674     {
3675         struct tagcache_header tch;
3676         int oldseek = myidx.tag_seek[tag];
3677
3678         if (TAGCACHE_IS_NUMERIC(tag))
3679             continue;
3680
3681         /**
3682          * Replace tag seek with a hash value of the field string data.
3683          * That way runtime statistics of moved or altered files can be
3684          * resurrected.
3685          */
3686 #ifdef HAVE_TC_RAMCACHE
3687         if (tc_stat.ramcache && tag != tag_filename)
3688         {
3689             struct tagfile_entry *tfe;
3690             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3691
3692             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3693             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3694             myidx.tag_seek[tag] = *seek;
3695         }
3696         else
3697 #endif
3698         {
3699             struct tagfile_entry tfe;
3700
3701             /* Open the index file, which contains the tag names. */
3702             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3703                 goto cleanup;
3704
3705             /* Skip the header block */
3706             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3707             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3708             {
3709                 logf("delete_entry(): read error #3");
3710                 goto cleanup;
3711             }
3712
3713             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3714             {
3715                 logf("delete_entry(): read error #4");
3716                 goto cleanup;
3717             }
3718
3719             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3720         }
3721
3722         if (in_use[tag])
3723         {
3724             logf("in use: %d/%d", tag, in_use[tag]);
3725             if (fd >= 0)
3726             {
3727                 close(fd);
3728                 fd = -1;
3729             }
3730             continue;
3731         }
3732
3733 #ifdef HAVE_TC_RAMCACHE
3734         /* Delete from ram. */
3735         if (tc_stat.ramcache && tag != tag_filename)
3736         {
3737             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3738             tagentry->tag_data[0] = '\0';
3739         }
3740 #endif
3741
3742         /* Open the index file, which contains the tag names. */
3743         if (fd < 0)
3744         {
3745             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3746                 goto cleanup;
3747         }
3748
3749         /* Skip the header block */
3750         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3751
3752         /* Debug, print 10 first characters of the tag
3753         read(fd, buf, 10);
3754         buf[10]='\0';
3755         logf("TAG:%s", buf);
3756         lseek(fd, -10, SEEK_CUR);
3757         */
3758
3759         /* Write first data byte in tag as \0 */
3760         write(fd, "", 1);
3761
3762         /* Now tag data has been removed */
3763         close(fd);
3764         fd = -1;
3765     }
3766
3767     /* Write index entry back into master index. */
3768     lseek(masterfd, sizeof(struct master_header) +
3769           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3770     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3771     {
3772         logf("delete_entry(): write_error #2");
3773         goto cleanup;
3774     }
3775
3776     close(masterfd);
3777
3778     return true;
3779
3780     cleanup:
3781     if (fd >= 0)
3782         close(fd);
3783     if (masterfd >= 0)
3784         close(masterfd);
3785
3786     return false;
3787 }
3788
3789 #ifndef __PCTOOL__
3790 /**
3791  * Returns true if there is an event waiting in the queue
3792  * that requires the current operation to be aborted.
3793  */
3794 static bool check_event_queue(void)
3795 {
3796     struct queue_event ev;
3797
3798     if(!queue_peek(&tagcache_queue, &ev))
3799         return false;
3800
3801     switch (ev.id)
3802     {
3803         case Q_STOP_SCAN:
3804         case SYS_POWEROFF:
3805         case SYS_USB_CONNECTED:
3806             return true;
3807     }
3808
3809     return false;
3810 }
3811 #endif
3812
3813 #ifdef HAVE_TC_RAMCACHE
3814 static bool allocate_tagcache(void)
3815 {
3816     struct master_header tcmh;
3817     int fd;
3818
3819     /* Load the header. */
3820     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3821     {
3822         hdr = NULL;
3823         return false;
3824     }
3825
3826     close(fd);
3827
3828     /**
3829      * Now calculate the required cache size plus
3830      * some extra space for alignment fixes.
3831      */
3832     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3833         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3834     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3835     memset(hdr, 0, sizeof(struct ramcache_header));
3836     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3837     hdr->indices = (struct index_entry *)(hdr + 1);
3838     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3839
3840     return true;
3841 }
3842
3843 # ifdef HAVE_EEPROM_SETTINGS
3844 static bool tagcache_dumpload(void)
3845 {
3846     struct statefile_header shdr;
3847     char path[MAX_PATH];
3848     const char *file;
3849     int fd, rc;
3850     long offpos;
3851     int i;
3852
3853     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3854                                     path, sizeof(path));
3855     fd = open(file, O_RDONLY);
3856     if (fd < 0)
3857     {
3858         logf("no tagcache statedump");
3859         return false;
3860     }
3861
3862     /* Check the statefile memory placement */
3863     hdr = buffer_alloc(0);
3864     rc = read(fd, &shdr, sizeof(struct statefile_header));
3865     if (rc != sizeof(struct statefile_header)
3866         /* || (long)hdr != (long)shdr.hdr */)
3867     {
3868         logf("incorrect statefile");
3869         hdr = NULL;
3870         close(fd);
3871         return false;
3872     }
3873
3874     offpos = (long)hdr - (long)shdr.hdr;
3875
3876     /* Lets allocate real memory and load it */
3877     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3878     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3879     close(fd);
3880
3881     if (rc != shdr.tc_stat.ramcache_allocated)
3882     {
3883         logf("read failure!");
3884         hdr = NULL;
3885         return false;
3886     }
3887
3888     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3889
3890     /* Now fix the pointers */
3891     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3892     for (i = 0; i < TAG_COUNT; i++)
3893         hdr->tags[i] += offpos;
3894
3895     return true;
3896 }
3897
3898 static bool tagcache_dumpsave(void)
3899 {
3900     struct statefile_header shdr;
3901     char path[MAX_PATH];
3902     const char *file;
3903     int fd;
3904
3905     if (!tc_stat.ramcache)
3906         return false;
3907
3908     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3909                                     path, sizeof(path));
3910     fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3911     if (fd < 0)
3912     {
3913         logf("failed to create a statedump");
3914         return false;
3915     }
3916
3917     /* Create the header */
3918     shdr.hdr = hdr;
3919     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3920     write(fd, &shdr, sizeof(struct statefile_header));
3921
3922     /* And dump the data too */
3923     write(fd, hdr, tc_stat.ramcache_allocated);
3924     close(fd);
3925
3926     return true;
3927 }
3928 # endif
3929
3930 static bool load_tagcache(void)
3931 {
3932     struct tagcache_header *tch;
3933     long bytesleft = tc_stat.ramcache_allocated;
3934     struct index_entry *idx;
3935     int rc, fd;
3936     char *p, path[MAX_PATH];
3937     const char *file;
3938     int i, tag;
3939
3940 # ifdef HAVE_DIRCACHE
3941     while (dircache_is_initializing())
3942         sleep(1);
3943
3944     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3945 # endif
3946
3947     logf("loading tagcache to ram...");
3948
3949     file = get_user_file_path(TAGCACHE_FILE_MASTER,
3950                               IS_FILE|NEED_WRITE,
3951                               path, sizeof(path));
3952     fd = open(file, O_RDONLY);
3953     if (fd < 0)
3954     {
3955         logf("tagcache open failed");
3956         return false;
3957     }
3958
3959     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3960         != sizeof(struct master_header)
3961         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3962     {
3963         logf("incorrect header");
3964         return false;
3965     }
3966
3967     idx = hdr->indices;
3968
3969     /* Load the master index table. */
3970     for (i = 0; i < hdr->h.tch.entry_count; i++)
3971     {
3972         rc = ecread_index_entry(fd, idx);
3973         if (rc != sizeof(struct index_entry))
3974         {
3975             logf("read error #10");
3976             close(fd);
3977             return false;
3978         }
3979
3980         bytesleft -= sizeof(struct index_entry);
3981         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3982         {
3983             logf("too big tagcache.");
3984             close(fd);
3985             return false;
3986         }
3987
3988         idx++;
3989     }
3990
3991     close(fd);
3992
3993     /* Load the tags. */
3994     p = (char *)idx;
3995     for (tag = 0; tag < TAG_COUNT; tag++)
3996     {
3997         struct tagfile_entry *fe;
3998         char buf[TAG_MAXLEN+32];
3999
4000         if (TAGCACHE_IS_NUMERIC(tag))
4001             continue ;
4002
4003         //p = ((void *)p+1);
4004         p = (char *)((long)p & ~0x03) + 0x04;
4005         hdr->tags[tag] = p;
4006
4007         /* Check the header. */
4008         tch = (struct tagcache_header *)p;
4009         p += sizeof(struct tagcache_header);
4010
4011         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
4012             return false;
4013
4014         for (hdr->entry_count[tag] = 0;
4015              hdr->entry_count[tag] < tch->entry_count;
4016              hdr->entry_count[tag]++)
4017         {
4018             long pos;
4019
4020             if (do_timed_yield())
4021             {
4022                 /* Abort if we got a critical event in queue */
4023                 if (check_event_queue())
4024                     return false;
4025             }
4026
4027             fe = (struct tagfile_entry *)p;
4028             pos = lseek(fd, 0, SEEK_CUR);
4029             rc = ecread_tagfile_entry(fd, fe);
4030             if (rc != sizeof(struct tagfile_entry))
4031             {
4032                 /* End of lookup table. */
4033                 logf("read error #11");
4034                 close(fd);
4035                 return false;
4036             }
4037
4038             /* We have a special handling for the filename tags. */
4039             if (tag == tag_filename)
4040             {
4041 # ifdef HAVE_DIRCACHE
4042                 const struct dircache_entry *dc;
4043 # endif
4044
4045                 // FIXME: This is wrong!
4046                 // idx = &hdr->indices[hdr->entry_count[i]];
4047                 idx = &hdr->indices[fe->idx_id];
4048
4049                 if (fe->tag_length >= (long)sizeof(buf)-1)
4050                 {
4051                     read(fd, buf, 10);
4052                     buf[10] = '\0';
4053                     logf("TAG:%s", buf);
4054                     logf("too long filename");
4055                     close(fd);
4056                     return false;
4057                 }
4058
4059                 rc = read(fd, buf, fe->tag_length);
4060                 if (rc != fe->tag_length)
4061                 {
4062                     logf("read error #12");
4063                     close(fd);
4064                     return false;
4065                 }
4066
4067                 /* Check if the entry has already been removed */
4068                 if (idx->flag & FLAG_DELETED)
4069                     continue;
4070
4071                 /* This flag must not be used yet. */
4072                 if (idx->flag & FLAG_DIRCACHE)
4073                 {
4074                     logf("internal error!");
4075                     close(fd);
4076                     return false;
4077                 }
4078
4079                 if (idx->tag_seek[tag] != pos)
4080                 {
4081                     logf("corrupt data structures!");
4082                     close(fd);
4083                     return false;
4084                 }
4085
4086 # ifdef HAVE_DIRCACHE
4087                 if (dircache_is_enabled())
4088                 {
4089                     dc = dircache_get_entry_ptr(buf);
4090                     if (dc == NULL)
4091                     {
4092                         logf("Entry no longer valid.");
4093                         logf("-> %s", buf);
4094                         if (global_settings.tagcache_autoupdate)
4095                             delete_entry(fe->idx_id);
4096                         continue ;
4097                     }
4098
4099                     idx->flag |= FLAG_DIRCACHE;
4100                     idx->tag_seek[tag_filename] = (long)dc;
4101                 }
4102                 else
4103 # endif
4104                 {
4105                     /* This will be very slow unless dircache is enabled
4106                        or target is flash based, but do it anyway for
4107                        consistency. */
4108                     /* Check if entry has been removed. */
4109                     if (global_settings.tagcache_autoupdate)
4110                     {
4111                         if (!file_exists(buf))
4112                         {
4113                             logf("Entry no longer valid.");
4114                             logf("-> %s", buf);
4115                             delete_entry(fe->idx_id);
4116                             continue;
4117                         }
4118                     }
4119                 }
4120
4121                 continue ;
4122             }
4123
4124             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4125             if (bytesleft < 0)
4126             {
4127                 logf("too big tagcache #2");
4128                 logf("tl: %ld", fe->tag_length);
4129                 logf("bl: %ld", bytesleft);
4130                 close(fd);
4131                 return false;
4132             }
4133
4134             p = fe->tag_data;
4135             rc = read(fd, fe->tag_data, fe->tag_length);
4136             p += rc;
4137
4138             if (rc != fe->tag_length)
4139             {
4140                 logf("read error #13");
4141                 logf("rc=0x%04x", rc); // 0x431
4142                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4143                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4144                 logf("tag=0x%02x", tag); // 0x00
4145                 close(fd);
4146                 return false;
4147             }
4148         }
4149         close(fd);
4150     }
4151
4152     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4153     logf("tagcache loaded into ram!");
4154
4155     return true;
4156 }
4157 #endif /* HAVE_TC_RAMCACHE */
4158
4159 static bool check_deleted_files(void)
4160 {
4161     int fd;
4162     char buf[TAG_MAXLEN+32], path[MAX_PATH];
4163     const char *file;
4164     struct tagfile_entry tfe;
4165
4166     logf("reverse scan...");
4167     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4168     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
4169     fd = open(file, O_RDONLY);
4170
4171     if (fd < 0)
4172     {
4173         logf("%s open fail", buf);
4174         return false;
4175     }
4176
4177     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4178     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4179 #ifndef __PCTOOL__
4180            && !check_event_queue()
4181 #endif
4182            )
4183     {
4184         if (tfe.tag_length >= (long)sizeof(buf)-1)
4185         {
4186             logf("too long tag");
4187             close(fd);
4188             return false;
4189         }
4190
4191         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4192         {
4193             logf("read error #14");
4194             close(fd);
4195             return false;
4196         }
4197
4198         /* Check if the file has already deleted from the db. */
4199         if (*buf == '\0')
4200             continue;
4201
4202         /* Now check if the file exists. */
4203         if (!file_exists(buf))
4204         {
4205             logf("Entry no longer valid.");
4206             logf("-> %s / %ld", buf, tfe.tag_length);
4207             delete_entry(tfe.idx_id);
4208         }
4209     }
4210
4211     close(fd);
4212
4213     logf("done");
4214
4215     return true;
4216 }
4217
4218
4219 /* Note that this function must not be inlined, otherwise the whole point
4220  * of having the code in a separate function is lost.
4221  */
4222 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4223     int *ignore, int *unignore)
4224 {
4225     char newpath[MAX_PATH];
4226
4227     /* check for a database.ignore file */
4228     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4229     *ignore = file_exists(newpath);
4230     /* check for a database.unignore file */
4231     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4232     *unignore = file_exists(newpath);
4233 }
4234
4235
4236 static bool check_dir(const char *dirname, int add_files)
4237 {
4238     DIR *dir;
4239     int len;
4240     int success = false;
4241     int ignore, unignore;
4242
4243     dir = opendir(dirname);
4244     if (!dir)
4245     {
4246         logf("tagcache: opendir(%s) failed", dirname);
4247         return false;
4248     }
4249
4250     /* check for a database.ignore and database.unignore */
4251     check_ignore(dirname, &ignore, &unignore);
4252
4253     /* don't do anything if both ignore and unignore are there */
4254     if (ignore != unignore)
4255         add_files = unignore;
4256
4257     /* Recursively scan the dir. */
4258 #ifdef __PCTOOL__
4259     while (1)
4260 #else
4261     while (!check_event_queue())
4262 #endif
4263     {
4264         struct dirent *entry;
4265
4266         entry = readdir(dir);
4267
4268         if (entry == NULL)
4269         {
4270             success = true;
4271             break ;
4272         }
4273
4274         struct dirinfo info = dir_get_info(dir, entry);
4275
4276         if (!strcmp((char *)entry->d_name, ".") ||
4277             !strcmp((char *)entry->d_name, ".."))
4278             continue;
4279
4280         yield();
4281
4282         len = strlen(curpath);
4283         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4284                  entry->d_name);
4285
4286         processed_dir_count++;
4287         if (info.attribute & ATTR_DIRECTORY)
4288             check_dir(curpath, add_files);
4289         else if (add_files)
4290         {
4291             tc_stat.curentry = curpath;
4292
4293             /* Add a new entry to the temporary db file. */
4294             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4295 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4296                          , dir->internal_entry
4297 #endif
4298                          );
4299
4300             /* Wait until current path for debug screen is read and unset. */
4301             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4302                 yield();
4303
4304             tc_stat.curentry = NULL;
4305         }
4306
4307         curpath[len] = '\0';
4308     }
4309
4310     closedir(dir);
4311
4312     return success;
4313 }
4314
4315 void tagcache_screensync_event(void)
4316 {
4317     tc_stat.curentry = NULL;
4318 }
4319
4320 void tagcache_screensync_enable(bool state)
4321 {
4322     tc_stat.syncscreen = state;
4323 }
4324
4325 void tagcache_build(const char *path)
4326 {
4327     struct tagcache_header header;
4328     bool ret;
4329     char buf[MAX_PATH];
4330     const char *file;
4331
4332     curpath[0] = '\0';
4333     data_size = 0;
4334     total_entry_count = 0;
4335     processed_dir_count = 0;
4336
4337 #ifdef HAVE_DIRCACHE
4338     while (dircache_is_initializing())
4339         sleep(1);
4340 #endif
4341
4342     logf("updating tagcache");
4343
4344     file = get_user_file_path(TAGCACHE_FILE_TEMP,
4345                                     IS_FILE|NEED_WRITE, buf, sizeof(buf));
4346
4347
4348     if (file_exists(file))
4349     {
4350         logf("skipping, cache already waiting for commit");
4351         return ;
4352     }
4353
4354     cachefd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0666);
4355     if (cachefd < 0)
4356     {
4357         logf("master file open failed: %s", file);
4358         return ;
4359     }
4360
4361     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4362
4363     cpu_boost(true);
4364
4365     logf("Scanning files...");
4366     /* Scan for new files. */
4367     memset(&header, 0, sizeof(struct tagcache_header));
4368     write(cachefd, &header, sizeof(struct tagcache_header));
4369
4370     if (strcmp("/", path) != 0)
4371         strcpy(curpath, path);
4372     ret = check_dir(path, true);
4373
4374     /* Write the header. */
4375     header.magic = TAGCACHE_MAGIC;
4376     header.datasize = data_size;
4377     header.entry_count = total_entry_count;
4378     lseek(cachefd, 0, SEEK_SET);
4379     write(cachefd, &header, sizeof(struct tagcache_header));
4380     close(cachefd);
4381
4382     if (filenametag_fd >= 0)
4383     {
4384         close(filenametag_fd);
4385         filenametag_fd = -1;
4386     }
4387
4388     if (!ret)
4389     {
4390         logf("Aborted.");
4391         cpu_boost(false);
4392         return ;
4393     }
4394
4395     /* Commit changes to the database. */
4396 #ifdef __PCTOOL__
4397     allocate_tempbuf();
4398 #endif
4399     if (commit())
4400     {
4401         remove(file);
4402         logf("tagcache built!");
4403     }
4404 #ifdef __PCTOOL__
4405     free_tempbuf();
4406 #endif
4407
4408 #ifdef HAVE_TC_RAMCACHE
4409     if (hdr)
4410     {
4411         /* Import runtime statistics if we just initialized the db. */
4412         if (hdr->h.serial == 0)
4413             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4414     }
4415 #endif
4416
4417     cpu_boost(false);
4418 }
4419
4420 #ifdef HAVE_TC_RAMCACHE
4421 static void load_ramcache(void)
4422 {
4423     if (!hdr)
4424         return ;
4425
4426     cpu_boost(true);
4427
4428     /* At first we should load the cache (if exists). */
4429     tc_stat.ramcache = load_tagcache();
4430
4431     if (!tc_stat.ramcache)
4432     {
4433         /* If loading failed, it must indicate some problem with the db
4434          * so disable it entirely to prevent further issues. */
4435         tc_stat.ready = false;
4436         hdr = NULL;
4437     }
4438
4439     cpu_boost(false);
4440 }
4441
4442 void tagcache_unload_ramcache(void)
4443 {
4444     tc_stat.ramcache = false;
4445     /* Just to make sure there is no statefile present. */
4446
4447 #if 0
4448     char path[MAX_PATH];
4449     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4450                 path, sizeof(path)));
4451 #endif
4452 }
4453 #endif
4454
4455 #ifndef __PCTOOL__
4456 static void tagcache_thread(void)
4457 {
4458     struct queue_event ev;
4459     bool check_done = false;
4460     char path[MAX_PATH];
4461
4462     /* If the previous cache build/update was interrupted, commit
4463      * the changes first in foreground. */
4464     cpu_boost(true);
4465     allocate_tempbuf();
4466     commit();
4467     free_tempbuf();
4468
4469 #ifdef HAVE_TC_RAMCACHE
4470 # ifdef HAVE_EEPROM_SETTINGS
4471     if (firmware_settings.initialized && firmware_settings.disk_clean
4472         && global_settings.tagcache_ram)
4473     {
4474         check_done = tagcache_dumpload();
4475     }
4476
4477     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4478                 path, sizeof(path)));
4479 # endif
4480
4481     /* Allocate space for the tagcache if found on disk. */
4482     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4483         allocate_tagcache();
4484 #endif
4485
4486     cpu_boost(false);
4487     tc_stat.initialized = true;
4488
4489     /* Don't delay bootup with the header check but do it on background. */
4490     if (!tc_stat.ready)
4491     {
4492         sleep(HZ);
4493         tc_stat.ready = check_all_headers();
4494         tc_stat.readyvalid = true;
4495     }
4496
4497     while (1)
4498     {
4499         run_command_queue(false);
4500
4501         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4502
4503         switch (ev.id)
4504         {
4505             case Q_IMPORT_CHANGELOG:
4506                 tagcache_import_changelog();
4507                 break;
4508
4509             case Q_REBUILD:
4510                 remove_files();
4511                 remove(get_user_file_path(TAGCACHE_FILE_TEMP,
4512                                 IS_FILE|NEED_WRITE, path, sizeof(path)));
4513                 tagcache_build("/");
4514                 break;
4515
4516             case Q_UPDATE:
4517                 tagcache_build("/");
4518 #ifdef HAVE_TC_RAMCACHE
4519                 load_ramcache();
4520 #endif
4521                 check_deleted_files();
4522                 break ;
4523
4524             case Q_START_SCAN:
4525                 check_done = false;
4526             case SYS_TIMEOUT:
4527                 if (check_done || !tc_stat.ready)
4528                     break ;
4529
4530 #ifdef HAVE_TC_RAMCACHE
4531                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4532                 {
4533                     load_ramcache();
4534                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4535                         tagcache_build("/");
4536                 }
4537                 else
4538 #endif
4539                 if (global_settings.tagcache_autoupdate)
4540                 {
4541                     tagcache_build("/");
4542
4543                     /* This will be very slow unless dircache is enabled
4544                        or target is flash based, but do it anyway for
4545                        consistency. */
4546                     check_deleted_files();
4547                 }
4548
4549                 logf("tagcache check done");
4550
4551                 check_done = true;
4552                 break ;
4553
4554             case Q_STOP_SCAN:
4555                 break ;
4556
4557             case SYS_POWEROFF:
4558                 break ;
4559
4560 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4561             case SYS_USB_CONNECTED:
4562                 logf("USB: TagCache");
4563                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4564                 usb_wait_for_disconnect(&tagcache_queue);
4565                 break ;
4566 #endif
4567         }
4568     }
4569 }
4570
4571 bool tagcache_prepare_shutdown(void)
4572 {
4573     if (tagcache_get_commit_step() > 0)
4574         return false;
4575
4576     tagcache_stop_scan();
4577     while (read_lock || write_lock)
4578         sleep(1);
4579
4580     return true;
4581 }
4582
4583 void tagcache_shutdown(void)
4584 {
4585     /* Flush the command queue. */
4586     run_command_queue(true);
4587
4588 #ifdef HAVE_EEPROM_SETTINGS
4589     if (tc_stat.ramcache)
4590         tagcache_dumpsave();
4591 #endif
4592 }
4593
4594 static int get_progress(void)
4595 {
4596     int total_count = -1;
4597
4598 #ifdef HAVE_DIRCACHE
4599     if (dircache_is_enabled())
4600     {
4601         total_count = dircache_get_entry_count();
4602     }
4603     else
4604 #endif
4605 #ifdef HAVE_TC_RAMCACHE
4606     {
4607         if (hdr && tc_stat.ramcache)
4608             total_count = hdr->h.tch.entry_count;
4609     }
4610 #endif
4611
4612     if (total_count < 0)
4613         return -1;
4614
4615     return processed_dir_count * 100 / total_count;
4616 }
4617
4618 struct tagcache_stat* tagcache_get_stat(void)
4619 {
4620     tc_stat.progress = get_progress();
4621     tc_stat.processed_entries = processed_dir_count;
4622
4623     return &tc_stat;
4624 }
4625
4626 void tagcache_start_scan(void)
4627 {
4628     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4629 }
4630
4631 bool tagcache_update(void)
4632 {
4633     if (!tc_stat.ready)
4634         return false;
4635
4636     queue_post(&tagcache_queue, Q_UPDATE, 0);
4637     return false;
4638 }
4639
4640 bool tagcache_rebuild()
4641 {
4642     queue_post(&tagcache_queue, Q_REBUILD, 0);
4643     return false;
4644 }
4645
4646 void tagcache_stop_scan(void)
4647 {
4648     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4649 }
4650
4651 #ifdef HAVE_TC_RAMCACHE
4652 bool tagcache_is_ramcache(void)
4653 {
4654     return tc_stat.ramcache;
4655 }
4656 #endif
4657
4658 #endif /* !__PCTOOL__ */
4659
4660
4661 void tagcache_init(void)
4662 {
4663     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4664     memset(&current_tcmh, 0, sizeof(struct master_header));
4665     filenametag_fd = -1;
4666     write_lock = read_lock = 0;
4667
4668 #ifndef __PCTOOL__
4669     mutex_init(&command_queue_mutex);
4670     queue_init(&tagcache_queue, true);
4671     create_thread(tagcache_thread, tagcache_stack,
4672                   sizeof(tagcache_stack), 0, tagcache_thread_name
4673                   IF_PRIO(, PRIORITY_BACKGROUND)
4674                   IF_COP(, CPU));
4675 #else
4676     tc_stat.initialized = true;
4677     allocate_tempbuf();
4678     commit();
4679     free_tempbuf();
4680     tc_stat.ready = check_all_headers();
4681 #endif
4682 }
4683
4684 #ifdef __PCTOOL__
4685 void tagcache_reverse_scan(void)
4686 {
4687     logf("Checking for deleted files");
4688     check_deleted_files();
4689 }
4690 #endif
4691
4692 bool tagcache_is_initialized(void)
4693 {
4694     return tc_stat.initialized;
4695 }
4696 bool tagcache_is_usable(void)
4697 {
4698     return tc_stat.initialized && tc_stat.ready;
4699 }
4700 int tagcache_get_commit_step(void)
4701 {
4702     return tc_stat.commit_step;
4703 }
4704 int tagcache_get_max_commit_step(void)
4705 {
4706     return (int)(SORTED_TAGS_COUNT)+1;
4707 }
4708