apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #ifdef APPLICATION
  64 #include <unistd.h> /* readlink() */
  65 #include <limits.h> /* PATH_MAX */
  66 #endif
  67 #include "config.h"
  68 #include "ata_idle_notify.h"
  69 #include "thread.h"
  70 #include "kernel.h"
  71 #include "system.h"
  72 #include "logf.h"
  73 #include "string-extra.h"
  74 #include "usb.h"
  75 #include "metadata.h"
  76 #include "tagcache.h"
  77 #include "buffer.h"
  78 #include "crc32.h"
  79 #include "misc.h"
  80 #include "settings.h"
  81 #include "dir.h"
  82 #include "filefuncs.h"
  83 #include "structec.h"
  84 #include "debug.h"
  85
  86 #ifndef __PCTOOL__
  87 #include "lang.h"
  88 #include "eeprom_settings.h"
  89 #endif
  90
  91 #ifdef __PCTOOL__
  92 #define yield() do { } while(0)
  93 #define sim_sleep(timeout) do { } while(0)
  94 #define do_timed_yield() do { } while(0)
  95 #endif
  96
  97 #ifndef __PCTOOL__
  98 /* Tag Cache thread. */
  99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
 100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
 101 static const char tagcache_thread_name[] = "tagcache";
 102 #endif
 103
 104 /* Previous path when scanning directory tree recursively. */
 105 static char curpath[TAG_MAXLEN+32];
 106
 107 /* Used when removing duplicates. */
 108 static char *tempbuf;     /* Allocated when needed. */
 109 static long tempbufidx;   /* Current location in buffer. */
 110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 111 static long tempbuf_left; /* Buffer space left. */
 112 static long tempbuf_pos;
 113
 114 #define SORTED_TAGS_COUNT 8
 115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 118     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 119 /* Tags we want to get sorted (loaded to the tempbuf). */
 120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 123
 124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 126     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 127     (1LU << tag_albumartist) | (1LU << tag_grouping))
 128
 129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 130 static const char *tags_str[] = { "artist", "album", "genre", "title",
 131     "filename", "composer", "comment", "albumartist", "grouping", "year",
 132     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 133     "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
 134
 135 /* Status information of the tagcache. */
 136 static struct tagcache_stat tc_stat;
 137
 138 /* Queue commands. */
 139 enum tagcache_queue {
 140     Q_STOP_SCAN = 0,
 141     Q_START_SCAN,
 142     Q_IMPORT_CHANGELOG,
 143     Q_UPDATE,
 144     Q_REBUILD,
 145
 146     /* Internal tagcache command queue. */
 147     CMD_UPDATE_MASTER_HEADER,
 148     CMD_UPDATE_NUMERIC,
 149 };
 150
 151 struct tagcache_command_entry {
 152     int32_t command;
 153     int32_t idx_id;
 154     int32_t tag;
 155     int32_t data;
 156 };
 157
 158 #ifndef __PCTOOL__
 159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 160 static volatile int command_queue_widx = 0;
 161 static volatile int command_queue_ridx = 0;
 162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
 163 #endif
 164
 165 /* Tag database structures. */
 166
 167 /* Variable-length tag entry in tag files. */
 168 struct tagfile_entry {
 169     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 170     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 171     char tag_data[0];  /* Begin of the tag data */
 172 };
 173
 174 /* Fixed-size tag entry in master db index. */
 175 struct index_entry {
 176     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 177     int32_t flag;                /* Status flags */
 178 };
 179
 180 /* Header is the same in every file. */
 181 struct tagcache_header {
 182     int32_t magic;       /* Header version number */
 183     int32_t datasize;    /* Data size in bytes */
 184     int32_t entry_count; /* Number of entries in this file */
 185 };
 186
 187 struct master_header {
 188     struct tagcache_header tch;
 189     int32_t serial; /* Increasing counting number */
 190     int32_t commitid; /* Number of commits so far */
 191     int32_t dirty;
 192 };
 193
 194 /* For the endianess correction */
 195 static const char *tagfile_entry_ec   = "ll";
 196 /**
 197  Note: This should be (1 + TAG_COUNT) amount of l's.
 198  */
 199 static const char *index_entry_ec     = "llllllllllllllllllllll";
 200
 201 static const char *tagcache_header_ec = "lll";
 202 static const char *master_header_ec   = "llllll";
 203
 204 static struct master_header current_tcmh;
 205
 206 #ifdef HAVE_TC_RAMCACHE
 207 /* Header is created when loading database to ram. */
 208 struct ramcache_header {
 209     struct master_header h;      /* Header from the master index */
 210     struct index_entry *indices; /* Master index file content */
 211     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 212     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 213 };
 214
 215 # ifdef HAVE_EEPROM_SETTINGS
 216 struct statefile_header {
 217     struct ramcache_header *hdr;
 218     struct tagcache_stat tc_stat;
 219 };
 220 # endif
 221
 222 /* Pointer to allocated ramcache_header */
 223 static struct ramcache_header *hdr;
 224 #endif
 225
 226 /**
 227  * Full tag entries stored in a temporary file waiting
 228  * for commit to the cache. */
 229 struct temp_file_entry {
 230     long tag_offset[TAG_COUNT];
 231     short tag_length[TAG_COUNT];
 232     long flag;
 233
 234     long data_length;
 235 };
 236
 237 struct tempbuf_id_list {
 238     long id;
 239     struct tempbuf_id_list *next;
 240 };
 241
 242 struct tempbuf_searchidx {
 243     long idx_id;
 244     char *str;
 245     int seek;
 246     struct tempbuf_id_list idlist;
 247 };
 248
 249 /* Lookup buffer for fixing messed up index while after sorting. */
 250 static long commit_entry_count;
 251 static long lookup_buffer_depth;
 252 static struct tempbuf_searchidx **lookup;
 253
 254 /* Used when building the temporary file. */
 255 static int cachefd = -1, filenametag_fd;
 256 static int total_entry_count = 0;
 257 static int data_size = 0;
 258 static int processed_dir_count;
 259
 260 /* Thread safe locking */
 261 static volatile int write_lock;
 262 static volatile int read_lock;
 263
 264 static bool delete_entry(long idx_id);
 265
 266 const char* tagcache_tag_to_str(int tag)
 267 {
 268     return tags_str[tag];
 269 }
 270
 271 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 272 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 283 {
 284     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 285 }
 286
 287 #ifdef HAVE_DIRCACHE
 288 /**
 289  * Returns true if specified flag is still present, i.e., dircache
 290  * has not been reloaded.
 291  */
 292 static bool is_dircache_intact(void)
 293 {
 294     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 295 }
 296 #endif
 297
 298 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 299 {
 300     int fd;
 301     char buf[MAX_PATH];
 302     int rc;
 303
 304     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 305         return -1;
 306
 307     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 308
 309     fd = open(buf, write ? O_RDWR : O_RDONLY);
 310     if (fd < 0)
 311     {
 312         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 313         tc_stat.ready = false;
 314         return fd;
 315     }
 316
 317     /* Check the header. */
 318     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 319     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 320     {
 321         logf("header error");
 322         tc_stat.ready = false;
 323         close(fd);
 324         return -2;
 325     }
 326
 327     return fd;
 328 }
 329
 330 static int open_master_fd(struct master_header *hdr, bool write)
 331 {
 332     int fd;
 333     int rc;
 334
 335     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 336     if (fd < 0)
 337     {
 338         logf("master file open failed for R/W");
 339         tc_stat.ready = false;
 340         return fd;
 341     }
 342
 343     tc_stat.econ = false;
 344
 345     /* Check the header. */
 346     rc = read(fd, hdr, sizeof(struct master_header));
 347     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 348     {
 349         /* Success. */
 350         return fd;
 351     }
 352
 353     /* Trying to read again, this time with endianess correction enabled. */
 354     lseek(fd, 0, SEEK_SET);
 355
 356     rc = ecread(fd, hdr, 1, master_header_ec, true);
 357     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 358     {
 359         logf("header error");
 360         tc_stat.ready = false;
 361         close(fd);
 362         return -2;
 363     }
 364
 365     tc_stat.econ = true;
 366
 367     return fd;
 368 }
 369
 370 #ifndef __PCTOOL__
 371 static bool do_timed_yield(void)
 372 {
 373     /* Sorting can lock up for quite a while, so yield occasionally */
 374     static long wakeup_tick = 0;
 375     if (TIME_AFTER(current_tick, wakeup_tick))
 376     {
 377         wakeup_tick = current_tick + (HZ/4);
 378         yield();
 379         return true;
 380     }
 381     return false;
 382 }
 383 #endif
 384
 385 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 386 static long find_entry_ram(const char *filename,
 387                            const struct dircache_entry *dc)
 388 {
 389     static long last_pos = 0;
 390     int i;
 391
 392     /* Check if tagcache is loaded into ram. */
 393     if (!tc_stat.ramcache)
 394         return -1;
 395
 396     if (dc == NULL)
 397         dc = dircache_get_entry_ptr(filename);
 398
 399     if (dc == NULL)
 400     {
 401         logf("tagcache: file not found.");
 402         return -1;
 403     }
 404
 405     try_again:
 406
 407     if (last_pos > 0)
 408         i = last_pos;
 409     else
 410         i = 0;
 411
 412     for (; i < hdr->h.tch.entry_count; i++)
 413     {
 414         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 415         {
 416             last_pos = MAX(0, i - 3);
 417             return i;
 418         }
 419
 420         do_timed_yield();
 421     }
 422
 423     if (last_pos > 0)
 424     {
 425         last_pos = 0;
 426         goto try_again;
 427     }
 428
 429     return -1;
 430 }
 431 #endif
 432
 433 static long find_entry_disk(const char *filename_raw, bool localfd)
 434 {
 435     struct tagcache_header tch;
 436     static long last_pos = -1;
 437     long pos_history[POS_HISTORY_COUNT];
 438     long pos_history_idx = 0;
 439     bool found = false;
 440     struct tagfile_entry tfe;
 441     int fd;
 442     char buf[TAG_MAXLEN+32];
 443     int i;
 444     int pos = -1;
 445
 446     const char *filename = filename_raw;
 447 #ifdef APPLICATION
 448     char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
 449     if (realpath(filename, pathbuf) == pathbuf)
 450         filename = pathbuf;
 451 #endif
 452
 453     if (!tc_stat.ready)
 454         return -2;
 455
 456     fd = filenametag_fd;
 457     if (fd < 0 || localfd)
 458     {
 459         last_pos = -1;
 460         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 461             return -1;
 462     }
 463
 464     check_again:
 465
 466     if (last_pos > 0)
 467         lseek(fd, last_pos, SEEK_SET);
 468     else
 469         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 470
 471     while (true)
 472     {
 473         pos = lseek(fd, 0, SEEK_CUR);
 474         for (i = pos_history_idx-1; i >= 0; i--)
 475             pos_history[i+1] = pos_history[i];
 476         pos_history[0] = pos;
 477
 478         if (ecread_tagfile_entry(fd, &tfe)
 479             != sizeof(struct tagfile_entry))
 480         {
 481             break ;
 482         }
 483
 484         if (tfe.tag_length >= (long)sizeof(buf))
 485         {
 486             logf("too long tag #1");
 487             close(fd);
 488             if (!localfd)
 489                 filenametag_fd = -1;
 490             last_pos = -1;
 491             return -2;
 492         }
 493
 494         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 495         {
 496             logf("read error #2");
 497             close(fd);
 498             if (!localfd)
 499                 filenametag_fd = -1;
 500             last_pos = -1;
 501             return -3;
 502         }
 503
 504         if (!strcasecmp(filename, buf))
 505         {
 506             last_pos = pos_history[pos_history_idx];
 507             found = true;
 508             break ;
 509         }
 510
 511         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 512             pos_history_idx++;
 513     }
 514
 515     /* Not found? */
 516     if (!found)
 517     {
 518         if (last_pos > 0)
 519         {
 520             last_pos = -1;
 521             logf("seek again");
 522             goto check_again;
 523         }
 524
 525         if (fd != filenametag_fd || localfd)
 526             close(fd);
 527         return -4;
 528     }
 529
 530     if (fd != filenametag_fd || localfd)
 531         close(fd);
 532
 533     return tfe.idx_id;
 534 }
 535
 536 static int find_index(const char *filename)
 537 {
 538     long idx_id = -1;
 539
 540 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 541     if (tc_stat.ramcache && is_dircache_intact())
 542         idx_id = find_entry_ram(filename, NULL);
 543 #endif
 544
 545     if (idx_id < 0)
 546         idx_id = find_entry_disk(filename, true);
 547
 548     return idx_id;
 549 }
 550
 551 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 552 {
 553     int idx_id;
 554
 555     if (!tc_stat.ready)
 556         return false;
 557
 558     idx_id = find_index(filename);
 559     if (idx_id < 0)
 560         return false;
 561
 562     if (!tagcache_search(tcs, tag_filename))
 563         return false;
 564
 565     tcs->entry_count = 0;
 566     tcs->idx_id = idx_id;
 567
 568     return true;
 569 }
 570
 571 static bool get_index(int masterfd, int idxid,
 572                       struct index_entry *idx, bool use_ram)
 573 {
 574     bool localfd = false;
 575
 576     if (idxid < 0)
 577     {
 578         logf("Incorrect idxid: %d", idxid);
 579         return false;
 580     }
 581
 582 #ifdef HAVE_TC_RAMCACHE
 583     if (tc_stat.ramcache && use_ram)
 584     {
 585         if (hdr->indices[idxid].flag & FLAG_DELETED)
 586             return false;
 587
 588 # ifdef HAVE_DIRCACHE
 589         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 590             || is_dircache_intact())
 591 #endif
 592         {
 593             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 594             return true;
 595         }
 596     }
 597 #else
 598     (void)use_ram;
 599 #endif
 600
 601     if (masterfd < 0)
 602     {
 603         struct master_header tcmh;
 604
 605         localfd = true;
 606         masterfd = open_master_fd(&tcmh, false);
 607         if (masterfd < 0)
 608             return false;
 609     }
 610
 611     lseek(masterfd, idxid * sizeof(struct index_entry)
 612           + sizeof(struct master_header), SEEK_SET);
 613     if (ecread_index_entry(masterfd, idx)
 614         != sizeof(struct index_entry))
 615     {
 616         logf("read error #3");
 617         if (localfd)
 618             close(masterfd);
 619
 620         return false;
 621     }
 622
 623     if (localfd)
 624         close(masterfd);
 625
 626     if (idx->flag & FLAG_DELETED)
 627         return false;
 628
 629     return true;
 630 }
 631
 632 #ifndef __PCTOOL__
 633
 634 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 635 {
 636     /* We need to exclude all memory only flags & tags when writing to disk. */
 637     if (idx->flag & FLAG_DIRCACHE)
 638     {
 639         logf("memory only flags!");
 640         return false;
 641     }
 642
 643 #ifdef HAVE_TC_RAMCACHE
 644     /* Only update numeric data. Writing the whole index to RAM by memcpy
 645      * destroys dircache pointers!
 646      */
 647     if (tc_stat.ramcache)
 648     {
 649         int tag;
 650         struct index_entry *idx_ram = &hdr->indices[idxid];
 651
 652         for (tag = 0; tag < TAG_COUNT; tag++)
 653         {
 654             if (TAGCACHE_IS_NUMERIC(tag))
 655             {
 656                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 657             }
 658         }
 659
 660         /* Don't touch the dircache flag or attributes. */
 661         idx_ram->flag = (idx->flag & 0x0000ffff)
 662             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 663     }
 664 #endif
 665
 666     lseek(masterfd, idxid * sizeof(struct index_entry)
 667           + sizeof(struct master_header), SEEK_SET);
 668     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 669     {
 670         logf("write error #3");
 671         logf("idxid: %d", idxid);
 672         return false;
 673     }
 674
 675     return true;
 676 }
 677
 678 #endif /* !__PCTOOL__ */
 679
 680 static bool open_files(struct tagcache_search *tcs, int tag)
 681 {
 682     if (tcs->idxfd[tag] < 0)
 683     {
 684         char fn[MAX_PATH];
 685
 686         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 687         tcs->idxfd[tag] = open(fn, O_RDONLY);
 688     }
 689
 690     if (tcs->idxfd[tag] < 0)
 691     {
 692         logf("File not open!");
 693         return false;
 694     }
 695
 696     return true;
 697 }
 698
 699 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 700                      int tag, char *buf, long size)
 701 {
 702     struct tagfile_entry tfe;
 703     long seek;
 704
 705     *buf = '\0';
 706
 707     if (TAGCACHE_IS_NUMERIC(tag))
 708         return false;
 709
 710     seek = idx->tag_seek[tag];
 711     if (seek < 0)
 712     {
 713         logf("Retrieve failed");
 714         return false;
 715     }
 716
 717 #ifdef HAVE_TC_RAMCACHE
 718     if (tcs->ramsearch)
 719     {
 720         struct tagfile_entry *ep;
 721
 722 # ifdef HAVE_DIRCACHE
 723         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 724             && is_dircache_intact())
 725         {
 726             dircache_copy_path((struct dircache_entry *)seek,
 727                                buf, size);
 728             return true;
 729         }
 730         else
 731 # endif
 732         if (tag != tag_filename)
 733         {
 734             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 735             strlcpy(buf, ep->tag_data, size);
 736
 737             return true;
 738         }
 739     }
 740 #endif
 741
 742     if (!open_files(tcs, tag))
 743         return false;
 744
 745     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 746     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 747         != sizeof(struct tagfile_entry))
 748     {
 749         logf("read error #5");
 750         return false;
 751     }
 752
 753     if (tfe.tag_length >= size)
 754     {
 755         logf("too small buffer");
 756         return false;
 757     }
 758
 759     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 760         tfe.tag_length)
 761     {
 762         logf("read error #6");
 763         return false;
 764     }
 765
 766     buf[tfe.tag_length] = '\0';
 767
 768     return true;
 769 }
 770
 771 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 772
 773 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 774 {
 775 #ifndef __PCTOOL__
 776     if (! COMMAND_QUEUE_IS_EMPTY)
 777     {
 778         /* Attempt to find tag data through store-to-load forwarding in
 779            command queue */
 780         long result = -1;
 781
 782         mutex_lock(&command_queue_mutex);
 783
 784         int ridx = command_queue_widx;
 785
 786         while (ridx != command_queue_ridx)
 787         {
 788             if (--ridx < 0)
 789                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 790
 791             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 792                 && command_queue[ridx].idx_id == idx_id
 793                 && command_queue[ridx].tag == tag)
 794             {
 795                 result = command_queue[ridx].data;
 796                 break;
 797             }
 798         }
 799
 800         mutex_unlock(&command_queue_mutex);
 801
 802         if (result >= 0)
 803         {
 804             logf("read_numeric_tag: "
 805                  "Recovered tag %d value %lX from write queue",
 806                  tag, result);
 807             return result;
 808         }
 809     }
 810 #endif
 811
 812     return idx->tag_seek[tag];
 813 }
 814
 815
 816 static long check_virtual_tags(int tag, int idx_id,
 817                                const struct index_entry *idx)
 818 {
 819     long data = 0;
 820
 821     switch (tag)
 822     {
 823         case tag_virt_length_sec:
 824             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 825             break;
 826
 827         case tag_virt_length_min:
 828             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 829             break;
 830
 831         case tag_virt_playtime_sec:
 832             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 833             break;
 834
 835         case tag_virt_playtime_min:
 836             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 837             break;
 838
 839         case tag_virt_autoscore:
 840             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 841                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 842             {
 843                 data = 0;
 844             }
 845             else
 846             {
 847                 /* A straight calculus gives:
 848                      autoscore = 100 * playtime / length / playcout (1)
 849                    Now, consider the euclidian division of playtime by length:
 850                      playtime = alpha * length + beta
 851                    With:
 852                      0 <= beta < length
 853                    Now, (1) becomes:
 854                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 855                    Both terms should be small enough to avoid any overflow
 856                 */
 857                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 858                               / read_numeric_tag(tag_length, idx_id, idx))
 859                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 860                                  % read_numeric_tag(tag_length, idx_id, idx)))
 861                          / read_numeric_tag(tag_length, idx_id, idx);
 862                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 863             }
 864             break;
 865
 866         /* How many commits before the file has been added to the DB. */
 867         case tag_virt_entryage:
 868             data = current_tcmh.commitid
 869                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 870             break;
 871
 872         default:
 873             data = read_numeric_tag(tag, idx_id, idx);
 874     }
 875
 876     return data;
 877 }
 878
 879 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 880 {
 881     struct index_entry idx;
 882
 883     if (!tc_stat.ready)
 884         return false;
 885
 886     if (!TAGCACHE_IS_NUMERIC(tag))
 887         return -1;
 888
 889     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 890         return -2;
 891
 892     return check_virtual_tags(tag, tcs->idx_id, &idx);
 893 }
 894
 895 inline static bool str_ends_with(const char *str1, const char *str2)
 896 {
 897     int str_len = strlen(str1);
 898     int clause_len = strlen(str2);
 899
 900     if (clause_len > str_len)
 901         return false;
 902
 903     return !strcasecmp(&str1[str_len - clause_len], str2);
 904 }
 905
 906 inline static bool str_oneof(const char *str, const char *list)
 907 {
 908     const char *sep;
 909     int l, len = strlen(str);
 910
 911     while (*list)
 912     {
 913         sep = strchr(list, '|');
 914         l = sep ? (long)sep - (long)list : (int)strlen(list);
 915         if ((l==len) && !strncasecmp(str, list, len))
 916             return true;
 917         list += sep ? l + 1 : l;
 918     }
 919
 920     return false;
 921 }
 922
 923 static bool check_against_clause(long numeric, const char *str,
 924                                  const struct tagcache_search_clause *clause)
 925 {
 926     if (clause->numeric)
 927     {
 928         switch (clause->type)
 929         {
 930             case clause_is:
 931                 return numeric == clause->numeric_data;
 932             case clause_is_not:
 933                 return numeric != clause->numeric_data;
 934             case clause_gt:
 935                 return numeric > clause->numeric_data;
 936             case clause_gteq:
 937                 return numeric >= clause->numeric_data;
 938             case clause_lt:
 939                 return numeric < clause->numeric_data;
 940             case clause_lteq:
 941                 return numeric <= clause->numeric_data;
 942             default:
 943                 logf("Incorrect numeric tag: %d", clause->type);
 944         }
 945     }
 946     else
 947     {
 948         switch (clause->type)
 949         {
 950             case clause_is:
 951                 return !strcasecmp(clause->str, str);
 952             case clause_is_not:
 953                 return strcasecmp(clause->str, str);
 954             case clause_gt:
 955                 return 0>strcasecmp(clause->str, str);
 956             case clause_gteq:
 957                 return 0>=strcasecmp(clause->str, str);
 958             case clause_lt:
 959                 return 0<strcasecmp(clause->str, str);
 960             case clause_lteq:
 961                 return 0<=strcasecmp(clause->str, str);
 962             case clause_contains:
 963                 return (strcasestr(str, clause->str) != NULL);
 964             case clause_not_contains:
 965                 return (strcasestr(str, clause->str) == NULL);
 966             case clause_begins_with:
 967                 return (strcasestr(str, clause->str) == str);
 968             case clause_not_begins_with:
 969                 return (strcasestr(str, clause->str) != str);
 970             case clause_ends_with:
 971                 return str_ends_with(str, clause->str);
 972             case clause_not_ends_with:
 973                 return !str_ends_with(str, clause->str);
 974             case clause_oneof:
 975                 return str_oneof(str, clause->str);
 976
 977             default:
 978                 logf("Incorrect tag: %d", clause->type);
 979         }
 980     }
 981
 982     return false;
 983 }
 984
 985 static bool check_clauses(struct tagcache_search *tcs,
 986                           struct index_entry *idx,
 987                           struct tagcache_search_clause **clause, int count)
 988 {
 989     int i;
 990
 991 #ifdef HAVE_TC_RAMCACHE
 992     if (tcs->ramsearch)
 993     {
 994         /* Go through all conditional clauses. */
 995         for (i = 0; i < count; i++)
 996         {
 997             struct tagfile_entry *tfe;
 998             int seek;
 999             char buf[256];
1000             char *str = NULL;
1001
1002             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1003
1004             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1005             {
1006                 if (clause[i]->tag == tag_filename)
1007                 {
1008                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1009                     str = buf;
1010                 }
1011                 else
1012                 {
1013                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1014                     str = tfe->tag_data;
1015                 }
1016             }
1017
1018             if (!check_against_clause(seek, str, clause[i]))
1019                 return false;
1020         }
1021     }
1022     else
1023 #endif
1024     {
1025         /* Check for conditions. */
1026         for (i = 0; i < count; i++)
1027         {
1028             struct tagfile_entry tfe;
1029             int seek;
1030             char str[256];
1031
1032             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1033
1034             memset(str, 0, sizeof str);
1035             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1036             {
1037                 int fd = tcs->idxfd[clause[i]->tag];
1038                 lseek(fd, seek, SEEK_SET);
1039                 ecread_tagfile_entry(fd, &tfe);
1040                 if (tfe.tag_length >= (int)sizeof(str))
1041                 {
1042                     logf("Too long tag read!");
1043                     break ;
1044                 }
1045
1046                 read(fd, str, tfe.tag_length);
1047
1048                 /* Check if entry has been deleted. */
1049                 if (str[0] == '\0')
1050                     break;
1051             }
1052
1053             if (!check_against_clause(seek, str, clause[i]))
1054                 return false;
1055         }
1056     }
1057
1058     return true;
1059 }
1060
1061 bool tagcache_check_clauses(struct tagcache_search *tcs,
1062                             struct tagcache_search_clause **clause, int count)
1063 {
1064     struct index_entry idx;
1065
1066     if (count == 0)
1067         return true;
1068
1069     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1070         return false;
1071
1072     return check_clauses(tcs, &idx, clause, count);
1073 }
1074
1075 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1076 {
1077     int i;
1078
1079     /* If uniq buffer is not defined we must return true for search to work. */
1080     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1081         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1082     {
1083         return true;
1084     }
1085
1086     for (i = 0; i < tcs->unique_list_count; i++)
1087     {
1088         /* Return false if entry is found. */
1089         if (tcs->unique_list[i] == id)
1090             return false;
1091     }
1092
1093     if (tcs->unique_list_count < tcs->unique_list_capacity)
1094     {
1095         tcs->unique_list[i] = id;
1096         tcs->unique_list_count++;
1097     }
1098
1099     return true;
1100 }
1101
1102 static bool build_lookup_list(struct tagcache_search *tcs)
1103 {
1104     struct index_entry entry;
1105     int i, j;
1106
1107     tcs->seek_list_count = 0;
1108
1109 #ifdef HAVE_TC_RAMCACHE
1110     if (tcs->ramsearch
1111 # ifdef HAVE_DIRCACHE
1112         && (tcs->type != tag_filename || is_dircache_intact())
1113 # endif
1114         )
1115     {
1116         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1117         {
1118             struct tagcache_seeklist_entry *seeklist;
1119             struct index_entry *idx = &hdr->indices[i];
1120             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1121                 break ;
1122
1123             /* Skip deleted files. */
1124             if (idx->flag & FLAG_DELETED)
1125                 continue;
1126
1127             /* Go through all filters.. */
1128             for (j = 0; j < tcs->filter_count; j++)
1129             {
1130                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1131                 {
1132                     break ;
1133                 }
1134             }
1135
1136             if (j < tcs->filter_count)
1137                 continue ;
1138
1139             /* Check for conditions. */
1140             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1141                 continue;
1142
1143             /* Add to the seek list if not already in uniq buffer. */
1144             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1145                 continue;
1146
1147             /* Lets add it. */
1148             seeklist = &tcs->seeklist[tcs->seek_list_count];
1149             seeklist->seek = idx->tag_seek[tcs->type];
1150             seeklist->flag = idx->flag;
1151             seeklist->idx_id = i;
1152             tcs->seek_list_count++;
1153         }
1154
1155         tcs->seek_pos = i;
1156
1157         return tcs->seek_list_count > 0;
1158     }
1159 #endif
1160
1161     if (tcs->masterfd < 0)
1162     {
1163         struct master_header tcmh;
1164         tcs->masterfd = open_master_fd(&tcmh, false);
1165     }
1166
1167     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1168             sizeof(struct master_header), SEEK_SET);
1169
1170     while (ecread_index_entry(tcs->masterfd, &entry)
1171            == sizeof(struct index_entry))
1172     {
1173         struct tagcache_seeklist_entry *seeklist;
1174
1175         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1176             break ;
1177
1178         i = tcs->seek_pos;
1179         tcs->seek_pos++;
1180
1181         /* Check if entry has been deleted. */
1182         if (entry.flag & FLAG_DELETED)
1183             continue;
1184
1185         /* Go through all filters.. */
1186         for (j = 0; j < tcs->filter_count; j++)
1187         {
1188             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1189                 break ;
1190         }
1191
1192         if (j < tcs->filter_count)
1193             continue ;
1194
1195         /* Check for conditions. */
1196         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1197             continue;
1198
1199         /* Add to the seek list if not already in uniq buffer. */
1200         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1201             continue;
1202
1203         /* Lets add it. */
1204         seeklist = &tcs->seeklist[tcs->seek_list_count];
1205         seeklist->seek = entry.tag_seek[tcs->type];
1206         seeklist->flag = entry.flag;
1207         seeklist->idx_id = i;
1208         tcs->seek_list_count++;
1209
1210         yield();
1211     }
1212
1213     return tcs->seek_list_count > 0;
1214 }
1215
1216
1217 static void remove_files(void)
1218 {
1219     int i;
1220     char buf[MAX_PATH];
1221
1222     tc_stat.ready = false;
1223     tc_stat.ramcache = false;
1224     tc_stat.econ = false;
1225     remove(TAGCACHE_FILE_MASTER);
1226     for (i = 0; i < TAG_COUNT; i++)
1227     {
1228         if (TAGCACHE_IS_NUMERIC(i))
1229             continue;
1230
1231         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1232         remove(buf);
1233     }
1234 }
1235
1236
1237 static bool check_all_headers(void)
1238 {
1239     struct master_header myhdr;
1240     struct tagcache_header tch;
1241     int tag;
1242     int fd;
1243
1244     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1245         return false;
1246
1247     close(fd);
1248     if (myhdr.dirty)
1249     {
1250         logf("tagcache is dirty!");
1251         return false;
1252     }
1253
1254     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1255
1256     for (tag = 0; tag < TAG_COUNT; tag++)
1257     {
1258         if (TAGCACHE_IS_NUMERIC(tag))
1259             continue;
1260
1261         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1262             return false;
1263
1264         close(fd);
1265     }
1266
1267     return true;
1268 }
1269
1270 bool tagcache_is_busy(void)
1271 {
1272     return read_lock || write_lock;
1273 }
1274
1275 bool tagcache_search(struct tagcache_search *tcs, int tag)
1276 {
1277     struct tagcache_header tag_hdr;
1278     struct master_header   master_hdr;
1279     int i;
1280
1281     while (read_lock)
1282         sleep(1);
1283
1284     memset(tcs, 0, sizeof(struct tagcache_search));
1285     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1286         return false;
1287
1288     tcs->position = sizeof(struct tagcache_header);
1289     tcs->type = tag;
1290     tcs->seek_pos = 0;
1291     tcs->list_position = 0;
1292     tcs->seek_list_count = 0;
1293     tcs->filter_count = 0;
1294     tcs->masterfd = -1;
1295
1296     for (i = 0; i < TAG_COUNT; i++)
1297         tcs->idxfd[i] = -1;
1298
1299 #ifndef HAVE_TC_RAMCACHE
1300     tcs->ramsearch = false;
1301 #else
1302     tcs->ramsearch = tc_stat.ramcache;
1303     if (tcs->ramsearch)
1304     {
1305         tcs->entry_count = hdr->entry_count[tcs->type];
1306     }
1307     else
1308 #endif
1309     {
1310         /* Always open as R/W so we can pass tcs to functions that modify data also
1311          * without failing. */
1312         tcs->masterfd = open_master_fd(&master_hdr, true);
1313         if (tcs->masterfd < 0)
1314             return false;
1315
1316         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1317         {
1318             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1319             if (tcs->idxfd[tcs->type] < 0)
1320                 return false;
1321
1322             tcs->entry_count = tag_hdr.entry_count;
1323         }
1324         else
1325         {
1326             tcs->entry_count = master_hdr.tch.entry_count;
1327         }
1328     }
1329
1330     tcs->valid = true;
1331     tcs->initialized = true;
1332     write_lock++;
1333
1334     return true;
1335 }
1336
1337 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1338                                  void *buffer, long length)
1339 {
1340     tcs->unique_list = (unsigned long *)buffer;
1341     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1342     tcs->unique_list_count = 0;
1343 }
1344
1345 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1346                                 int tag, int seek)
1347 {
1348     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1349         return false;
1350
1351     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1352         return false;
1353
1354     tcs->filter_tag[tcs->filter_count] = tag;
1355     tcs->filter_seek[tcs->filter_count] = seek;
1356     tcs->filter_count++;
1357
1358     return true;
1359 }
1360
1361 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1362                                 struct tagcache_search_clause *clause)
1363 {
1364     int i;
1365
1366     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1367     {
1368         logf("Too many clauses");
1369         return false;
1370     }
1371
1372     /* Check if there is already a similar filter in present (filters are
1373      * much faster than clauses).
1374      */
1375     for (i = 0; i < tcs->filter_count; i++)
1376     {
1377         if (tcs->filter_tag[i] == clause->tag)
1378             return true;
1379     }
1380
1381     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1382     {
1383         char buf[MAX_PATH];
1384
1385         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1386         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1387     }
1388
1389     tcs->clause[tcs->clause_count] = clause;
1390     tcs->clause_count++;
1391
1392     return true;
1393 }
1394
1395 static bool get_next(struct tagcache_search *tcs)
1396 {
1397     static char buf[TAG_MAXLEN+32];
1398     struct tagfile_entry entry;
1399     long flag = 0;
1400
1401     if (!tcs->valid || !tc_stat.ready)
1402         return false;
1403
1404     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1405 #ifdef HAVE_TC_RAMCACHE
1406         && !tcs->ramsearch
1407 #endif
1408         )
1409         return false;
1410
1411     /* Relative fetch. */
1412     if (tcs->filter_count > 0 || tcs->clause_count > 0
1413         || TAGCACHE_IS_NUMERIC(tcs->type)
1414 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1415         /* We need to retrieve flag status for dircache. */
1416         || (tcs->ramsearch && tcs->type == tag_filename)
1417 #endif
1418         )
1419     {
1420         struct tagcache_seeklist_entry *seeklist;
1421
1422         /* Check for end of list. */
1423         if (tcs->list_position == tcs->seek_list_count)
1424         {
1425             tcs->list_position = 0;
1426
1427             /* Try to fetch more. */
1428             if (!build_lookup_list(tcs))
1429             {
1430                 tcs->valid = false;
1431                 return false;
1432             }
1433         }
1434
1435         seeklist = &tcs->seeklist[tcs->list_position];
1436         flag = seeklist->flag;
1437         tcs->position = seeklist->seek;
1438         tcs->idx_id = seeklist->idx_id;
1439         tcs->list_position++;
1440     }
1441     else
1442     {
1443         if (tcs->entry_count == 0)
1444         {
1445             tcs->valid = false;
1446             return false;
1447         }
1448
1449         tcs->entry_count--;
1450     }
1451
1452     tcs->result_seek = tcs->position;
1453
1454     if (TAGCACHE_IS_NUMERIC(tcs->type))
1455     {
1456         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1457         tcs->result = buf;
1458         tcs->result_len = strlen(buf) + 1;
1459         return true;
1460     }
1461
1462     /* Direct fetch. */
1463 #ifdef HAVE_TC_RAMCACHE
1464     if (tcs->ramsearch)
1465     {
1466
1467 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1468         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1469             && is_dircache_intact())
1470         {
1471             dircache_copy_path((struct dircache_entry *)tcs->position,
1472                                buf, sizeof buf);
1473             tcs->result = buf;
1474             tcs->result_len = strlen(buf) + 1;
1475             tcs->ramresult = false;
1476
1477             return true;
1478         }
1479         else
1480 #endif
1481         if (tcs->type != tag_filename)
1482         {
1483             struct tagfile_entry *ep;
1484
1485             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1486             tcs->result = ep->tag_data;
1487             tcs->result_len = strlen(tcs->result) + 1;
1488             tcs->idx_id = ep->idx_id;
1489             tcs->ramresult = true;
1490
1491             /* Increase position for the next run. This may get overwritten. */
1492             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1493
1494             return true;
1495         }
1496     }
1497 #endif
1498
1499     if (!open_files(tcs, tcs->type))
1500     {
1501         tcs->valid = false;
1502         return false;
1503     }
1504
1505     /* Seek stream to the correct position and continue to direct fetch. */
1506     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1507
1508     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1509     {
1510         logf("read error #5");
1511         tcs->valid = false;
1512         return false;
1513     }
1514
1515     if (entry.tag_length > (long)sizeof(buf))
1516     {
1517         tcs->valid = false;
1518         logf("too long tag #2");
1519         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1520         return false;
1521     }
1522
1523     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1524     {
1525         tcs->valid = false;
1526         logf("read error #4");
1527         return false;
1528     }
1529
1530     /**
1531      Update the position for the next read (this may be overridden
1532      if filters or clauses are being used).
1533      */
1534     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1535     tcs->result = buf;
1536     tcs->result_len = strlen(tcs->result) + 1;
1537     tcs->idx_id = entry.idx_id;
1538     tcs->ramresult = false;
1539
1540     return true;
1541 }
1542
1543 bool tagcache_get_next(struct tagcache_search *tcs)
1544 {
1545     while (get_next(tcs))
1546     {
1547         if (tcs->result_len > 1)
1548             return true;
1549     }
1550
1551     return false;
1552 }
1553
1554 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1555                        int tag, char *buf, long size)
1556 {
1557     struct index_entry idx;
1558
1559     *buf = '\0';
1560     if (!get_index(tcs->masterfd, idxid, &idx, true))
1561         return false;
1562
1563     return retrieve(tcs, &idx, tag, buf, size);
1564 }
1565
1566 static bool update_master_header(void)
1567 {
1568     struct master_header myhdr;
1569     int fd;
1570
1571     if (!tc_stat.ready)
1572         return false;
1573
1574     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1575         return false;
1576
1577     myhdr.serial = current_tcmh.serial;
1578     myhdr.commitid = current_tcmh.commitid;
1579     myhdr.dirty = current_tcmh.dirty;
1580
1581     /* Write it back */
1582     lseek(fd, 0, SEEK_SET);
1583     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1584     close(fd);
1585
1586 #ifdef HAVE_TC_RAMCACHE
1587     if (hdr)
1588     {
1589         hdr->h.serial = current_tcmh.serial;
1590         hdr->h.commitid = current_tcmh.commitid;
1591         hdr->h.dirty = current_tcmh.dirty;
1592     }
1593 #endif
1594
1595     return true;
1596 }
1597
1598 #if 0
1599
1600 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1601 {
1602     struct tagentry *entry;
1603
1604     if (tcs->type != tag_title)
1605         return ;
1606
1607     /* We will need reserve buffer for this. */
1608     if (tcs->ramcache)
1609     {
1610         struct tagfile_entry *ep;
1611
1612         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1613         tcs->seek_list[tcs->seek_list_count];
1614     }
1615
1616     entry = find_entry_ram();
1617
1618 }
1619 #endif
1620
1621 void tagcache_search_finish(struct tagcache_search *tcs)
1622 {
1623     int i;
1624
1625     if (!tcs->initialized)
1626         return;
1627
1628     if (tcs->masterfd >= 0)
1629     {
1630         close(tcs->masterfd);
1631         tcs->masterfd = -1;
1632     }
1633
1634     for (i = 0; i < TAG_COUNT; i++)
1635     {
1636         if (tcs->idxfd[i] >= 0)
1637         {
1638             close(tcs->idxfd[i]);
1639             tcs->idxfd[i] = -1;
1640         }
1641     }
1642
1643     tcs->ramsearch = false;
1644     tcs->valid = false;
1645     tcs->initialized = 0;
1646     if (write_lock > 0)
1647         write_lock--;
1648 }
1649
1650 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1651 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1652 {
1653     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1654 }
1655
1656 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1657 {
1658     return check_virtual_tags(tag, idx_id, entry);
1659 }
1660
1661 static char* get_tag_string(const struct index_entry *entry, int tag)
1662 {
1663     char* s = get_tag(entry, tag)->tag_data;
1664     return strcmp(s, UNTAGGED) ? s : NULL;
1665 }
1666
1667 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1668 {
1669     struct index_entry *entry;
1670     int idx_id;
1671
1672     if (!tc_stat.ready || !tc_stat.ramcache)
1673         return false;
1674
1675     /* Find the corresponding entry in tagcache. */
1676     idx_id = find_entry_ram(filename, NULL);
1677     if (idx_id < 0)
1678         return false;
1679
1680     entry = &hdr->indices[idx_id];
1681
1682     memset(id3, 0, sizeof(struct mp3entry));
1683
1684     id3->title        = get_tag_string(entry, tag_title);
1685     id3->artist       = get_tag_string(entry, tag_artist);
1686     id3->album        = get_tag_string(entry, tag_album);
1687     id3->genre_string = get_tag_string(entry, tag_genre);
1688     id3->composer     = get_tag_string(entry, tag_composer);
1689     id3->comment      = get_tag_string(entry, tag_comment);
1690     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1691     id3->grouping     = get_tag_string(entry, tag_grouping);
1692
1693     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1694     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1695     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1696     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1697     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1698     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1699
1700     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1701     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1702     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1703     if (id3->bitrate == 0)
1704         id3->bitrate = 1;
1705
1706 #if CONFIG_CODEC == SWCODEC
1707     if (global_settings.autoresume_enable)
1708     {
1709         id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1710         logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1711              id3->title, id3->offset);
1712     }
1713 #endif
1714
1715     return true;
1716 }
1717 #endif
1718
1719 static inline void write_item(const char *item)
1720 {
1721     int len = strlen(item) + 1;
1722
1723     data_size += len;
1724     write(cachefd, item, len);
1725 }
1726
1727 static int check_if_empty(char **tag)
1728 {
1729     int length;
1730
1731     if (*tag == NULL || **tag == '\0')
1732     {
1733         *tag = UNTAGGED;
1734         return sizeof(UNTAGGED); /* Tag length */
1735     }
1736
1737     length = strlen(*tag);
1738     if (length > TAG_MAXLEN)
1739     {
1740         logf("over length tag: %s", *tag);
1741         length = TAG_MAXLEN;
1742         (*tag)[length] = '\0';
1743     }
1744
1745     return length + 1;
1746 }
1747
1748 #define ADD_TAG(entry,tag,data) \
1749     /* Adding tag */ \
1750     entry.tag_offset[tag] = offset; \
1751     entry.tag_length[tag] = check_if_empty(data); \
1752     offset += entry.tag_length[tag]
1753 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1754  * idea, as it uses lots of stack and is called from a recursive function
1755  * (check_dir).
1756  */
1757 static void __attribute__ ((noinline)) add_tagcache(char *path,
1758                                                     unsigned long mtime
1759 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1760                                                     ,const struct dircache_entry *dc
1761 #endif
1762                                                    )
1763 {
1764     struct mp3entry id3;
1765     struct temp_file_entry entry;
1766     bool ret;
1767     int fd;
1768     int idx_id = -1;
1769     char tracknumfix[3];
1770     int offset = 0;
1771     int path_length = strlen(path);
1772     bool has_albumartist;
1773     bool has_grouping;
1774
1775 #ifdef SIMULATOR
1776     /* Crude logging for the sim - to aid in debugging */
1777     int logfd = open(ROCKBOX_DIR "/database.log",
1778                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1779     if (logfd >= 0) {
1780         write(logfd, path, strlen(path));
1781         write(logfd, "\n", 1);
1782         close(logfd);
1783     }
1784 #endif
1785
1786     if (cachefd < 0)
1787         return ;
1788
1789     /* Check for overlength file path. */
1790     if (path_length > TAG_MAXLEN)
1791     {
1792         /* Path can't be shortened. */
1793         logf("Too long path: %s", path);
1794         return ;
1795     }
1796
1797     /* Check if the file is supported. */
1798     if (probe_file_format(path) == AFMT_UNKNOWN)
1799         return ;
1800
1801     /* Check if the file is already cached. */
1802 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1803     if (tc_stat.ramcache && is_dircache_intact())
1804     {
1805         idx_id = find_entry_ram(path, dc);
1806     }
1807 #endif
1808
1809     /* Be sure the entry doesn't exist. */
1810     if (filenametag_fd >= 0 && idx_id < 0)
1811         idx_id = find_entry_disk(path, false);
1812
1813     /* Check if file has been modified. */
1814     if (idx_id >= 0)
1815     {
1816         struct index_entry idx;
1817
1818         /* TODO: Mark that the index exists (for fast reverse scan) */
1819         //found_idx[idx_id/8] |= idx_id%8;
1820
1821         if (!get_index(-1, idx_id, &idx, true))
1822         {
1823             logf("failed to retrieve index entry");
1824             return ;
1825         }
1826
1827         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1828         {
1829             /* No changes to file. */
1830             return ;
1831         }
1832
1833         /* Metadata might have been changed. Delete the entry. */
1834         logf("Re-adding: %s", path);
1835         if (!delete_entry(idx_id))
1836         {
1837             logf("delete_entry failed: %d", idx_id);
1838             return ;
1839         }
1840     }
1841
1842     fd = open(path, O_RDONLY);
1843     if (fd < 0)
1844     {
1845         logf("open fail: %s", path);
1846         return ;
1847     }
1848
1849     memset(&id3, 0, sizeof(struct mp3entry));
1850     memset(&entry, 0, sizeof(struct temp_file_entry));
1851     memset(&tracknumfix, 0, sizeof(tracknumfix));
1852     ret = get_metadata(&id3, fd, path);
1853     close(fd);
1854
1855     if (!ret)
1856         return ;
1857
1858     logf("-> %s", path);
1859
1860     /* Generate track number if missing. */
1861     if (id3.tracknum <= 0)
1862     {
1863         const char *p = strrchr(path, '.');
1864
1865         if (p == NULL)
1866             p = &path[strlen(path)-1];
1867
1868         while (*p != '/')
1869         {
1870             if (isdigit(*p) && isdigit(*(p-1)))
1871             {
1872                 tracknumfix[1] = *p--;
1873                 tracknumfix[0] = *p;
1874                 break;
1875             }
1876             p--;
1877         }
1878
1879         if (tracknumfix[0] != '\0')
1880         {
1881             id3.tracknum = atoi(tracknumfix);
1882             /* Set a flag to indicate track number has been generated. */
1883             entry.flag |= FLAG_TRKNUMGEN;
1884         }
1885         else
1886         {
1887             /* Unable to generate track number. */
1888             id3.tracknum = -1;
1889         }
1890     }
1891
1892     /* Numeric tags */
1893     entry.tag_offset[tag_year] = id3.year;
1894     entry.tag_offset[tag_discnumber] = id3.discnum;
1895     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1896     entry.tag_offset[tag_length] = id3.length;
1897     entry.tag_offset[tag_bitrate] = id3.bitrate;
1898     entry.tag_offset[tag_mtime] = mtime;
1899
1900     /* String tags. */
1901     has_albumartist = id3.albumartist != NULL
1902         && strlen(id3.albumartist) > 0;
1903     has_grouping = id3.grouping != NULL
1904         && strlen(id3.grouping) > 0;
1905
1906     ADD_TAG(entry, tag_filename, &path);
1907     ADD_TAG(entry, tag_title, &id3.title);
1908     ADD_TAG(entry, tag_artist, &id3.artist);
1909     ADD_TAG(entry, tag_album, &id3.album);
1910     ADD_TAG(entry, tag_genre, &id3.genre_string);
1911     ADD_TAG(entry, tag_composer, &id3.composer);
1912     ADD_TAG(entry, tag_comment, &id3.comment);
1913     if (has_albumartist)
1914     {
1915         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1916     }
1917     else
1918     {
1919         ADD_TAG(entry, tag_albumartist, &id3.artist);
1920     }
1921     if (has_grouping)
1922     {
1923         ADD_TAG(entry, tag_grouping, &id3.grouping);
1924     }
1925     else
1926     {
1927         ADD_TAG(entry, tag_grouping, &id3.title);
1928     }
1929     entry.data_length = offset;
1930
1931     /* Write the header */
1932     write(cachefd, &entry, sizeof(struct temp_file_entry));
1933
1934     /* And tags also... Correct order is critical */
1935     write_item(path);
1936     write_item(id3.title);
1937     write_item(id3.artist);
1938     write_item(id3.album);
1939     write_item(id3.genre_string);
1940     write_item(id3.composer);
1941     write_item(id3.comment);
1942     if (has_albumartist)
1943     {
1944         write_item(id3.albumartist);
1945     }
1946     else
1947     {
1948         write_item(id3.artist);
1949     }
1950     if (has_grouping)
1951     {
1952         write_item(id3.grouping);
1953     }
1954     else
1955     {
1956         write_item(id3.title);
1957     }
1958     total_entry_count++;
1959 }
1960
1961 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1962 {
1963     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1964     int len = strlen(str)+1;
1965     int i;
1966     unsigned crc32;
1967     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1968     char buf[TAG_MAXLEN+32];
1969
1970     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1971         buf[i] = tolower(str[i]);
1972     buf[i] = '\0';
1973
1974     crc32 = crc_32(buf, i, 0xffffffff);
1975
1976     if (unique)
1977     {
1978         /* Check if the crc does not exist -> entry does not exist for sure. */
1979         for (i = 0; i < tempbufidx; i++)
1980         {
1981             if (crcbuf[-i] != crc32)
1982                 continue;
1983
1984             if (!strcasecmp(str, index[i].str))
1985             {
1986                 if (id < 0 || id >= lookup_buffer_depth)
1987                 {
1988                     logf("lookup buf overf.: %d", id);
1989                     return false;
1990                 }
1991
1992                 lookup[id] = &index[i];
1993                 return true;
1994             }
1995         }
1996     }
1997
1998     /* Insert to CRC buffer. */
1999     crcbuf[-tempbufidx] = crc32;
2000     tempbuf_left -= 4;
2001
2002     /* Insert it to the buffer. */
2003     tempbuf_left -= len;
2004     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
2005         return false;
2006
2007     if (id >= lookup_buffer_depth)
2008     {
2009         logf("lookup buf overf. #2: %d", id);
2010         return false;
2011     }
2012
2013     if (id >= 0)
2014     {
2015         lookup[id] = &index[tempbufidx];
2016         index[tempbufidx].idlist.id = id;
2017     }
2018     else
2019         index[tempbufidx].idlist.id = -1;
2020
2021     index[tempbufidx].idlist.next = NULL;
2022     index[tempbufidx].idx_id = idx_id;
2023     index[tempbufidx].seek = -1;
2024     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2025     memcpy(index[tempbufidx].str, str, len);
2026     tempbuf_pos += len;
2027     tempbufidx++;
2028
2029     return true;
2030 }
2031
2032 static int compare(const void *p1, const void *p2)
2033 {
2034     do_timed_yield();
2035
2036     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2037     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2038
2039     if (strcmp(e1->str, UNTAGGED) == 0)
2040     {
2041         if (strcmp(e2->str, UNTAGGED) == 0)
2042             return 0;
2043         return -1;
2044     }
2045     else if (strcmp(e2->str, UNTAGGED) == 0)
2046         return 1;
2047
2048     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2049 }
2050
2051 static int tempbuf_sort(int fd)
2052 {
2053     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2054     struct tagfile_entry fe;
2055     int i;
2056     int length;
2057
2058     /* Generate reverse lookup entries. */
2059     for (i = 0; i < lookup_buffer_depth; i++)
2060     {
2061         struct tempbuf_id_list *idlist;
2062
2063         if (!lookup[i])
2064             continue;
2065
2066         if (lookup[i]->idlist.id == i)
2067             continue;
2068
2069         idlist = &lookup[i]->idlist;
2070         while (idlist->next != NULL)
2071             idlist = idlist->next;
2072
2073         tempbuf_left -= sizeof(struct tempbuf_id_list);
2074         if (tempbuf_left - 4 < 0)
2075             return -1;
2076
2077         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2078         if (tempbuf_pos & 0x03)
2079         {
2080             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2081             tempbuf_left -= 3;
2082             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2083         }
2084         tempbuf_pos += sizeof(struct tempbuf_id_list);
2085
2086         idlist = idlist->next;
2087         idlist->id = i;
2088         idlist->next = NULL;
2089
2090         do_timed_yield();
2091     }
2092
2093     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2094     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2095
2096     for (i = 0; i < tempbufidx; i++)
2097     {
2098         struct tempbuf_id_list *idlist = &index[i].idlist;
2099
2100         /* Fix the lookup list. */
2101         while (idlist != NULL)
2102         {
2103             if (idlist->id >= 0)
2104                 lookup[idlist->id] = &index[i];
2105             idlist = idlist->next;
2106         }
2107
2108         index[i].seek = lseek(fd, 0, SEEK_CUR);
2109         length = strlen(index[i].str) + 1;
2110         fe.tag_length = length;
2111         fe.idx_id = index[i].idx_id;
2112
2113         /* Check the chunk alignment. */
2114         if ((fe.tag_length + sizeof(struct tagfile_entry))
2115             % TAGFILE_ENTRY_CHUNK_LENGTH)
2116         {
2117             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2118                 ((fe.tag_length + sizeof(struct tagfile_entry))
2119                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2120         }
2121
2122 #ifdef TAGCACHE_STRICT_ALIGN
2123         /* Make sure the entry is long aligned. */
2124         if (index[i].seek & 0x03)
2125         {
2126             logf("tempbuf_sort: alignment error!");
2127             return -3;
2128         }
2129 #endif
2130
2131         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2132             sizeof(struct tagfile_entry))
2133         {
2134             logf("tempbuf_sort: write error #1");
2135             return -1;
2136         }
2137
2138         if (write(fd, index[i].str, length) != length)
2139         {
2140             logf("tempbuf_sort: write error #2");
2141             return -2;
2142         }
2143
2144         /* Write some padding. */
2145         if (fe.tag_length - length > 0)
2146             write(fd, "XXXXXXXX", fe.tag_length - length);
2147     }
2148
2149     return i;
2150 }
2151
2152 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2153 {
2154     if (id < 0 || id >= lookup_buffer_depth)
2155         return NULL;
2156
2157     return lookup[id];
2158 }
2159
2160
2161 inline static int tempbuf_find_location(int id)
2162 {
2163     struct tempbuf_searchidx *entry;
2164
2165     entry = tempbuf_locate(id);
2166     if (entry == NULL)
2167         return -1;
2168
2169     return entry->seek;
2170 }
2171
2172 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2173 {
2174     struct master_header tcmh;
2175     struct index_entry idx;
2176     int masterfd;
2177     int masterfd_pos;
2178     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2179     int max_entries;
2180     int entries_processed = 0;
2181     int i, j;
2182     char buf[TAG_MAXLEN];
2183
2184     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2185
2186     logf("Building numeric indices...");
2187     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2188
2189     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2190         return false;
2191
2192     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2193                          SEEK_CUR);
2194     if (masterfd_pos == filesize(masterfd))
2195     {
2196         logf("we can't append!");
2197         close(masterfd);
2198         return false;
2199     }
2200
2201     while (entries_processed < h->entry_count)
2202     {
2203         int count = MIN(h->entry_count - entries_processed, max_entries);
2204
2205         /* Read in as many entries as possible. */
2206         for (i = 0; i < count; i++)
2207         {
2208             struct temp_file_entry *tfe = &entrybuf[i];
2209             int datastart;
2210
2211             /* Read in numeric data. */
2212             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2213                 sizeof(struct temp_file_entry))
2214             {
2215                 logf("read fail #1");
2216                 close(masterfd);
2217                 return false;
2218             }
2219
2220             datastart = lseek(tmpfd, 0, SEEK_CUR);
2221
2222             /**
2223              * Read string data from the following tags:
2224              * - tag_filename
2225              * - tag_artist
2226              * - tag_album
2227              * - tag_title
2228              *
2229              * A crc32 hash is calculated from the read data
2230              * and stored back to the data offset field kept in memory.
2231              */
2232 #define tmpdb_read_string_tag(tag) \
2233     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2234     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2235     { \
2236         logf("read fail: buffer overflow"); \
2237         close(masterfd); \
2238         return false; \
2239     } \
2240     \
2241     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2242         tfe->tag_length[tag]) \
2243     { \
2244         logf("read fail #2"); \
2245         close(masterfd); \
2246         return false; \
2247     } \
2248     \
2249     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2250     lseek(tmpfd, datastart, SEEK_SET)
2251
2252             tmpdb_read_string_tag(tag_filename);
2253             tmpdb_read_string_tag(tag_artist);
2254             tmpdb_read_string_tag(tag_album);
2255             tmpdb_read_string_tag(tag_title);
2256
2257             /* Seek to the end of the string data. */
2258             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2259         }
2260
2261         /* Backup the master index position. */
2262         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2263         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2264
2265         /* Check if we can resurrect some deleted runtime statistics data. */
2266         for (i = 0; i < tcmh.tch.entry_count; i++)
2267         {
2268             /* Read the index entry. */
2269             if (ecread_index_entry(masterfd, &idx)
2270                 != sizeof(struct index_entry))
2271             {
2272                 logf("read fail #3");
2273                 close(masterfd);
2274                 return false;
2275             }
2276
2277             /**
2278              * Skip unless the entry is marked as being deleted
2279              * or the data has already been resurrected.
2280              */
2281             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2282                 continue;
2283
2284             /* Now try to match the entry. */
2285             /**
2286              * To succesfully match a song, the following conditions
2287              * must apply:
2288              *
2289              * For numeric fields: tag_length
2290              * - Full identical match is required
2291              *
2292              * If tag_filename matches, no further checking necessary.
2293              *
2294              * For string hashes: tag_artist, tag_album, tag_title
2295              * - Two of these must match
2296              */
2297             for (j = 0; j < count; j++)
2298             {
2299                 struct temp_file_entry *tfe = &entrybuf[j];
2300
2301                 /* Try to match numeric fields first. */
2302                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2303                     continue;
2304
2305                 /* Now it's time to do the hash matching. */
2306                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2307                 {
2308                     int match_count = 0;
2309
2310                     /* No filename match, check if we can match two other tags. */
2311 #define tmpdb_match(tag) \
2312     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2313         match_count++
2314
2315                     tmpdb_match(tag_artist);
2316                     tmpdb_match(tag_album);
2317                     tmpdb_match(tag_title);
2318
2319                     if (match_count < 2)
2320                     {
2321                         /* Still no match found, give up. */
2322                         continue;
2323                     }
2324                 }
2325
2326                 /* A match found, now copy & resurrect the statistical data. */
2327 #define tmpdb_copy_tag(tag) \
2328     tfe->tag_offset[tag] = idx.tag_seek[tag]
2329
2330                 tmpdb_copy_tag(tag_playcount);
2331                 tmpdb_copy_tag(tag_rating);
2332                 tmpdb_copy_tag(tag_playtime);
2333                 tmpdb_copy_tag(tag_lastplayed);
2334                 tmpdb_copy_tag(tag_commitid);
2335                 tmpdb_copy_tag(tag_lastoffset);
2336
2337                 /* Avoid processing this entry again. */
2338                 idx.flag |= FLAG_RESURRECTED;
2339
2340                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2341                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2342                 {
2343                     logf("masterfd writeback fail #1");
2344                     close(masterfd);
2345                     return false;
2346                 }
2347
2348                 logf("Entry resurrected");
2349             }
2350         }
2351
2352
2353         /* Restore the master index position. */
2354         lseek(masterfd, masterfd_pos, SEEK_SET);
2355
2356         /* Commit the data to the index. */
2357         for (i = 0; i < count; i++)
2358         {
2359             int loc = lseek(masterfd, 0, SEEK_CUR);
2360
2361             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2362             {
2363                 logf("read fail #3");
2364                 close(masterfd);
2365                 return false;
2366             }
2367
2368             for (j = 0; j < TAG_COUNT; j++)
2369             {
2370                 if (!TAGCACHE_IS_NUMERIC(j))
2371                     continue;
2372
2373                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2374             }
2375             idx.flag = entrybuf[i].flag;
2376
2377             if (idx.tag_seek[tag_commitid])
2378             {
2379                 /* Data has been resurrected. */
2380                 idx.flag |= FLAG_DIRTYNUM;
2381             }
2382             else if (tc_stat.ready && current_tcmh.commitid > 0)
2383             {
2384                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2385                 idx.flag |= FLAG_DIRTYNUM;
2386             }
2387
2388             /* Write back the updated index. */
2389             lseek(masterfd, loc, SEEK_SET);
2390             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2391             {
2392                 logf("write fail");
2393                 close(masterfd);
2394                 return false;
2395             }
2396         }
2397
2398         entries_processed += count;
2399         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2400     }
2401
2402     close(masterfd);
2403
2404     return true;
2405 }
2406
2407 /**
2408  * Return values:
2409  *     > 0   success
2410  *    == 0   temporary failure
2411  *     < 0   fatal error
2412  */
2413 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2414 {
2415     int i;
2416     struct tagcache_header tch;
2417     struct master_header   tcmh;
2418     struct index_entry idxbuf[IDX_BUF_DEPTH];
2419     int idxbuf_pos;
2420     char buf[TAG_MAXLEN+32];
2421     int fd = -1, masterfd;
2422     bool error = false;
2423     int init;
2424     int masterfd_pos;
2425
2426     logf("Building index: %d", index_type);
2427
2428     /* Check the number of entries we need to allocate ram for. */
2429     commit_entry_count = h->entry_count + 1;
2430
2431     masterfd = open_master_fd(&tcmh, false);
2432     if (masterfd >= 0)
2433     {
2434         commit_entry_count += tcmh.tch.entry_count;
2435         close(masterfd);
2436     }
2437     else
2438         remove_files(); /* Just to be sure we are clean. */
2439
2440     /* Open the index file, which contains the tag names. */
2441     fd = open_tag_fd(&tch, index_type, true);
2442     if (fd >= 0)
2443     {
2444         logf("tch.datasize=%ld", tch.datasize);
2445         lookup_buffer_depth = 1 +
2446         /* First part */ commit_entry_count +
2447         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2448     }
2449     else
2450     {
2451         lookup_buffer_depth = 1 +
2452         /* First part */ commit_entry_count +
2453         /* Second part */ 0;
2454     }
2455
2456     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2457     logf("commit_entry_count=%ld", commit_entry_count);
2458
2459     /* Allocate buffer for all index entries from both old and new
2460      * tag files. */
2461     tempbufidx = 0;
2462     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2463
2464     /* Allocate lookup buffer. The first portion of commit_entry_count
2465      * contains the new tags in the temporary file and the second
2466      * part for locating entries already in the db.
2467      *
2468      *  New tags  Old tags
2469      * +---------+---------------------------+
2470      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2471      * +---------+---------------------------+
2472      *
2473      * Old tags are inserted to a temporary buffer with position:
2474      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2475      * And new tags with index:
2476      *     tempbuf_insert(idx, ...);
2477      *
2478      * The buffer is sorted and written into tag file:
2479      *     tempbuf_sort(...);
2480      * leaving master index locations messed up.
2481      *
2482      * That is fixed using the lookup buffer for old tags:
2483      *     new_seek = tempbuf_find_location(old_seek, ...);
2484      * and for new tags:
2485      *     new_seek = tempbuf_find_location(idx);
2486      */
2487     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2488     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2489     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2490
2491     /* And calculate the remaining data space used mainly for storing
2492      * tag data (strings). */
2493     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2494     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2495     {
2496         logf("Buffer way too small!");
2497         return 0;
2498     }
2499
2500     if (fd >= 0)
2501     {
2502         /**
2503          * If tag file contains unique tags (sorted index), we will load
2504          * it entirely into memory so we can resort it later for use with
2505          * chunked browsing.
2506          */
2507         if (TAGCACHE_IS_SORTED(index_type))
2508         {
2509             logf("loading tags...");
2510             for (i = 0; i < tch.entry_count; i++)
2511             {
2512                 struct tagfile_entry entry;
2513                 int loc = lseek(fd, 0, SEEK_CUR);
2514                 bool ret;
2515
2516                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2517                 {
2518                     logf("read error #7");
2519                     close(fd);
2520                     return -2;
2521                 }
2522
2523                 if (entry.tag_length >= (int)sizeof(buf))
2524                 {
2525                     logf("too long tag #3");
2526                     close(fd);
2527                     return -2;
2528                 }
2529
2530                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2531                 {
2532                     logf("read error #8");
2533                     close(fd);
2534                     return -2;
2535                 }
2536
2537                 /* Skip deleted entries. */
2538                 if (buf[0] == '\0')
2539                     continue;
2540
2541                 /**
2542                  * Save the tag and tag id in the memory buffer. Tag id
2543                  * is saved so we can later reindex the master lookup
2544                  * table when the index gets resorted.
2545                  */
2546                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2547                                      + commit_entry_count, entry.idx_id,
2548                                      TAGCACHE_IS_UNIQUE(index_type));
2549                 if (!ret)
2550                 {
2551                     close(fd);
2552                     return -3;
2553                 }
2554                 do_timed_yield();
2555             }
2556             logf("done");
2557         }
2558         else
2559             tempbufidx = tch.entry_count;
2560     }
2561     else
2562     {
2563         /**
2564          * Creating new index file to store the tags. No need to preload
2565          * anything whether the index type is sorted or not.
2566          */
2567         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2568         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2569         if (fd < 0)
2570         {
2571             logf("%s open fail", buf);
2572             return -2;
2573         }
2574
2575         tch.magic = TAGCACHE_MAGIC;
2576         tch.entry_count = 0;
2577         tch.datasize = 0;
2578
2579         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2580             != sizeof(struct tagcache_header))
2581         {
2582             logf("header write failed");
2583             close(fd);
2584             return -2;
2585         }
2586     }
2587
2588     /* Loading the tag lookup file as "master file". */
2589     logf("Loading index file");
2590     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2591
2592     if (masterfd < 0)
2593     {
2594         logf("Creating new DB");
2595         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2596
2597         if (masterfd < 0)
2598         {
2599             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2600             close(fd);
2601             return -2;
2602         }
2603
2604         /* Write the header (write real values later). */
2605         memset(&tcmh, 0, sizeof(struct master_header));
2606         tcmh.tch = *h;
2607         tcmh.tch.entry_count = 0;
2608         tcmh.tch.datasize = 0;
2609         tcmh.dirty = true;
2610         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2611         init = true;
2612         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2613     }
2614     else
2615     {
2616         /**
2617          * Master file already exists so we need to process the current
2618          * file first.
2619          */
2620         init = false;
2621
2622         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2623             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2624         {
2625             logf("header error");
2626             close(fd);
2627             close(masterfd);
2628             return -2;
2629         }
2630
2631         /**
2632          * If we reach end of the master file, we need to expand it to
2633          * hold new tags. If the current index is not sorted, we can
2634          * simply append new data to end of the file.
2635          * However, if the index is sorted, we need to update all tag
2636          * pointers in the master file for the current index.
2637          */
2638         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2639             SEEK_CUR);
2640         if (masterfd_pos == filesize(masterfd))
2641         {
2642             logf("appending...");
2643             init = true;
2644         }
2645     }
2646
2647     /**
2648      * Load new unique tags in memory to be sorted later and added
2649      * to the master lookup file.
2650      */
2651     if (TAGCACHE_IS_SORTED(index_type))
2652     {
2653         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2654         /* h is the header of the temporary file containing new tags. */
2655         logf("inserting new tags...");
2656         for (i = 0; i < h->entry_count; i++)
2657         {
2658             struct temp_file_entry entry;
2659
2660             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2661                 sizeof(struct temp_file_entry))
2662             {
2663                 logf("read fail #3");
2664                 error = true;
2665                 goto error_exit;
2666             }
2667
2668             /* Read data. */
2669             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2670             {
2671                 logf("too long entry!");
2672                 error = true;
2673                 goto error_exit;
2674             }
2675
2676             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2677             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2678                 entry.tag_length[index_type])
2679             {
2680                 logf("read fail #4");
2681                 error = true;
2682                 goto error_exit;
2683             }
2684
2685             if (TAGCACHE_IS_UNIQUE(index_type))
2686                 error = !tempbuf_insert(buf, i, -1, true);
2687             else
2688                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2689
2690             if (error)
2691             {
2692                 logf("insert error");
2693                 goto error_exit;
2694             }
2695
2696             /* Skip to next. */
2697             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2698                     entry.tag_length[index_type], SEEK_CUR);
2699             do_timed_yield();
2700         }
2701         logf("done");
2702
2703         /* Sort the buffer data and write it to the index file. */
2704         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2705         /**
2706          * We need to truncate the index file now. There can be junk left
2707          * at the end of file (however, we _should_ always follow the
2708          * entry_count and don't crash with that).
2709          */
2710         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2711
2712         i = tempbuf_sort(fd);
2713         if (i < 0)
2714             goto error_exit;
2715         logf("sorted %d tags", i);
2716
2717         /**
2718          * Now update all indexes in the master lookup file.
2719          */
2720         logf("updating indices...");
2721         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2722         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2723         {
2724             int j;
2725             int loc = lseek(masterfd, 0, SEEK_CUR);
2726
2727             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2728
2729             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2730                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2731             {
2732                 logf("read fail #5");
2733                 error = true;
2734                 goto error_exit ;
2735             }
2736             lseek(masterfd, loc, SEEK_SET);
2737
2738             for (j = 0; j < idxbuf_pos; j++)
2739             {
2740                 if (idxbuf[j].flag & FLAG_DELETED)
2741                 {
2742                     /* We can just ignore deleted entries. */
2743                     // idxbuf[j].tag_seek[index_type] = 0;
2744                     continue;
2745                 }
2746
2747                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2748                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2749                     + commit_entry_count);
2750
2751                 if (idxbuf[j].tag_seek[index_type] < 0)
2752                 {
2753                     logf("update error: %ld/%d/%ld",
2754                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2755                     error = true;
2756                     goto error_exit;
2757                 }
2758
2759                 do_timed_yield();
2760             }
2761
2762             /* Write back the updated index. */
2763             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2764                         index_entry_ec, tc_stat.econ) !=
2765                 (int)sizeof(struct index_entry)*idxbuf_pos)
2766             {
2767                 logf("write fail");
2768                 error = true;
2769                 goto error_exit;
2770             }
2771         }
2772         logf("done");
2773     }
2774
2775     /**
2776      * Walk through the temporary file containing the new tags.
2777      */
2778     // build_normal_index(h, tmpfd, masterfd, idx);
2779     logf("updating new indices...");
2780     lseek(masterfd, masterfd_pos, SEEK_SET);
2781     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2782     lseek(fd, 0, SEEK_END);
2783     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2784     {
2785         int j;
2786
2787         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2788         if (init)
2789         {
2790             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2791         }
2792         else
2793         {
2794             int loc = lseek(masterfd, 0, SEEK_CUR);
2795
2796             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2797                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2798             {
2799                 logf("read fail #6");
2800                 error = true;
2801                 break ;
2802             }
2803             lseek(masterfd, loc, SEEK_SET);
2804         }
2805
2806         /* Read entry headers. */
2807         for (j = 0; j < idxbuf_pos; j++)
2808         {
2809             if (!TAGCACHE_IS_SORTED(index_type))
2810             {
2811                 struct temp_file_entry entry;
2812                 struct tagfile_entry fe;
2813
2814                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2815                     sizeof(struct temp_file_entry))
2816                 {
2817                     logf("read fail #7");
2818                     error = true;
2819                     break ;
2820                 }
2821
2822                 /* Read data. */
2823                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2824                 {
2825                     logf("too long entry!");
2826                     logf("length=%d", entry.tag_length[index_type]);
2827                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2828                     error = true;
2829                     break ;
2830                 }
2831
2832                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2833                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2834                     entry.tag_length[index_type])
2835                 {
2836                     logf("read fail #8");
2837                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2838                     logf("length=0x%02x", entry.tag_length[index_type]);
2839                     error = true;
2840                     break ;
2841                 }
2842
2843                 /* Write to index file. */
2844                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2845                 fe.tag_length = entry.tag_length[index_type];
2846                 fe.idx_id = tcmh.tch.entry_count + i + j;
2847                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2848                 write(fd, buf, fe.tag_length);
2849                 tempbufidx++;
2850
2851                 /* Skip to next. */
2852                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2853                       entry.tag_length[index_type], SEEK_CUR);
2854             }
2855             else
2856             {
2857                 /* Locate the correct entry from the sorted array. */
2858                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2859                 if (idxbuf[j].tag_seek[index_type] < 0)
2860                 {
2861                     logf("entry not found (%d)", j);
2862                     error = true;
2863                     break ;
2864                 }
2865             }
2866         }
2867
2868         /* Write index. */
2869         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2870                   index_entry_ec, tc_stat.econ) !=
2871             (int)sizeof(struct index_entry)*idxbuf_pos)
2872         {
2873             logf("tagcache: write fail #4");
2874             error = true;
2875             break ;
2876         }
2877
2878         do_timed_yield();
2879     }
2880     logf("done");
2881
2882     /* Finally write the header. */
2883     tch.magic = TAGCACHE_MAGIC;
2884     tch.entry_count = tempbufidx;
2885     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2886     lseek(fd, 0, SEEK_SET);
2887     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2888
2889     if (index_type != tag_filename)
2890         h->datasize += tch.datasize;
2891     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2892     error_exit:
2893
2894     close(fd);
2895     close(masterfd);
2896
2897     if (error)
2898         return -2;
2899
2900     return 1;
2901 }
2902
2903 static bool commit(void)
2904 {
2905     struct tagcache_header tch;
2906     struct master_header   tcmh;
2907     int i, len, rc;
2908     int tmpfd;
2909     int masterfd;
2910 #ifdef HAVE_DIRCACHE
2911     bool dircache_buffer_stolen = false;
2912 #endif
2913     bool local_allocation = false;
2914
2915     logf("committing tagcache");
2916
2917     while (write_lock)
2918         sleep(1);
2919
2920     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2921     if (tmpfd < 0)
2922     {
2923         logf("nothing to commit");
2924         return true;
2925     }
2926
2927
2928     /* Load the header. */
2929     len = sizeof(struct tagcache_header);
2930     rc = read(tmpfd, &tch, len);
2931
2932     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2933     {
2934         logf("incorrect tmpheader");
2935         close(tmpfd);
2936         remove(TAGCACHE_FILE_TEMP);
2937         return false;
2938     }
2939
2940     if (tch.entry_count == 0)
2941     {
2942         logf("nothing to commit");
2943         close(tmpfd);
2944         remove(TAGCACHE_FILE_TEMP);
2945         return true;
2946     }
2947
2948     /* Fully initialize existing headers (if any) before going further. */
2949     tc_stat.ready = check_all_headers();
2950
2951 #ifdef HAVE_EEPROM_SETTINGS
2952     remove(TAGCACHE_STATEFILE);
2953 #endif
2954
2955     /* At first be sure to unload the ramcache! */
2956 #ifdef HAVE_TC_RAMCACHE
2957     tc_stat.ramcache = false;
2958 #endif
2959
2960     read_lock++;
2961
2962     /* Try to steal every buffer we can :) */
2963     if (tempbuf_size == 0)
2964         local_allocation = true;
2965
2966 #ifdef HAVE_DIRCACHE
2967     if (tempbuf_size == 0)
2968     {
2969         /* Try to steal the dircache buffer. */
2970         tempbuf = dircache_steal_buffer(&tempbuf_size);
2971         tempbuf_size &= ~0x03;
2972
2973         if (tempbuf_size > 0)
2974         {
2975             dircache_buffer_stolen = true;
2976         }
2977     }
2978 #endif
2979
2980 #ifdef HAVE_TC_RAMCACHE
2981     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2982     {
2983         tempbuf = (char *)(hdr + 1);
2984         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2985         tempbuf_size &= ~0x03;
2986     }
2987 #endif
2988
2989     /* And finally fail if there are no buffers available. */
2990     if (tempbuf_size == 0)
2991     {
2992         logf("delaying commit until next boot");
2993         tc_stat.commit_delayed = true;
2994         close(tmpfd);
2995         read_lock--;
2996         return false;
2997     }
2998
2999     logf("commit %ld entries...", tch.entry_count);
3000
3001     /* Mark DB dirty so it will stay disabled if commit fails. */
3002     current_tcmh.dirty = true;
3003     update_master_header();
3004
3005     /* Now create the index files. */
3006     tc_stat.commit_step = 0;
3007     tch.datasize = 0;
3008     tc_stat.commit_delayed = false;
3009
3010     for (i = 0; i < TAG_COUNT; i++)
3011     {
3012         int ret;
3013
3014         if (TAGCACHE_IS_NUMERIC(i))
3015             continue;
3016
3017         tc_stat.commit_step++;
3018         ret = build_index(i, &tch, tmpfd);
3019         if (ret <= 0)
3020         {
3021             close(tmpfd);
3022             logf("tagcache failed init");
3023             if (ret == 0)
3024                 tc_stat.commit_delayed = true;
3025
3026             tc_stat.commit_step = 0;
3027             read_lock--;
3028             return false;
3029         }
3030     }
3031
3032     if (!build_numeric_indices(&tch, tmpfd))
3033     {
3034         logf("Failure to commit numeric indices");
3035         close(tmpfd);
3036         tc_stat.commit_step = 0;
3037         read_lock--;
3038         return false;
3039     }
3040
3041     close(tmpfd);
3042     remove(TAGCACHE_FILE_TEMP);
3043
3044     tc_stat.commit_step = 0;
3045
3046     /* Update the master index headers. */
3047     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3048     {
3049         read_lock--;
3050         return false;
3051     }
3052
3053     tcmh.tch.entry_count += tch.entry_count;
3054     tcmh.tch.datasize = sizeof(struct master_header)
3055         + sizeof(struct index_entry) * tcmh.tch.entry_count
3056         + tch.datasize;
3057     tcmh.dirty = false;
3058     tcmh.commitid++;
3059
3060     lseek(masterfd, 0, SEEK_SET);
3061     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3062     close(masterfd);
3063
3064     logf("tagcache committed");
3065     tc_stat.ready = check_all_headers();
3066     tc_stat.readyvalid = true;
3067
3068     if (local_allocation)
3069     {
3070         tempbuf = NULL;
3071         tempbuf_size = 0;
3072     }
3073
3074 #ifdef HAVE_DIRCACHE
3075     /* Rebuild the dircache, if we stole the buffer. */
3076     if (dircache_buffer_stolen)
3077         dircache_build(0);
3078 #endif
3079
3080 #ifdef HAVE_TC_RAMCACHE
3081     /* Reload tagcache. */
3082     if (tc_stat.ramcache_allocated > 0)
3083         tagcache_start_scan();
3084 #endif
3085
3086     read_lock--;
3087
3088     return true;
3089 }
3090
3091 static void allocate_tempbuf(void)
3092 {
3093     /* Yeah, malloc would be really nice now :) */
3094 #ifdef __PCTOOL__
3095     tempbuf_size = 32*1024*1024;
3096     tempbuf = malloc(tempbuf_size);
3097 #else
3098     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3099     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3100     audiobuf += tempbuf_size;
3101 #endif
3102 }
3103
3104 static void free_tempbuf(void)
3105 {
3106     if (tempbuf_size == 0)
3107         return ;
3108
3109 #ifdef __PCTOOL__
3110     free(tempbuf);
3111 #else
3112     audiobuf -= tempbuf_size;
3113 #endif
3114     tempbuf = NULL;
3115     tempbuf_size = 0;
3116 }
3117
3118 #ifndef __PCTOOL__
3119
3120 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3121 {
3122     struct index_entry idx;
3123
3124     if (!tc_stat.ready)
3125         return false;
3126
3127     if (!TAGCACHE_IS_NUMERIC(tag))
3128         return false;
3129
3130     if (!get_index(masterfd, idx_id, &idx, false))
3131         return false;
3132
3133     idx.tag_seek[tag] = data;
3134     idx.flag |= FLAG_DIRTYNUM;
3135
3136     return write_index(masterfd, idx_id, &idx);
3137 }
3138
3139 #if 0
3140 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3141                                    int tag, long data)
3142 {
3143     struct master_header myhdr;
3144
3145     if (tcs->masterfd < 0)
3146     {
3147         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3148             return false;
3149     }
3150
3151     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3152 }
3153 #endif
3154
3155 static bool command_queue_is_full(void)
3156 {
3157     int next;
3158
3159     next = command_queue_widx + 1;
3160     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3161         next = 0;
3162
3163     return (next == command_queue_ridx);
3164 }
3165
3166 static void command_queue_sync_callback(void *data)
3167 {
3168     (void)data;
3169     struct master_header myhdr;
3170     int masterfd;
3171
3172     mutex_lock(&command_queue_mutex);
3173
3174     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3175         return;
3176
3177     while (command_queue_ridx != command_queue_widx)
3178     {
3179         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3180
3181         switch (ce->command)
3182         {
3183             case CMD_UPDATE_MASTER_HEADER:
3184             {
3185                 close(masterfd);
3186                 update_master_header();
3187
3188                 /* Re-open the masterfd. */
3189                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3190                     return;
3191
3192                 break;
3193             }
3194             case CMD_UPDATE_NUMERIC:
3195             {
3196                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3197                 break;
3198             }
3199         }
3200
3201         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3202             command_queue_ridx = 0;
3203     }
3204
3205     close(masterfd);
3206
3207     tc_stat.queue_length = 0;
3208     mutex_unlock(&command_queue_mutex);
3209 }
3210
3211 static void run_command_queue(bool force)
3212 {
3213     if (COMMAND_QUEUE_IS_EMPTY)
3214         return;
3215
3216     if (force || command_queue_is_full())
3217         command_queue_sync_callback(NULL);
3218     else
3219         register_storage_idle_func(command_queue_sync_callback);
3220 }
3221
3222 static void queue_command(int cmd, long idx_id, int tag, long data)
3223 {
3224     while (1)
3225     {
3226         int next;
3227
3228         mutex_lock(&command_queue_mutex);
3229         next = command_queue_widx + 1;
3230         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3231             next = 0;
3232
3233         /* Make sure queue is not full. */
3234         if (next != command_queue_ridx)
3235         {
3236             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3237
3238             ce->command = cmd;
3239             ce->idx_id = idx_id;
3240             ce->tag = tag;
3241             ce->data = data;
3242
3243             command_queue_widx = next;
3244
3245             tc_stat.queue_length++;
3246
3247             mutex_unlock(&command_queue_mutex);
3248             break;
3249         }
3250
3251         /* Queue is full, try again later... */
3252         mutex_unlock(&command_queue_mutex);
3253         sleep(1);
3254     }
3255 }
3256
3257 long tagcache_increase_serial(void)
3258 {
3259     long old;
3260
3261     if (!tc_stat.ready)
3262         return -2;
3263
3264     while (read_lock)
3265         sleep(1);
3266
3267     old = current_tcmh.serial++;
3268     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3269
3270     return old;
3271 }
3272
3273 void tagcache_update_numeric(int idx_id, int tag, long data)
3274 {
3275     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3276 }
3277 #endif /* !__PCTOOL__ */
3278
3279 long tagcache_get_serial(void)
3280 {
3281     return current_tcmh.serial;
3282 }
3283
3284 long tagcache_get_commitid(void)
3285 {
3286     return current_tcmh.commitid;
3287 }
3288
3289 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3290 {
3291     char buf[512];
3292     int i;
3293
3294     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3295     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3296     {
3297         if (*datastr == '\0')
3298             break;
3299
3300         if (*datastr == '"' || *datastr == '\\')
3301             buf[i++] = '\\';
3302
3303         buf[i] = *(datastr++);
3304     }
3305
3306     strcpy(&buf[i], "\" ");
3307
3308     write(fd, buf, i + 2);
3309
3310     return true;
3311 }
3312
3313 #ifndef __PCTOOL__
3314
3315 static bool read_tag(char *dest, long size,
3316                      const char *src, const char *tagstr)
3317 {
3318     int pos;
3319     char current_tag[32];
3320
3321     while (*src != '\0')
3322     {
3323         /* Skip all whitespace */
3324         while (*src == ' ')
3325             src++;
3326
3327         if (*src == '\0')
3328             break;
3329
3330         pos = 0;
3331         /* Read in tag name */
3332         while (*src != '=' && *src != ' ')
3333         {
3334             current_tag[pos] = *src;
3335             src++;
3336             pos++;
3337
3338             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3339                 return false;
3340         }
3341         current_tag[pos] = '\0';
3342
3343         /* Read in tag data */
3344
3345         /* Find the start. */
3346         while (*src != '"' && *src != '\0')
3347             src++;
3348
3349         if (*src == '\0' || *(++src) == '\0')
3350             return false;
3351
3352         /* Read the data. */
3353         for (pos = 0; pos < size; pos++)
3354         {
3355             if (*src == '\0')
3356                 break;
3357
3358             if (*src == '\\')
3359             {
3360                 dest[pos] = *(src+1);
3361                 src += 2;
3362                 continue;
3363             }
3364
3365             dest[pos] = *src;
3366
3367             if (*src == '"')
3368             {
3369                 src++;
3370                 break;
3371             }
3372
3373             if (*src == '\0')
3374                 break;
3375
3376             src++;
3377         }
3378         dest[pos] = '\0';
3379
3380         if (!strcasecmp(tagstr, current_tag))
3381             return true;
3382     }
3383
3384     return false;
3385 }
3386
3387 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3388 {
3389     struct index_entry idx;
3390     char tag_data[TAG_MAXLEN+32];
3391     int idx_id;
3392     long masterfd = (long)parameters;
3393     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3394                                 tag_lastplayed, tag_commitid, tag_lastoffset };
3395     int i;
3396     (void)line_n;
3397
3398     if (*buf == '#')
3399         return 0;
3400
3401     logf("%d/%s", line_n, buf);
3402     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3403     {
3404         logf("filename missing");
3405         logf("-> %s", buf);
3406         return 0;
3407     }
3408
3409     idx_id = find_index(tag_data);
3410     if (idx_id < 0)
3411     {
3412         logf("entry not found");
3413         return 0;
3414     }
3415
3416     if (!get_index(masterfd, idx_id, &idx, false))
3417     {
3418         logf("failed to retrieve index entry");
3419         return 0;
3420     }
3421
3422     /* Stop if tag has already been modified. */
3423     if (idx.flag & FLAG_DIRTYNUM)
3424         return 0;
3425
3426     logf("import: %s", tag_data);
3427
3428     idx.flag |= FLAG_DIRTYNUM;
3429     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3430     {
3431         int data;
3432
3433         if (!read_tag(tag_data, sizeof tag_data, buf,
3434                       tagcache_tag_to_str(import_tags[i])))
3435         {
3436             continue;
3437         }
3438
3439         data = atoi(tag_data);
3440         if (data < 0)
3441             continue;
3442
3443         idx.tag_seek[import_tags[i]] = data;
3444
3445         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3446             current_tcmh.serial = data + 1;
3447         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3448             current_tcmh.commitid = data + 1;
3449     }
3450
3451     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3452 }
3453
3454 bool tagcache_import_changelog(void)
3455 {
3456     struct master_header myhdr;
3457     struct tagcache_header tch;
3458     int clfd;
3459     long masterfd;
3460     char buf[2048];
3461
3462     if (!tc_stat.ready)
3463         return false;
3464
3465     while (read_lock)
3466         sleep(1);
3467
3468     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3469     if (clfd < 0)
3470     {
3471         logf("failure to open changelog");
3472         return false;
3473     }
3474
3475     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3476     {
3477         close(clfd);
3478         return false;
3479     }
3480
3481     write_lock++;
3482
3483     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3484
3485     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3486                   parse_changelog_line);
3487
3488     close(clfd);
3489     close(masterfd);
3490
3491     if (filenametag_fd >= 0)
3492     {
3493         close(filenametag_fd);
3494         filenametag_fd = -1;
3495     }
3496
3497     write_lock--;
3498
3499     update_master_header();
3500
3501     return true;
3502 }
3503
3504 #endif /* !__PCTOOL__ */
3505
3506 bool tagcache_create_changelog(struct tagcache_search *tcs)
3507 {
3508     struct master_header myhdr;
3509     struct index_entry idx;
3510     char buf[TAG_MAXLEN+32];
3511     char temp[32];
3512     int clfd;
3513     int i, j;
3514
3515     if (!tc_stat.ready)
3516         return false;
3517
3518     if (!tagcache_search(tcs, tag_filename))
3519         return false;
3520
3521     /* Initialize the changelog */
3522     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3523     if (clfd < 0)
3524     {
3525         logf("failure to open changelog");
3526         return false;
3527     }
3528
3529     if (tcs->masterfd < 0)
3530     {
3531         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3532             return false;
3533     }
3534     else
3535     {
3536         lseek(tcs->masterfd, 0, SEEK_SET);
3537         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3538     }
3539
3540     write(clfd, "## Changelog version 1\n", 23);
3541
3542     for (i = 0; i < myhdr.tch.entry_count; i++)
3543     {
3544         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3545         {
3546             logf("read error #9");
3547             tagcache_search_finish(tcs);
3548             close(clfd);
3549             return false;
3550         }
3551
3552         /* Skip until the entry found has been modified. */
3553         if (! (idx.flag & FLAG_DIRTYNUM) )
3554             continue;
3555
3556         /* Skip deleted entries too. */
3557         if (idx.flag & FLAG_DELETED)
3558             continue;
3559
3560         /* Now retrieve all tags. */
3561         for (j = 0; j < TAG_COUNT; j++)
3562         {
3563             if (TAGCACHE_IS_NUMERIC(j))
3564             {
3565                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3566                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3567                 continue;
3568             }
3569
3570             tcs->type = j;
3571             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3572             write_tag(clfd, tagcache_tag_to_str(j), buf);
3573         }
3574
3575         write(clfd, "\n", 1);
3576         do_timed_yield();
3577     }
3578
3579     close(clfd);
3580
3581     tagcache_search_finish(tcs);
3582
3583     return true;
3584 }
3585
3586 static bool delete_entry(long idx_id)
3587 {
3588     int fd = -1;
3589     int masterfd = -1;
3590     int tag, i;
3591     struct index_entry idx, myidx;
3592     struct master_header myhdr;
3593     char buf[TAG_MAXLEN+32];
3594     int in_use[TAG_COUNT];
3595
3596     logf("delete_entry(): %ld", idx_id);
3597
3598 #ifdef HAVE_TC_RAMCACHE
3599     /* At first mark the entry removed from ram cache. */
3600     if (tc_stat.ramcache)
3601         hdr->indices[idx_id].flag |= FLAG_DELETED;
3602 #endif
3603
3604     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3605         return false;
3606
3607     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3608     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3609     {
3610         logf("delete_entry(): read error");
3611         goto cleanup;
3612     }
3613
3614     if (myidx.flag & FLAG_DELETED)
3615     {
3616         logf("delete_entry(): already deleted!");
3617         goto cleanup;
3618     }
3619
3620     myidx.flag |= FLAG_DELETED;
3621     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3622     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3623     {
3624         logf("delete_entry(): write_error #1");
3625         goto cleanup;
3626     }
3627
3628     /* Now check which tags are no longer in use (if any) */
3629     for (tag = 0; tag < TAG_COUNT; tag++)
3630         in_use[tag] = 0;
3631
3632     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3633     for (i = 0; i < myhdr.tch.entry_count; i++)
3634     {
3635         struct index_entry *idxp;
3636
3637 #ifdef HAVE_TC_RAMCACHE
3638         /* Use RAM DB if available for greater speed */
3639         if (tc_stat.ramcache)
3640             idxp = &hdr->indices[i];
3641         else
3642 #endif
3643         {
3644             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3645             {
3646                 logf("delete_entry(): read error #2");
3647                 goto cleanup;
3648             }
3649             idxp = &idx;
3650         }
3651
3652         if (idxp->flag & FLAG_DELETED)
3653             continue;
3654
3655         for (tag = 0; tag < TAG_COUNT; tag++)
3656         {
3657             if (TAGCACHE_IS_NUMERIC(tag))
3658                 continue;
3659
3660             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3661                 in_use[tag]++;
3662         }
3663     }
3664
3665     /* Now delete all tags no longer in use. */
3666     for (tag = 0; tag < TAG_COUNT; tag++)
3667     {
3668         struct tagcache_header tch;
3669         int oldseek = myidx.tag_seek[tag];
3670
3671         if (TAGCACHE_IS_NUMERIC(tag))
3672             continue;
3673
3674         /**
3675          * Replace tag seek with a hash value of the field string data.
3676          * That way runtime statistics of moved or altered files can be
3677          * resurrected.
3678          */
3679 #ifdef HAVE_TC_RAMCACHE
3680         if (tc_stat.ramcache && tag != tag_filename)
3681         {
3682             struct tagfile_entry *tfe;
3683             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3684
3685             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3686             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3687             myidx.tag_seek[tag] = *seek;
3688         }
3689         else
3690 #endif
3691         {
3692             struct tagfile_entry tfe;
3693
3694             /* Open the index file, which contains the tag names. */
3695             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3696                 goto cleanup;
3697
3698             /* Skip the header block */
3699             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3700             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3701             {
3702                 logf("delete_entry(): read error #3");
3703                 goto cleanup;
3704             }
3705
3706             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3707             {
3708                 logf("delete_entry(): read error #4");
3709                 goto cleanup;
3710             }
3711
3712             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3713         }
3714
3715         if (in_use[tag])
3716         {
3717             logf("in use: %d/%d", tag, in_use[tag]);
3718             if (fd >= 0)
3719             {
3720                 close(fd);
3721                 fd = -1;
3722             }
3723             continue;
3724         }
3725
3726 #ifdef HAVE_TC_RAMCACHE
3727         /* Delete from ram. */
3728         if (tc_stat.ramcache && tag != tag_filename)
3729         {
3730             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3731             tagentry->tag_data[0] = '\0';
3732         }
3733 #endif
3734
3735         /* Open the index file, which contains the tag names. */
3736         if (fd < 0)
3737         {
3738             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3739                 goto cleanup;
3740         }
3741
3742         /* Skip the header block */
3743         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3744
3745         /* Debug, print 10 first characters of the tag
3746         read(fd, buf, 10);
3747         buf[10]='\0';
3748         logf("TAG:%s", buf);
3749         lseek(fd, -10, SEEK_CUR);
3750         */
3751
3752         /* Write first data byte in tag as \0 */
3753         write(fd, "", 1);
3754
3755         /* Now tag data has been removed */
3756         close(fd);
3757         fd = -1;
3758     }
3759
3760     /* Write index entry back into master index. */
3761     lseek(masterfd, sizeof(struct master_header) +
3762           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3763     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3764     {
3765         logf("delete_entry(): write_error #2");
3766         goto cleanup;
3767     }
3768
3769     close(masterfd);
3770
3771     return true;
3772
3773     cleanup:
3774     if (fd >= 0)
3775         close(fd);
3776     if (masterfd >= 0)
3777         close(masterfd);
3778
3779     return false;
3780 }
3781
3782 #ifndef __PCTOOL__
3783 /**
3784  * Returns true if there is an event waiting in the queue
3785  * that requires the current operation to be aborted.
3786  */
3787 static bool check_event_queue(void)
3788 {
3789     struct queue_event ev;
3790
3791     if(!queue_peek(&tagcache_queue, &ev))
3792         return false;
3793
3794     switch (ev.id)
3795     {
3796         case Q_STOP_SCAN:
3797         case SYS_POWEROFF:
3798         case SYS_USB_CONNECTED:
3799             return true;
3800     }
3801
3802     return false;
3803 }
3804 #endif
3805
3806 #ifdef HAVE_TC_RAMCACHE
3807 static bool allocate_tagcache(void)
3808 {
3809     struct master_header tcmh;
3810     int fd;
3811
3812     /* Load the header. */
3813     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3814     {
3815         hdr = NULL;
3816         return false;
3817     }
3818
3819     close(fd);
3820
3821     /**
3822      * Now calculate the required cache size plus
3823      * some extra space for alignment fixes.
3824      */
3825     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3826         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3827     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3828     memset(hdr, 0, sizeof(struct ramcache_header));
3829     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3830     hdr->indices = (struct index_entry *)(hdr + 1);
3831     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3832
3833     return true;
3834 }
3835
3836 # ifdef HAVE_EEPROM_SETTINGS
3837 static bool tagcache_dumpload(void)
3838 {
3839     struct statefile_header shdr;
3840     int fd, rc;
3841     long offpos;
3842     int i;
3843
3844     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3845     if (fd < 0)
3846     {
3847         logf("no tagcache statedump");
3848         return false;
3849     }
3850
3851     /* Check the statefile memory placement */
3852     hdr = buffer_alloc(0);
3853     rc = read(fd, &shdr, sizeof(struct statefile_header));
3854     if (rc != sizeof(struct statefile_header)
3855         /* || (long)hdr != (long)shdr.hdr */)
3856     {
3857         logf("incorrect statefile");
3858         hdr = NULL;
3859         close(fd);
3860         return false;
3861     }
3862
3863     offpos = (long)hdr - (long)shdr.hdr;
3864
3865     /* Lets allocate real memory and load it */
3866     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3867     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3868     close(fd);
3869
3870     if (rc != shdr.tc_stat.ramcache_allocated)
3871     {
3872         logf("read failure!");
3873         hdr = NULL;
3874         return false;
3875     }
3876
3877     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3878
3879     /* Now fix the pointers */
3880     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3881     for (i = 0; i < TAG_COUNT; i++)
3882         hdr->tags[i] += offpos;
3883
3884     return true;
3885 }
3886
3887 static bool tagcache_dumpsave(void)
3888 {
3889     struct statefile_header shdr;
3890     int fd;
3891
3892     if (!tc_stat.ramcache)
3893         return false;
3894
3895     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3896     if (fd < 0)
3897     {
3898         logf("failed to create a statedump");
3899         return false;
3900     }
3901
3902     /* Create the header */
3903     shdr.hdr = hdr;
3904     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3905     write(fd, &shdr, sizeof(struct statefile_header));
3906
3907     /* And dump the data too */
3908     write(fd, hdr, tc_stat.ramcache_allocated);
3909     close(fd);
3910
3911     return true;
3912 }
3913 # endif
3914
3915 static bool load_tagcache(void)
3916 {
3917     struct tagcache_header *tch;
3918     long bytesleft = tc_stat.ramcache_allocated;
3919     struct index_entry *idx;
3920     int rc, fd;
3921     char *p;
3922     int i, tag;
3923
3924 # ifdef HAVE_DIRCACHE
3925     while (dircache_is_initializing())
3926         sleep(1);
3927
3928     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3929 # endif
3930
3931     logf("loading tagcache to ram...");
3932
3933     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3934     if (fd < 0)
3935     {
3936         logf("tagcache open failed");
3937         return false;
3938     }
3939
3940     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3941         != sizeof(struct master_header)
3942         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3943     {
3944         logf("incorrect header");
3945         return false;
3946     }
3947
3948     idx = hdr->indices;
3949
3950     /* Load the master index table. */
3951     for (i = 0; i < hdr->h.tch.entry_count; i++)
3952     {
3953         rc = ecread_index_entry(fd, idx);
3954         if (rc != sizeof(struct index_entry))
3955         {
3956             logf("read error #10");
3957             close(fd);
3958             return false;
3959         }
3960
3961         bytesleft -= sizeof(struct index_entry);
3962         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3963         {
3964             logf("too big tagcache.");
3965             close(fd);
3966             return false;
3967         }
3968
3969         idx++;
3970     }
3971
3972     close(fd);
3973
3974     /* Load the tags. */
3975     p = (char *)idx;
3976     for (tag = 0; tag < TAG_COUNT; tag++)
3977     {
3978         struct tagfile_entry *fe;
3979         char buf[TAG_MAXLEN+32];
3980
3981         if (TAGCACHE_IS_NUMERIC(tag))
3982             continue ;
3983
3984         //p = ((void *)p+1);
3985         p = (char *)((long)p & ~0x03) + 0x04;
3986         hdr->tags[tag] = p;
3987
3988         /* Check the header. */
3989         tch = (struct tagcache_header *)p;
3990         p += sizeof(struct tagcache_header);
3991
3992         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3993             return false;
3994
3995         for (hdr->entry_count[tag] = 0;
3996              hdr->entry_count[tag] < tch->entry_count;
3997              hdr->entry_count[tag]++)
3998         {
3999             long pos;
4000
4001             if (do_timed_yield())
4002             {
4003                 /* Abort if we got a critical event in queue */
4004                 if (check_event_queue())
4005                     return false;
4006             }
4007
4008             fe = (struct tagfile_entry *)p;
4009             pos = lseek(fd, 0, SEEK_CUR);
4010             rc = ecread_tagfile_entry(fd, fe);
4011             if (rc != sizeof(struct tagfile_entry))
4012             {
4013                 /* End of lookup table. */
4014                 logf("read error #11");
4015                 close(fd);
4016                 return false;
4017             }
4018
4019             /* We have a special handling for the filename tags. */
4020             if (tag == tag_filename)
4021             {
4022 # ifdef HAVE_DIRCACHE
4023                 const struct dircache_entry *dc;
4024 # endif
4025
4026                 // FIXME: This is wrong!
4027                 // idx = &hdr->indices[hdr->entry_count[i]];
4028                 idx = &hdr->indices[fe->idx_id];
4029
4030                 if (fe->tag_length >= (long)sizeof(buf)-1)
4031                 {
4032                     read(fd, buf, 10);
4033                     buf[10] = '\0';
4034                     logf("TAG:%s", buf);
4035                     logf("too long filename");
4036                     close(fd);
4037                     return false;
4038                 }
4039
4040                 rc = read(fd, buf, fe->tag_length);
4041                 if (rc != fe->tag_length)
4042                 {
4043                     logf("read error #12");
4044                     close(fd);
4045                     return false;
4046                 }
4047
4048                 /* Check if the entry has already been removed */
4049                 if (idx->flag & FLAG_DELETED)
4050                     continue;
4051
4052                 /* This flag must not be used yet. */
4053                 if (idx->flag & FLAG_DIRCACHE)
4054                 {
4055                     logf("internal error!");
4056                     close(fd);
4057                     return false;
4058                 }
4059
4060                 if (idx->tag_seek[tag] != pos)
4061                 {
4062                     logf("corrupt data structures!");
4063                     close(fd);
4064                     return false;
4065                 }
4066
4067 # ifdef HAVE_DIRCACHE
4068                 if (dircache_is_enabled())
4069                 {
4070                     dc = dircache_get_entry_ptr(buf);
4071                     if (dc == NULL)
4072                     {
4073                         logf("Entry no longer valid.");
4074                         logf("-> %s", buf);
4075                         if (global_settings.tagcache_autoupdate)
4076                             delete_entry(fe->idx_id);
4077                         continue ;
4078                     }
4079
4080                     idx->flag |= FLAG_DIRCACHE;
4081                     idx->tag_seek[tag_filename] = (long)dc;
4082                 }
4083                 else
4084 # endif
4085                 {
4086                     /* This will be very slow unless dircache is enabled
4087                        or target is flash based, but do it anyway for
4088                        consistency. */
4089                     /* Check if entry has been removed. */
4090                     if (global_settings.tagcache_autoupdate)
4091                     {
4092                         if (!file_exists(buf))
4093                         {
4094                             logf("Entry no longer valid.");
4095                             logf("-> %s", buf);
4096                             delete_entry(fe->idx_id);
4097                             continue;
4098                         }
4099                     }
4100                 }
4101
4102                 continue ;
4103             }
4104
4105             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4106             if (bytesleft < 0)
4107             {
4108                 logf("too big tagcache #2");
4109                 logf("tl: %ld", fe->tag_length);
4110                 logf("bl: %ld", bytesleft);
4111                 close(fd);
4112                 return false;
4113             }
4114
4115             p = fe->tag_data;
4116             rc = read(fd, fe->tag_data, fe->tag_length);
4117             p += rc;
4118
4119             if (rc != fe->tag_length)
4120             {
4121                 logf("read error #13");
4122                 logf("rc=0x%04x", rc); // 0x431
4123                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4124                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4125                 logf("tag=0x%02x", tag); // 0x00
4126                 close(fd);
4127                 return false;
4128             }
4129         }
4130         close(fd);
4131     }
4132
4133     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4134     logf("tagcache loaded into ram!");
4135
4136     return true;
4137 }
4138 #endif /* HAVE_TC_RAMCACHE */
4139
4140 static bool check_deleted_files(void)
4141 {
4142     int fd;
4143     char buf[TAG_MAXLEN+32];
4144     struct tagfile_entry tfe;
4145
4146     logf("reverse scan...");
4147     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4148     fd = open(buf, O_RDONLY);
4149
4150     if (fd < 0)
4151     {
4152         logf("%s open fail", buf);
4153         return false;
4154     }
4155
4156     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4157     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4158 #ifndef __PCTOOL__
4159            && !check_event_queue()
4160 #endif
4161            )
4162     {
4163         if (tfe.tag_length >= (long)sizeof(buf)-1)
4164         {
4165             logf("too long tag");
4166             close(fd);
4167             return false;
4168         }
4169
4170         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4171         {
4172             logf("read error #14");
4173             close(fd);
4174             return false;
4175         }
4176
4177         /* Check if the file has already deleted from the db. */
4178         if (*buf == '\0')
4179             continue;
4180
4181         /* Now check if the file exists. */
4182         if (!file_exists(buf))
4183         {
4184             logf("Entry no longer valid.");
4185             logf("-> %s / %ld", buf, tfe.tag_length);
4186             delete_entry(tfe.idx_id);
4187         }
4188     }
4189
4190     close(fd);
4191
4192     logf("done");
4193
4194     return true;
4195 }
4196
4197
4198 /* Note that this function must not be inlined, otherwise the whole point
4199  * of having the code in a separate function is lost.
4200  */
4201 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4202     int *ignore, int *unignore)
4203 {
4204     char newpath[MAX_PATH];
4205
4206     /* check for a database.ignore file */
4207     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4208     *ignore = file_exists(newpath);
4209     /* check for a database.unignore file */
4210     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4211     *unignore = file_exists(newpath);
4212 }
4213
4214 static struct search_roots_ll {
4215     const char *path;
4216     struct search_roots_ll * next;
4217 } roots_ll;
4218
4219 #ifdef APPLICATION
4220 /*
4221  * This adds a path to the search roots, possibly during traveling through
4222  * the filesystem. It only adds if the path is not inside an already existing
4223  * search root.
4224  *
4225  * Returns true if it added the path to the search roots
4226  *
4227  * Windows 2000 and greater supports symlinks, but they don't provide
4228  * realpath() or readlink(), and symlinks are rarely used on them so
4229  * ignore this for windows for now
4230  **/
4231 static bool add_search_root(const char *name)
4232 {
4233     (void)name;
4234 #ifndef WIN32
4235     struct search_roots_ll *this, *prev = NULL;
4236     char target[MAX_PATH];
4237     /* Okay, realpath() is almost completely broken on android
4238      *
4239      * It doesn't accept NULL for resolved_name to dynamically allocate
4240      * the resulting path; and it assumes resolved_name to be PATH_MAX
4241      * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4242      * and blindly writes to the end if it
4243      *
4244      * therefore use sufficiently large static storage here
4245      * Note that PATH_MAX != MAX_PATH
4246      **/
4247     static char abs_target[PATH_MAX];
4248     ssize_t len;
4249
4250     len = readlink(name, target, sizeof(target));
4251     if (len < 0)
4252         return false;
4253
4254     target[len] = '\0';
4255     if (realpath(target, abs_target) == NULL)
4256         return false;
4257
4258     for(this = &roots_ll; this; prev = this, this = this->next)
4259     {
4260         size_t root_len = strlen(this->path);
4261         /* check if the link target is inside of an existing search root
4262          * don't add if target is inside, we'll scan it later */
4263         if (!strncmp(this->path, abs_target, root_len))
4264             return false;
4265     }
4266
4267     if (prev)
4268     {
4269         size_t len = strlen(abs_target) + 1; /* count \0 */
4270         this = malloc(sizeof(struct search_roots_ll) + len );
4271         if (!this || len > MAX_PATH)
4272         {
4273             logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4274             free(this);
4275             prev->next = NULL;
4276             return false;
4277         }
4278         this->path = ((char*)this) + sizeof(struct search_roots_ll);
4279         strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4280         this->next = NULL;
4281         prev->next = this;
4282         logf("Added %s to the search roots\n", abs_target);
4283         return true;
4284     }
4285 #endif
4286     return false;
4287 }
4288
4289 static int free_search_roots(struct search_roots_ll * start)
4290 {
4291     int ret = 0;
4292     if (start->next)
4293     {
4294         ret += free_search_roots(start->next);
4295         ret += sizeof(struct search_roots_ll);
4296         free(start->next);
4297     }
4298     return ret;
4299 }
4300 #else /* native, simulator */
4301 #define add_search_root(a) do {} while(0)
4302 #define free_search_roots(a) do {} while(0)
4303 #endif
4304
4305 static bool check_dir(const char *dirname, int add_files)
4306 {
4307     DIR *dir;
4308     int len;
4309     int success = false;
4310     int ignore, unignore;
4311
4312     dir = opendir(dirname);
4313     if (!dir)
4314     {
4315         logf("tagcache: opendir(%s) failed", dirname);
4316         return false;
4317     }
4318     /* check for a database.ignore and database.unignore */
4319     check_ignore(dirname, &ignore, &unignore);
4320
4321     /* don't do anything if both ignore and unignore are there */
4322     if (ignore != unignore)
4323         add_files = unignore;
4324
4325     /* Recursively scan the dir. */
4326 #ifdef __PCTOOL__
4327     while (1)
4328 #else
4329     while (!check_event_queue())
4330 #endif
4331     {
4332         struct dirent *entry = readdir(dir);
4333         if (entry == NULL)
4334         {
4335             success = true;
4336             break;
4337         }
4338
4339         if (!strcmp((char *)entry->d_name, ".") ||
4340             !strcmp((char *)entry->d_name, ".."))
4341             continue;
4342
4343         struct dirinfo info = dir_get_info(dir, entry);
4344
4345         yield();
4346
4347         len = strlen(curpath);
4348         /* don't add an extra / for curpath == / */
4349         if (len <= 1) len = 0;
4350         snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4351
4352         processed_dir_count++;
4353         if (info.attribute & ATTR_DIRECTORY)
4354 #ifndef SIMULATOR
4355         {   /* don't follow symlinks to dirs, but try to add it as a search root
4356              * this makes able to avoid looping in recursive symlinks */
4357             if (info.attribute & ATTR_LINK)
4358                 add_search_root(curpath);
4359             else
4360                 check_dir(curpath, add_files);
4361         }
4362 #else
4363             check_dir(curpath, add_files);
4364 #endif
4365         else if (add_files)
4366         {
4367             tc_stat.curentry = curpath;
4368
4369             /* Add a new entry to the temporary db file. */
4370             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4371 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4372                          , dir->internal_entry
4373 #endif
4374                          );
4375
4376             /* Wait until current path for debug screen is read and unset. */
4377             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4378                 yield();
4379
4380             tc_stat.curentry = NULL;
4381         }
4382
4383         curpath[len] = '\0';
4384     }
4385
4386     closedir(dir);
4387
4388     return success;
4389 }
4390
4391 void tagcache_screensync_event(void)
4392 {
4393     tc_stat.curentry = NULL;
4394 }
4395
4396 void tagcache_screensync_enable(bool state)
4397 {
4398     tc_stat.syncscreen = state;
4399 }
4400
4401 void tagcache_build(const char *path)
4402 {
4403     struct tagcache_header header;
4404     bool ret;
4405
4406     curpath[0] = '\0';
4407     data_size = 0;
4408     total_entry_count = 0;
4409     processed_dir_count = 0;
4410
4411 #ifdef HAVE_DIRCACHE
4412     while (dircache_is_initializing())
4413         sleep(1);
4414 #endif
4415
4416     logf("updating tagcache");
4417
4418     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4419     if (cachefd >= 0)
4420     {
4421         logf("skipping, cache already waiting for commit");
4422         close(cachefd);
4423         return ;
4424     }
4425
4426     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4427     if (cachefd < 0)
4428     {
4429         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4430         return ;
4431     }
4432
4433     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4434
4435     cpu_boost(true);
4436
4437     logf("Scanning files...");
4438     /* Scan for new files. */
4439     memset(&header, 0, sizeof(struct tagcache_header));
4440     write(cachefd, &header, sizeof(struct tagcache_header));
4441
4442     ret = true;
4443     roots_ll.path = path;
4444     roots_ll.next = NULL;
4445     struct search_roots_ll * this;
4446     /* check_dir might add new roots */
4447     for(this = &roots_ll; this; this = this->next)
4448     {
4449         strcpy(curpath, this->path);
4450         ret = ret && check_dir(this->path, true);
4451     }
4452     if (roots_ll.next)
4453         free_search_roots(roots_ll.next);
4454
4455     /* Write the header. */
4456     header.magic = TAGCACHE_MAGIC;
4457     header.datasize = data_size;
4458     header.entry_count = total_entry_count;
4459     lseek(cachefd, 0, SEEK_SET);
4460     write(cachefd, &header, sizeof(struct tagcache_header));
4461     close(cachefd);
4462
4463     if (filenametag_fd >= 0)
4464     {
4465         close(filenametag_fd);
4466         filenametag_fd = -1;
4467     }
4468
4469     if (!ret)
4470     {
4471         logf("Aborted.");
4472         cpu_boost(false);
4473         return ;
4474     }
4475
4476     /* Commit changes to the database. */
4477 #ifdef __PCTOOL__
4478     allocate_tempbuf();
4479 #endif
4480     if (commit())
4481     {
4482         remove(TAGCACHE_FILE_TEMP);
4483         logf("tagcache built!");
4484     }
4485 #ifdef __PCTOOL__
4486     free_tempbuf();
4487 #endif
4488
4489 #ifdef HAVE_TC_RAMCACHE
4490     if (hdr)
4491     {
4492         /* Import runtime statistics if we just initialized the db. */
4493         if (hdr->h.serial == 0)
4494             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4495     }
4496 #endif
4497
4498     cpu_boost(false);
4499 }
4500
4501 #ifdef HAVE_TC_RAMCACHE
4502 static void load_ramcache(void)
4503 {
4504     if (!hdr)
4505         return ;
4506
4507     cpu_boost(true);
4508
4509     /* At first we should load the cache (if exists). */
4510     tc_stat.ramcache = load_tagcache();
4511
4512     if (!tc_stat.ramcache)
4513     {
4514         /* If loading failed, it must indicate some problem with the db
4515          * so disable it entirely to prevent further issues. */
4516         tc_stat.ready = false;
4517         hdr = NULL;
4518     }
4519
4520     cpu_boost(false);
4521 }
4522
4523 void tagcache_unload_ramcache(void)
4524 {
4525     tc_stat.ramcache = false;
4526     /* Just to make sure there is no statefile present. */
4527     // remove(TAGCACHE_STATEFILE);
4528 }
4529 #endif
4530
4531 #ifndef __PCTOOL__
4532 static void tagcache_thread(void)
4533 {
4534     struct queue_event ev;
4535     bool check_done = false;
4536
4537     /* If the previous cache build/update was interrupted, commit
4538      * the changes first in foreground. */
4539     cpu_boost(true);
4540     allocate_tempbuf();
4541     commit();
4542     free_tempbuf();
4543
4544 #ifdef HAVE_TC_RAMCACHE
4545 # ifdef HAVE_EEPROM_SETTINGS
4546     if (firmware_settings.initialized && firmware_settings.disk_clean
4547         && global_settings.tagcache_ram)
4548     {
4549         check_done = tagcache_dumpload();
4550     }
4551
4552     remove(TAGCACHE_STATEFILE);
4553 # endif
4554
4555     /* Allocate space for the tagcache if found on disk. */
4556     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4557         allocate_tagcache();
4558 #endif
4559
4560     cpu_boost(false);
4561     tc_stat.initialized = true;
4562
4563     /* Don't delay bootup with the header check but do it on background. */
4564     if (!tc_stat.ready)
4565     {
4566         sleep(HZ);
4567         tc_stat.ready = check_all_headers();
4568         tc_stat.readyvalid = true;
4569     }
4570
4571     while (1)
4572     {
4573         run_command_queue(false);
4574
4575         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4576
4577         switch (ev.id)
4578         {
4579             case Q_IMPORT_CHANGELOG:
4580                 tagcache_import_changelog();
4581                 break;
4582
4583             case Q_REBUILD:
4584                 remove_files();
4585                 remove(TAGCACHE_FILE_TEMP);
4586                 tagcache_build("/");
4587                 break;
4588
4589             case Q_UPDATE:
4590                 tagcache_build("/");
4591 #ifdef HAVE_TC_RAMCACHE
4592                 load_ramcache();
4593 #endif
4594                 check_deleted_files();
4595                 break ;
4596
4597             case Q_START_SCAN:
4598                 check_done = false;
4599             case SYS_TIMEOUT:
4600                 if (check_done || !tc_stat.ready)
4601                     break ;
4602
4603 #ifdef HAVE_TC_RAMCACHE
4604                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4605                 {
4606                     load_ramcache();
4607                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4608                         tagcache_build("/");
4609                 }
4610                 else
4611 #endif
4612                 if (global_settings.tagcache_autoupdate)
4613                 {
4614                     tagcache_build("/");
4615
4616                     /* This will be very slow unless dircache is enabled
4617                        or target is flash based, but do it anyway for
4618                        consistency. */
4619                     check_deleted_files();
4620                 }
4621
4622                 logf("tagcache check done");
4623
4624                 check_done = true;
4625                 break ;
4626
4627             case Q_STOP_SCAN:
4628                 break ;
4629
4630             case SYS_POWEROFF:
4631                 break ;
4632
4633 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4634             case SYS_USB_CONNECTED:
4635                 logf("USB: TagCache");
4636                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4637                 usb_wait_for_disconnect(&tagcache_queue);
4638                 break ;
4639 #endif
4640         }
4641     }
4642 }
4643
4644 bool tagcache_prepare_shutdown(void)
4645 {
4646     if (tagcache_get_commit_step() > 0)
4647         return false;
4648
4649     tagcache_stop_scan();
4650     while (read_lock || write_lock)
4651         sleep(1);
4652
4653     return true;
4654 }
4655
4656 void tagcache_shutdown(void)
4657 {
4658     /* Flush the command queue. */
4659     run_command_queue(true);
4660
4661 #ifdef HAVE_EEPROM_SETTINGS
4662     if (tc_stat.ramcache)
4663         tagcache_dumpsave();
4664 #endif
4665 }
4666
4667 static int get_progress(void)
4668 {
4669     int total_count = -1;
4670
4671 #ifdef HAVE_DIRCACHE
4672     if (dircache_is_enabled())
4673     {
4674         total_count = dircache_get_entry_count();
4675     }
4676     else
4677 #endif
4678 #ifdef HAVE_TC_RAMCACHE
4679     {
4680         if (hdr && tc_stat.ramcache)
4681             total_count = hdr->h.tch.entry_count;
4682     }
4683 #endif
4684
4685     if (total_count < 0)
4686         return -1;
4687
4688     return processed_dir_count * 100 / total_count;
4689 }
4690
4691 struct tagcache_stat* tagcache_get_stat(void)
4692 {
4693     tc_stat.progress = get_progress();
4694     tc_stat.processed_entries = processed_dir_count;
4695
4696     return &tc_stat;
4697 }
4698
4699 void tagcache_start_scan(void)
4700 {
4701     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4702 }
4703
4704 bool tagcache_update(void)
4705 {
4706     if (!tc_stat.ready)
4707         return false;
4708
4709     queue_post(&tagcache_queue, Q_UPDATE, 0);
4710     return false;
4711 }
4712
4713 bool tagcache_rebuild()
4714 {
4715     queue_post(&tagcache_queue, Q_REBUILD, 0);
4716     return false;
4717 }
4718
4719 void tagcache_stop_scan(void)
4720 {
4721     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4722 }
4723
4724 #ifdef HAVE_TC_RAMCACHE
4725 bool tagcache_is_ramcache(void)
4726 {
4727     return tc_stat.ramcache;
4728 }
4729 #endif
4730
4731 #endif /* !__PCTOOL__ */
4732
4733
4734 void tagcache_init(void)
4735 {
4736     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4737     memset(&current_tcmh, 0, sizeof(struct master_header));
4738     filenametag_fd = -1;
4739     write_lock = read_lock = 0;
4740
4741 #ifndef __PCTOOL__
4742     mutex_init(&command_queue_mutex);
4743     queue_init(&tagcache_queue, true);
4744     create_thread(tagcache_thread, tagcache_stack,
4745                   sizeof(tagcache_stack), 0, tagcache_thread_name
4746                   IF_PRIO(, PRIORITY_BACKGROUND)
4747                   IF_COP(, CPU));
4748 #else
4749     tc_stat.initialized = true;
4750     allocate_tempbuf();
4751     commit();
4752     free_tempbuf();
4753     tc_stat.ready = check_all_headers();
4754 #endif
4755 }
4756
4757 #ifdef __PCTOOL__
4758 void tagcache_reverse_scan(void)
4759 {
4760     logf("Checking for deleted files");
4761     check_deleted_files();
4762 }
4763 #endif
4764
4765 bool tagcache_is_initialized(void)
4766 {
4767     return tc_stat.initialized;
4768 }
4769 bool tagcache_is_fully_initialized(void)
4770 {
4771     return tc_stat.readyvalid;
4772 }
4773 bool tagcache_is_usable(void)
4774 {
4775     return tc_stat.initialized && tc_stat.ready;
4776 }
4777 int tagcache_get_commit_step(void)
4778 {
4779     return tc_stat.commit_step;
4780 }
4781 int tagcache_get_max_commit_step(void)
4782 {
4783     return (int)(SORTED_TAGS_COUNT)+1;
4784 }
4785