apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #ifdef APPLICATION
  64 #include <unistd.h> /* readlink() */
  65 #include <limits.h> /* PATH_MAX */
  66 #endif
  67 #include "config.h"
  68 #include "ata_idle_notify.h"
  69 #include "thread.h"
  70 #include "kernel.h"
  71 #include "system.h"
  72 #include "logf.h"
  73 #include "string-extra.h"
  74 #include "usb.h"
  75 #include "metadata.h"
  76 #include "tagcache.h"
  77 #include "buffer.h"
  78 #include "crc32.h"
  79 #include "misc.h"
  80 #include "settings.h"
  81 #include "dir.h"
  82 #include "filefuncs.h"
  83 #include "structec.h"
  84 #include "debug.h"
  85
  86 #ifndef __PCTOOL__
  87 #include "lang.h"
  88 #include "eeprom_settings.h"
  89 #endif
  90
  91 #ifdef __PCTOOL__
  92 #define yield() do { } while(0)
  93 #define sim_sleep(timeout) do { } while(0)
  94 #define do_timed_yield() do { } while(0)
  95 #endif
  96
  97 #ifndef __PCTOOL__
  98 /* Tag Cache thread. */
  99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
 100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
 101 static const char tagcache_thread_name[] = "tagcache";
 102 #endif
 103
 104 /* Previous path when scanning directory tree recursively. */
 105 static char curpath[TAG_MAXLEN+32];
 106
 107 /* Used when removing duplicates. */
 108 static char *tempbuf;     /* Allocated when needed. */
 109 static long tempbufidx;   /* Current location in buffer. */
 110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 111 static long tempbuf_left; /* Buffer space left. */
 112 static long tempbuf_pos;
 113
 114 #define SORTED_TAGS_COUNT 8
 115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 118     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 119 /* Tags we want to get sorted (loaded to the tempbuf). */
 120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 123
 124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 126     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 127     (1LU << tag_albumartist) | (1LU << tag_grouping))
 128
 129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 130 static const char *tags_str[] = { "artist", "album", "genre", "title",
 131     "filename", "composer", "comment", "albumartist", "grouping", "year",
 132     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 133     "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
 134
 135 /* Status information of the tagcache. */
 136 static struct tagcache_stat tc_stat;
 137
 138 /* Queue commands. */
 139 enum tagcache_queue {
 140     Q_STOP_SCAN = 0,
 141     Q_START_SCAN,
 142     Q_IMPORT_CHANGELOG,
 143     Q_UPDATE,
 144     Q_REBUILD,
 145
 146     /* Internal tagcache command queue. */
 147     CMD_UPDATE_MASTER_HEADER,
 148     CMD_UPDATE_NUMERIC,
 149 };
 150
 151 struct tagcache_command_entry {
 152     int32_t command;
 153     int32_t idx_id;
 154     int32_t tag;
 155     int32_t data;
 156 };
 157
 158 #ifndef __PCTOOL__
 159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 160 static volatile int command_queue_widx = 0;
 161 static volatile int command_queue_ridx = 0;
 162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
 163 #endif
 164
 165 /* Tag database structures. */
 166
 167 /* Variable-length tag entry in tag files. */
 168 struct tagfile_entry {
 169     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 170     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 171     char tag_data[0];  /* Begin of the tag data */
 172 };
 173
 174 /* Fixed-size tag entry in master db index. */
 175 struct index_entry {
 176     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 177     int32_t flag;                /* Status flags */
 178 };
 179
 180 /* Header is the same in every file. */
 181 struct tagcache_header {
 182     int32_t magic;       /* Header version number */
 183     int32_t datasize;    /* Data size in bytes */
 184     int32_t entry_count; /* Number of entries in this file */
 185 };
 186
 187 struct master_header {
 188     struct tagcache_header tch;
 189     int32_t serial; /* Increasing counting number */
 190     int32_t commitid; /* Number of commits so far */
 191     int32_t dirty;
 192 };
 193
 194 /* For the endianess correction */
 195 static const char *tagfile_entry_ec   = "ll";
 196 /**
 197  Note: This should be (1 + TAG_COUNT) amount of l's.
 198  */
 199 static const char *index_entry_ec     = "llllllllllllllllllllll";
 200
 201 static const char *tagcache_header_ec = "lll";
 202 static const char *master_header_ec   = "llllll";
 203
 204 static struct master_header current_tcmh;
 205
 206 #ifdef HAVE_TC_RAMCACHE
 207 /* Header is created when loading database to ram. */
 208 struct ramcache_header {
 209     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 210     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 211     struct index_entry indices[0]; /* Master index file content */
 212 };
 213
 214 # ifdef HAVE_EEPROM_SETTINGS
 215 struct statefile_header {
 216     int32_t magic;                /* Statefile version number */
 217     struct master_header mh;      /* Header from the master index */
 218     struct ramcache_header *hdr;  /* Old load address of hdr for relocation */
 219     struct tagcache_stat tc_stat;
 220 };
 221 # endif
 222
 223 /* Pointer to allocated ramcache_header */
 224 static struct ramcache_header *ramcache_hdr;
 225 #endif
 226
 227 /**
 228  * Full tag entries stored in a temporary file waiting
 229  * for commit to the cache. */
 230 struct temp_file_entry {
 231     long tag_offset[TAG_COUNT];
 232     short tag_length[TAG_COUNT];
 233     long flag;
 234
 235     long data_length;
 236 };
 237
 238 struct tempbuf_id_list {
 239     long id;
 240     struct tempbuf_id_list *next;
 241 };
 242
 243 struct tempbuf_searchidx {
 244     long idx_id;
 245     char *str;
 246     int seek;
 247     struct tempbuf_id_list idlist;
 248 };
 249
 250 /* Lookup buffer for fixing messed up index while after sorting. */
 251 static long commit_entry_count;
 252 static long lookup_buffer_depth;
 253 static struct tempbuf_searchidx **lookup;
 254
 255 /* Used when building the temporary file. */
 256 static int cachefd = -1, filenametag_fd;
 257 static int total_entry_count = 0;
 258 static int data_size = 0;
 259 static int processed_dir_count;
 260
 261 /* Thread safe locking */
 262 static volatile int write_lock;
 263 static volatile int read_lock;
 264
 265 static bool delete_entry(long idx_id);
 266
 267 const char* tagcache_tag_to_str(int tag)
 268 {
 269     return tags_str[tag];
 270 }
 271
 272 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 273 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 274 {
 275     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 276 }
 277
 278 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 279 {
 280     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 281 }
 282
 283 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 284 {
 285     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 286 }
 287
 288 #ifdef HAVE_DIRCACHE
 289 /**
 290  * Returns true if specified flag is still present, i.e., dircache
 291  * has not been reloaded.
 292  */
 293 static bool is_dircache_intact(void)
 294 {
 295     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 296 }
 297 #endif
 298
 299 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 300 {
 301     int fd;
 302     char buf[MAX_PATH];
 303     int rc;
 304
 305     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 306         return -1;
 307
 308     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 309
 310     fd = open(buf, write ? O_RDWR : O_RDONLY);
 311     if (fd < 0)
 312     {
 313         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 314         tc_stat.ready = false;
 315         return fd;
 316     }
 317
 318     /* Check the header. */
 319     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 320     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 321     {
 322         logf("header error");
 323         tc_stat.ready = false;
 324         close(fd);
 325         return -2;
 326     }
 327
 328     return fd;
 329 }
 330
 331 static int open_master_fd(struct master_header *hdr, bool write)
 332 {
 333     int fd;
 334     int rc;
 335
 336     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 337     if (fd < 0)
 338     {
 339         logf("master file open failed for R/W");
 340         tc_stat.ready = false;
 341         return fd;
 342     }
 343
 344     tc_stat.econ = false;
 345
 346     /* Check the header. */
 347     rc = read(fd, hdr, sizeof(struct master_header));
 348     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 349     {
 350         /* Success. */
 351         return fd;
 352     }
 353
 354     /* Trying to read again, this time with endianess correction enabled. */
 355     lseek(fd, 0, SEEK_SET);
 356
 357     rc = ecread(fd, hdr, 1, master_header_ec, true);
 358     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 359     {
 360         logf("header error");
 361         tc_stat.ready = false;
 362         close(fd);
 363         return -2;
 364     }
 365
 366     tc_stat.econ = true;
 367
 368     return fd;
 369 }
 370
 371 #ifndef __PCTOOL__
 372 static bool do_timed_yield(void)
 373 {
 374     /* Sorting can lock up for quite a while, so yield occasionally */
 375     static long wakeup_tick = 0;
 376     if (TIME_AFTER(current_tick, wakeup_tick))
 377     {
 378         wakeup_tick = current_tick + (HZ/4);
 379         yield();
 380         return true;
 381     }
 382     return false;
 383 }
 384 #endif
 385
 386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 387 /* find the ramcache entry corresponding to the file indicated by
 388  * filename and dc (it's corresponding dircache id). */
 389 static long find_entry_ram(const char *filename, int dc)
 390 {
 391     static long last_pos = 0;
 392     int i;
 393
 394     /* Check if tagcache is loaded into ram. */
 395     if (!tc_stat.ramcache || !is_dircache_intact())
 396         return -1;
 397
 398     if (dc < 0)
 399         dc = dircache_get_entry_id(filename);
 400
 401     if (dc < 0)
 402     {
 403         logf("tagcache: file not found.");
 404         return -1;
 405     }
 406
 407     try_again:
 408
 409     if (last_pos > 0)
 410         i = last_pos;
 411     else
 412         i = 0;
 413
 414     for (; i < current_tcmh.tch.entry_count; i++)
 415     {
 416         if (ramcache_hdr->indices[i].tag_seek[tag_filename] == dc)
 417         {
 418             last_pos = MAX(0, i - 3);
 419             return i;
 420         }
 421
 422         do_timed_yield();
 423     }
 424
 425     if (last_pos > 0)
 426     {
 427         last_pos = 0;
 428         goto try_again;
 429     }
 430
 431     return -1;
 432 }
 433 #endif
 434
 435 static long find_entry_disk(const char *filename_raw, bool localfd)
 436 {
 437     struct tagcache_header tch;
 438     static long last_pos = -1;
 439     long pos_history[POS_HISTORY_COUNT];
 440     long pos_history_idx = 0;
 441     bool found = false;
 442     struct tagfile_entry tfe;
 443     int fd;
 444     char buf[TAG_MAXLEN+32];
 445     int i;
 446     int pos = -1;
 447
 448     const char *filename = filename_raw;
 449 #ifdef APPLICATION
 450     char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
 451     if (realpath(filename, pathbuf) == pathbuf)
 452         filename = pathbuf;
 453 #endif
 454
 455     if (!tc_stat.ready)
 456         return -2;
 457
 458     fd = filenametag_fd;
 459     if (fd < 0 || localfd)
 460     {
 461         last_pos = -1;
 462         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 463             return -1;
 464     }
 465
 466     check_again:
 467
 468     if (last_pos > 0)
 469         lseek(fd, last_pos, SEEK_SET);
 470     else
 471         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 472
 473     while (true)
 474     {
 475         pos = lseek(fd, 0, SEEK_CUR);
 476         for (i = pos_history_idx-1; i >= 0; i--)
 477             pos_history[i+1] = pos_history[i];
 478         pos_history[0] = pos;
 479
 480         if (ecread_tagfile_entry(fd, &tfe)
 481             != sizeof(struct tagfile_entry))
 482         {
 483             break ;
 484         }
 485
 486         if (tfe.tag_length >= (long)sizeof(buf))
 487         {
 488             logf("too long tag #1");
 489             close(fd);
 490             if (!localfd)
 491                 filenametag_fd = -1;
 492             last_pos = -1;
 493             return -2;
 494         }
 495
 496         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 497         {
 498             logf("read error #2");
 499             close(fd);
 500             if (!localfd)
 501                 filenametag_fd = -1;
 502             last_pos = -1;
 503             return -3;
 504         }
 505
 506         if (!strcmp(filename, buf))
 507         {
 508             last_pos = pos_history[pos_history_idx];
 509             found = true;
 510             break ;
 511         }
 512
 513         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 514             pos_history_idx++;
 515     }
 516
 517     /* Not found? */
 518     if (!found)
 519     {
 520         if (last_pos > 0)
 521         {
 522             last_pos = -1;
 523             logf("seek again");
 524             goto check_again;
 525         }
 526
 527         if (fd != filenametag_fd || localfd)
 528             close(fd);
 529         return -4;
 530     }
 531
 532     if (fd != filenametag_fd || localfd)
 533         close(fd);
 534
 535     return tfe.idx_id;
 536 }
 537
 538 static int find_index(const char *filename)
 539 {
 540     long idx_id = -1;
 541
 542 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 543     idx_id = find_entry_ram(filename, -1);
 544 #endif
 545
 546     if (idx_id < 0)
 547         idx_id = find_entry_disk(filename, true);
 548
 549     return idx_id;
 550 }
 551
 552 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 553 {
 554     int idx_id;
 555
 556     if (!tc_stat.ready)
 557         return false;
 558
 559     idx_id = find_index(filename);
 560     if (idx_id < 0)
 561         return false;
 562
 563     if (!tagcache_search(tcs, tag_filename))
 564         return false;
 565
 566     tcs->entry_count = 0;
 567     tcs->idx_id = idx_id;
 568
 569     return true;
 570 }
 571
 572 static bool get_index(int masterfd, int idxid,
 573                       struct index_entry *idx, bool use_ram)
 574 {
 575     bool localfd = false;
 576
 577     if (idxid < 0)
 578     {
 579         logf("Incorrect idxid: %d", idxid);
 580         return false;
 581     }
 582
 583 #ifdef HAVE_TC_RAMCACHE
 584     if (tc_stat.ramcache && use_ram)
 585     {
 586         if (ramcache_hdr->indices[idxid].flag & FLAG_DELETED)
 587             return false;
 588
 589 # ifdef HAVE_DIRCACHE
 590         if (!(ramcache_hdr->indices[idxid].flag & FLAG_DIRCACHE)
 591             || is_dircache_intact())
 592 #endif
 593         {
 594             memcpy(idx, &ramcache_hdr->indices[idxid], sizeof(struct index_entry));
 595             return true;
 596         }
 597     }
 598 #else
 599     (void)use_ram;
 600 #endif
 601
 602     if (masterfd < 0)
 603     {
 604         struct master_header tcmh;
 605
 606         localfd = true;
 607         masterfd = open_master_fd(&tcmh, false);
 608         if (masterfd < 0)
 609             return false;
 610     }
 611
 612     lseek(masterfd, idxid * sizeof(struct index_entry)
 613           + sizeof(struct master_header), SEEK_SET);
 614     if (ecread_index_entry(masterfd, idx)
 615         != sizeof(struct index_entry))
 616     {
 617         logf("read error #3");
 618         if (localfd)
 619             close(masterfd);
 620
 621         return false;
 622     }
 623
 624     if (localfd)
 625         close(masterfd);
 626
 627     if (idx->flag & FLAG_DELETED)
 628         return false;
 629
 630     return true;
 631 }
 632
 633 #ifndef __PCTOOL__
 634
 635 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 636 {
 637     /* We need to exclude all memory only flags & tags when writing to disk. */
 638     if (idx->flag & FLAG_DIRCACHE)
 639     {
 640         logf("memory only flags!");
 641         return false;
 642     }
 643
 644 #ifdef HAVE_TC_RAMCACHE
 645     /* Only update numeric data. Writing the whole index to RAM by memcpy
 646      * destroys dircache pointers!
 647      */
 648     if (tc_stat.ramcache)
 649     {
 650         int tag;
 651         struct index_entry *idx_ram = &ramcache_hdr->indices[idxid];
 652
 653         for (tag = 0; tag < TAG_COUNT; tag++)
 654         {
 655             if (TAGCACHE_IS_NUMERIC(tag))
 656             {
 657                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 658             }
 659         }
 660
 661         /* Don't touch the dircache flag or attributes. */
 662         idx_ram->flag = (idx->flag & 0x0000ffff)
 663             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 664     }
 665 #endif
 666
 667     lseek(masterfd, idxid * sizeof(struct index_entry)
 668           + sizeof(struct master_header), SEEK_SET);
 669     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 670     {
 671         logf("write error #3");
 672         logf("idxid: %d", idxid);
 673         return false;
 674     }
 675
 676     return true;
 677 }
 678
 679 #endif /* !__PCTOOL__ */
 680
 681 static bool open_files(struct tagcache_search *tcs, int tag)
 682 {
 683     if (tcs->idxfd[tag] < 0)
 684     {
 685         char fn[MAX_PATH];
 686
 687         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 688         tcs->idxfd[tag] = open(fn, O_RDONLY);
 689     }
 690
 691     if (tcs->idxfd[tag] < 0)
 692     {
 693         logf("File not open!");
 694         return false;
 695     }
 696
 697     return true;
 698 }
 699
 700 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 701                      int tag, char *buf, long size)
 702 {
 703     struct tagfile_entry tfe;
 704     long seek;
 705
 706     *buf = '\0';
 707
 708     if (TAGCACHE_IS_NUMERIC(tag))
 709         return false;
 710
 711     seek = idx->tag_seek[tag];
 712     if (seek < 0)
 713     {
 714         logf("Retrieve failed");
 715         return false;
 716     }
 717
 718 #ifdef HAVE_TC_RAMCACHE
 719     if (tcs->ramsearch)
 720     {
 721         struct tagfile_entry *ep;
 722
 723 # ifdef HAVE_DIRCACHE
 724         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 725             && is_dircache_intact())
 726         {
 727             /* for tag_filename, seek is a dircache index */
 728             dircache_copy_path(seek, buf, size);
 729             return true;
 730         }
 731         else
 732 # endif
 733         if (tag != tag_filename)
 734         {
 735             ep = (struct tagfile_entry *)&ramcache_hdr->tags[tag][seek];
 736             strlcpy(buf, ep->tag_data, size);
 737
 738             return true;
 739         }
 740     }
 741 #endif
 742
 743     if (!open_files(tcs, tag))
 744         return false;
 745
 746     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 747     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 748         != sizeof(struct tagfile_entry))
 749     {
 750         logf("read error #5");
 751         return false;
 752     }
 753
 754     if (tfe.tag_length >= size)
 755     {
 756         logf("too small buffer");
 757         return false;
 758     }
 759
 760     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 761         tfe.tag_length)
 762     {
 763         logf("read error #6");
 764         return false;
 765     }
 766
 767     buf[tfe.tag_length] = '\0';
 768
 769     return true;
 770 }
 771
 772 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 773
 774 static long find_tag(int tag, int idx_id, const struct index_entry *idx)
 775 {
 776 #ifndef __PCTOOL__
 777     if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag))
 778     {
 779         /* Attempt to find tag data through store-to-load forwarding in
 780            command queue */
 781         long result = -1;
 782
 783         mutex_lock(&command_queue_mutex);
 784
 785         int ridx = command_queue_widx;
 786
 787         while (ridx != command_queue_ridx)
 788         {
 789             if (--ridx < 0)
 790                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 791
 792             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 793                 && command_queue[ridx].idx_id == idx_id
 794                 && command_queue[ridx].tag == tag)
 795             {
 796                 result = command_queue[ridx].data;
 797                 break;
 798             }
 799         }
 800
 801         mutex_unlock(&command_queue_mutex);
 802
 803         if (result >= 0)
 804         {
 805             logf("find_tag: "
 806                  "Recovered tag %d value %lX from write queue",
 807                  tag, result);
 808             return result;
 809         }
 810     }
 811 #endif
 812
 813     return idx->tag_seek[tag];
 814 }
 815
 816
 817 static long check_virtual_tags(int tag, int idx_id,
 818                                const struct index_entry *idx)
 819 {
 820     long data = 0;
 821
 822     switch (tag)
 823     {
 824         case tag_virt_length_sec:
 825             data = (find_tag(tag_length, idx_id, idx)/1000) % 60;
 826             break;
 827
 828         case tag_virt_length_min:
 829             data = (find_tag(tag_length, idx_id, idx)/1000) / 60;
 830             break;
 831
 832         case tag_virt_playtime_sec:
 833             data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60;
 834             break;
 835
 836         case tag_virt_playtime_min:
 837             data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60;
 838             break;
 839
 840         case tag_virt_autoscore:
 841             if (find_tag(tag_length, idx_id, idx) == 0
 842                 || find_tag(tag_playcount, idx_id, idx) == 0)
 843             {
 844                 data = 0;
 845             }
 846             else
 847             {
 848                 /* A straight calculus gives:
 849                      autoscore = 100 * playtime / length / playcout (1)
 850                    Now, consider the euclidian division of playtime by length:
 851                      playtime = alpha * length + beta
 852                    With:
 853                      0 <= beta < length
 854                    Now, (1) becomes:
 855                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 856                    Both terms should be small enough to avoid any overflow
 857                 */
 858                 data = 100 * (find_tag(tag_playtime, idx_id, idx)
 859                               / find_tag(tag_length, idx_id, idx))
 860                        + (100 * (find_tag(tag_playtime, idx_id, idx)
 861                                  % find_tag(tag_length, idx_id, idx)))
 862                          / find_tag(tag_length, idx_id, idx);
 863                 data /=  find_tag(tag_playcount, idx_id, idx);
 864             }
 865             break;
 866
 867         /* How many commits before the file has been added to the DB. */
 868         case tag_virt_entryage:
 869             data = current_tcmh.commitid
 870                    - find_tag(tag_commitid, idx_id, idx) - 1;
 871             break;
 872
 873         case tag_virt_basename:
 874             tag = tag_filename; /* return filename; caller handles basename */
 875             /* FALLTHRU */
 876
 877         default:
 878             data = find_tag(tag, idx_id, idx);
 879     }
 880
 881     return data;
 882 }
 883
 884 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 885 {
 886     struct index_entry idx;
 887
 888     if (!tc_stat.ready)
 889         return false;
 890
 891     if (!TAGCACHE_IS_NUMERIC(tag))
 892         return -1;
 893
 894     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 895         return -2;
 896
 897     return check_virtual_tags(tag, tcs->idx_id, &idx);
 898 }
 899
 900 inline static bool str_ends_with(const char *str1, const char *str2)
 901 {
 902     int str_len = strlen(str1);
 903     int clause_len = strlen(str2);
 904
 905     if (clause_len > str_len)
 906         return false;
 907
 908     return !strcasecmp(&str1[str_len - clause_len], str2);
 909 }
 910
 911 inline static bool str_oneof(const char *str, const char *list)
 912 {
 913     const char *sep;
 914     int l, len = strlen(str);
 915
 916     while (*list)
 917     {
 918         sep = strchr(list, '|');
 919         l = sep ? (long)sep - (long)list : (int)strlen(list);
 920         if ((l==len) && !strncasecmp(str, list, len))
 921             return true;
 922         list += sep ? l + 1 : l;
 923     }
 924
 925     return false;
 926 }
 927
 928 static bool check_against_clause(long numeric, const char *str,
 929                                  const struct tagcache_search_clause *clause)
 930 {
 931     if (clause->numeric)
 932     {
 933         switch (clause->type)
 934         {
 935             case clause_is:
 936                 return numeric == clause->numeric_data;
 937             case clause_is_not:
 938                 return numeric != clause->numeric_data;
 939             case clause_gt:
 940                 return numeric > clause->numeric_data;
 941             case clause_gteq:
 942                 return numeric >= clause->numeric_data;
 943             case clause_lt:
 944                 return numeric < clause->numeric_data;
 945             case clause_lteq:
 946                 return numeric <= clause->numeric_data;
 947             default:
 948                 logf("Incorrect numeric tag: %d", clause->type);
 949         }
 950     }
 951     else
 952     {
 953         switch (clause->type)
 954         {
 955             case clause_is:
 956                 return !strcasecmp(clause->str, str);
 957             case clause_is_not:
 958                 return strcasecmp(clause->str, str);
 959             case clause_gt:
 960                 return 0>strcasecmp(clause->str, str);
 961             case clause_gteq:
 962                 return 0>=strcasecmp(clause->str, str);
 963             case clause_lt:
 964                 return 0<strcasecmp(clause->str, str);
 965             case clause_lteq:
 966                 return 0<=strcasecmp(clause->str, str);
 967             case clause_contains:
 968                 return (strcasestr(str, clause->str) != NULL);
 969             case clause_not_contains:
 970                 return (strcasestr(str, clause->str) == NULL);
 971             case clause_begins_with:
 972                 return (strcasestr(str, clause->str) == str);
 973             case clause_not_begins_with:
 974                 return (strcasestr(str, clause->str) != str);
 975             case clause_ends_with:
 976                 return str_ends_with(str, clause->str);
 977             case clause_not_ends_with:
 978                 return !str_ends_with(str, clause->str);
 979             case clause_oneof:
 980                 return str_oneof(str, clause->str);
 981
 982             default:
 983                 logf("Incorrect tag: %d", clause->type);
 984         }
 985     }
 986
 987     return false;
 988 }
 989
 990 static bool check_clauses(struct tagcache_search *tcs,
 991                           struct index_entry *idx,
 992                           struct tagcache_search_clause **clauses, int count)
 993 {
 994     int i;
 995
 996     /* Go through all conditional clauses. */
 997     for (i = 0; i < count; i++)
 998     {
 999         int seek;
1000         char buf[256];
1001         char *str = buf;
1002         struct tagcache_search_clause *clause = clauses[i];
1003
1004         if (clause->type == clause_logical_or)
1005             break; /* all conditions before logical-or satisfied --
1006                       stop processing clauses */
1007
1008         seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1009
1010 #ifdef HAVE_TC_RAMCACHE
1011         if (tcs->ramsearch)
1012         {
1013             struct tagfile_entry *tfe;
1014
1015             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1016             {
1017                 if (clause->tag == tag_filename
1018                     || clause->tag == tag_virt_basename)
1019                 {
1020                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1021                 }
1022                 else
1023                 {
1024                     tfe = (struct tagfile_entry *)
1025                                         &ramcache_hdr->tags[clause->tag][seek];
1026                     str = tfe->tag_data;
1027                 }
1028             }
1029         }
1030         else
1031 #endif
1032         {
1033             struct tagfile_entry tfe;
1034
1035             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1036             {
1037                 int tag = clause->tag;
1038                 if (tag == tag_virt_basename)
1039                     tag = tag_filename;
1040
1041                 int fd = tcs->idxfd[tag];
1042                 lseek(fd, seek, SEEK_SET);
1043                 ecread_tagfile_entry(fd, &tfe);
1044                 if (tfe.tag_length >= (int)sizeof(buf))
1045                 {
1046                     logf("Too long tag read!");
1047                     return false;
1048                 }
1049
1050                 read(fd, str, tfe.tag_length);
1051                 str[tfe.tag_length] = '\0';
1052
1053                 /* Check if entry has been deleted. */
1054                 if (str[0] == '\0')
1055                     return false;
1056             }
1057         }
1058
1059         if (clause->tag == tag_virt_basename)
1060         {
1061             char *basename = strrchr(str, '/');
1062             if (basename)
1063                 str = basename + 1;
1064         }
1065
1066         if (!check_against_clause(seek, str, clause))
1067         {
1068             /* Clause failed -- try finding a logical-or clause */
1069             while (++i < count)
1070             {
1071                 if (clauses[i]->type == clause_logical_or)
1072                     break;
1073             }
1074
1075             if (i < count)        /* Found logical-or? */
1076                 continue;         /* Check clauses after logical-or */
1077
1078             return false;
1079         }
1080     }
1081
1082     return true;
1083 }
1084
1085 bool tagcache_check_clauses(struct tagcache_search *tcs,
1086                             struct tagcache_search_clause **clause, int count)
1087 {
1088     struct index_entry idx;
1089
1090     if (count == 0)
1091         return true;
1092
1093     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1094         return false;
1095
1096     return check_clauses(tcs, &idx, clause, count);
1097 }
1098
1099 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1100 {
1101     int i;
1102
1103     /* If uniq buffer is not defined we must return true for search to work. */
1104     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1105                                      && !TAGCACHE_IS_NUMERIC(tcs->type)))
1106     {
1107         return true;
1108     }
1109
1110     for (i = 0; i < tcs->unique_list_count; i++)
1111     {
1112         /* Return false if entry is found. */
1113         if (tcs->unique_list[i] == id)
1114             return false;
1115     }
1116
1117     if (tcs->unique_list_count < tcs->unique_list_capacity)
1118     {
1119         tcs->unique_list[i] = id;
1120         tcs->unique_list_count++;
1121     }
1122
1123     return true;
1124 }
1125
1126 static bool build_lookup_list(struct tagcache_search *tcs)
1127 {
1128     struct index_entry entry;
1129     int i, j;
1130
1131     tcs->seek_list_count = 0;
1132
1133 #ifdef HAVE_TC_RAMCACHE
1134     if (tcs->ramsearch
1135 # ifdef HAVE_DIRCACHE
1136         && (tcs->type != tag_filename || is_dircache_intact())
1137 # endif
1138         )
1139     {
1140         for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++)
1141         {
1142             struct tagcache_seeklist_entry *seeklist;
1143             struct index_entry *idx = &ramcache_hdr->indices[i];
1144             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1145                 break ;
1146
1147             /* Skip deleted files. */
1148             if (idx->flag & FLAG_DELETED)
1149                 continue;
1150
1151             /* Go through all filters.. */
1152             for (j = 0; j < tcs->filter_count; j++)
1153             {
1154                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1155                 {
1156                     break ;
1157                 }
1158             }
1159
1160             if (j < tcs->filter_count)
1161                 continue ;
1162
1163             /* Check for conditions. */
1164             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1165                 continue;
1166
1167             /* Add to the seek list if not already in uniq buffer. */
1168             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1169                 continue;
1170
1171             /* Lets add it. */
1172             seeklist = &tcs->seeklist[tcs->seek_list_count];
1173             seeklist->seek = idx->tag_seek[tcs->type];
1174             seeklist->flag = idx->flag;
1175             seeklist->idx_id = i;
1176             tcs->seek_list_count++;
1177         }
1178
1179         tcs->seek_pos = i;
1180
1181         return tcs->seek_list_count > 0;
1182     }
1183 #endif
1184
1185     if (tcs->masterfd < 0)
1186     {
1187         struct master_header tcmh;
1188         tcs->masterfd = open_master_fd(&tcmh, false);
1189     }
1190
1191     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1192             sizeof(struct master_header), SEEK_SET);
1193
1194     while (ecread_index_entry(tcs->masterfd, &entry)
1195            == sizeof(struct index_entry))
1196     {
1197         struct tagcache_seeklist_entry *seeklist;
1198
1199         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1200             break ;
1201
1202         i = tcs->seek_pos;
1203         tcs->seek_pos++;
1204
1205         /* Check if entry has been deleted. */
1206         if (entry.flag & FLAG_DELETED)
1207             continue;
1208
1209         /* Go through all filters.. */
1210         for (j = 0; j < tcs->filter_count; j++)
1211         {
1212             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1213                 break ;
1214         }
1215
1216         if (j < tcs->filter_count)
1217             continue ;
1218
1219         /* Check for conditions. */
1220         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1221             continue;
1222
1223         /* Add to the seek list if not already in uniq buffer. */
1224         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1225             continue;
1226
1227         /* Lets add it. */
1228         seeklist = &tcs->seeklist[tcs->seek_list_count];
1229         seeklist->seek = entry.tag_seek[tcs->type];
1230         seeklist->flag = entry.flag;
1231         seeklist->idx_id = i;
1232         tcs->seek_list_count++;
1233
1234         yield();
1235     }
1236
1237     return tcs->seek_list_count > 0;
1238 }
1239
1240
1241 static void remove_files(void)
1242 {
1243     int i;
1244     char buf[MAX_PATH];
1245
1246     tc_stat.ready = false;
1247     tc_stat.ramcache = false;
1248     tc_stat.econ = false;
1249     remove(TAGCACHE_FILE_MASTER);
1250     for (i = 0; i < TAG_COUNT; i++)
1251     {
1252         if (TAGCACHE_IS_NUMERIC(i))
1253             continue;
1254
1255         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1256         remove(buf);
1257     }
1258 }
1259
1260
1261 static bool check_all_headers(void)
1262 {
1263     struct master_header myhdr;
1264     struct tagcache_header tch;
1265     int tag;
1266     int fd;
1267
1268     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1269         return false;
1270
1271     close(fd);
1272     if (myhdr.dirty)
1273     {
1274         logf("tagcache is dirty!");
1275         return false;
1276     }
1277
1278     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1279
1280     for (tag = 0; tag < TAG_COUNT; tag++)
1281     {
1282         if (TAGCACHE_IS_NUMERIC(tag))
1283             continue;
1284
1285         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1286             return false;
1287
1288         close(fd);
1289     }
1290
1291     return true;
1292 }
1293
1294 bool tagcache_is_busy(void)
1295 {
1296     return read_lock || write_lock;
1297 }
1298
1299 bool tagcache_search(struct tagcache_search *tcs, int tag)
1300 {
1301     struct tagcache_header tag_hdr;
1302     struct master_header   master_hdr;
1303     int i;
1304
1305     while (read_lock)
1306         sleep(1);
1307
1308     memset(tcs, 0, sizeof(struct tagcache_search));
1309     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1310         return false;
1311
1312     tcs->position = sizeof(struct tagcache_header);
1313     tcs->type = tag;
1314     tcs->seek_pos = 0;
1315     tcs->list_position = 0;
1316     tcs->seek_list_count = 0;
1317     tcs->filter_count = 0;
1318     tcs->masterfd = -1;
1319
1320     for (i = 0; i < TAG_COUNT; i++)
1321         tcs->idxfd[i] = -1;
1322
1323 #ifndef HAVE_TC_RAMCACHE
1324     tcs->ramsearch = false;
1325 #else
1326     tcs->ramsearch = tc_stat.ramcache;
1327     if (tcs->ramsearch)
1328     {
1329         tcs->entry_count = ramcache_hdr->entry_count[tcs->type];
1330     }
1331     else
1332 #endif
1333     {
1334         /* Always open as R/W so we can pass tcs to functions that modify data also
1335          * without failing. */
1336         tcs->masterfd = open_master_fd(&master_hdr, true);
1337         if (tcs->masterfd < 0)
1338             return false;
1339
1340         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1341         {
1342             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1343             if (tcs->idxfd[tcs->type] < 0)
1344                 return false;
1345
1346             tcs->entry_count = tag_hdr.entry_count;
1347         }
1348         else
1349         {
1350             tcs->entry_count = master_hdr.tch.entry_count;
1351         }
1352     }
1353
1354     tcs->valid = true;
1355     tcs->initialized = true;
1356     write_lock++;
1357
1358     return true;
1359 }
1360
1361 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1362                                  void *buffer, long length)
1363 {
1364     tcs->unique_list = (unsigned long *)buffer;
1365     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1366     tcs->unique_list_count = 0;
1367 }
1368
1369 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1370                                 int tag, int seek)
1371 {
1372     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1373         return false;
1374
1375     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1376         return false;
1377
1378     tcs->filter_tag[tcs->filter_count] = tag;
1379     tcs->filter_seek[tcs->filter_count] = seek;
1380     tcs->filter_count++;
1381
1382     return true;
1383 }
1384
1385 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1386                                 struct tagcache_search_clause *clause)
1387 {
1388     int i;
1389
1390     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1391     {
1392         logf("Too many clauses");
1393         return false;
1394     }
1395
1396     if (clause->type != clause_logical_or)
1397     {
1398         /* Check if there is already a similar filter in present (filters are
1399          * much faster than clauses).
1400          */
1401         for (i = 0; i < tcs->filter_count; i++)
1402         {
1403             if (tcs->filter_tag[i] == clause->tag)
1404                 return true;
1405         }
1406
1407         if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1408         {
1409             char buf[MAX_PATH];
1410
1411             snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1412             tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1413         }
1414     }
1415
1416     tcs->clause[tcs->clause_count] = clause;
1417     tcs->clause_count++;
1418
1419     return true;
1420 }
1421
1422 static bool get_next(struct tagcache_search *tcs)
1423 {
1424     static char buf[TAG_MAXLEN+32];
1425     struct tagfile_entry entry;
1426 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1427     long flag = 0;
1428 #endif
1429
1430     if (!tcs->valid || !tc_stat.ready)
1431         return false;
1432
1433     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1434 #ifdef HAVE_TC_RAMCACHE
1435         && !tcs->ramsearch
1436 #endif
1437         )
1438         return false;
1439
1440     /* Relative fetch. */
1441     if (tcs->filter_count > 0 || tcs->clause_count > 0
1442         || TAGCACHE_IS_NUMERIC(tcs->type)
1443 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1444         /* We need to retrieve flag status for dircache. */
1445         || (tcs->ramsearch && tcs->type == tag_filename)
1446 #endif
1447         )
1448     {
1449         struct tagcache_seeklist_entry *seeklist;
1450
1451         /* Check for end of list. */
1452         if (tcs->list_position == tcs->seek_list_count)
1453         {
1454             tcs->list_position = 0;
1455
1456             /* Try to fetch more. */
1457             if (!build_lookup_list(tcs))
1458             {
1459                 tcs->valid = false;
1460                 return false;
1461             }
1462         }
1463
1464         seeklist = &tcs->seeklist[tcs->list_position];
1465 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1466         flag = seeklist->flag;
1467 #endif
1468         tcs->position = seeklist->seek;
1469         tcs->idx_id = seeklist->idx_id;
1470         tcs->list_position++;
1471     }
1472     else
1473     {
1474         if (tcs->entry_count == 0)
1475         {
1476             tcs->valid = false;
1477             return false;
1478         }
1479
1480         tcs->entry_count--;
1481     }
1482
1483     tcs->result_seek = tcs->position;
1484
1485     if (TAGCACHE_IS_NUMERIC(tcs->type))
1486     {
1487         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1488         tcs->result = buf;
1489         tcs->result_len = strlen(buf) + 1;
1490         return true;
1491     }
1492
1493     /* Direct fetch. */
1494 #ifdef HAVE_TC_RAMCACHE
1495     if (tcs->ramsearch)
1496     {
1497
1498 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1499         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1500             && is_dircache_intact())
1501         {
1502             size_t len = dircache_copy_path(tcs->position, buf, sizeof buf);
1503             tcs->result_len = len + 1;
1504             tcs->result = buf;
1505             tcs->ramresult = false;
1506
1507             return true;
1508         }
1509         else
1510 #endif
1511         if (tcs->type != tag_filename)
1512         {
1513             struct tagfile_entry *ep;
1514
1515             ep = (struct tagfile_entry *)&ramcache_hdr->tags[tcs->type][tcs->position];
1516             tcs->result = ep->tag_data;
1517             tcs->result_len = strlen(tcs->result) + 1;
1518             tcs->idx_id = ep->idx_id;
1519             tcs->ramresult = true;
1520
1521             /* Increase position for the next run. This may get overwritten. */
1522             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1523
1524             return true;
1525         }
1526     }
1527 #endif
1528
1529     if (!open_files(tcs, tcs->type))
1530     {
1531         tcs->valid = false;
1532         return false;
1533     }
1534
1535     /* Seek stream to the correct position and continue to direct fetch. */
1536     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1537
1538     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1539     {
1540         logf("read error #5");
1541         tcs->valid = false;
1542         return false;
1543     }
1544
1545     if (entry.tag_length > (long)sizeof(buf))
1546     {
1547         tcs->valid = false;
1548         logf("too long tag #2");
1549         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1550         return false;
1551     }
1552
1553     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1554     {
1555         tcs->valid = false;
1556         logf("read error #4");
1557         return false;
1558     }
1559
1560     /**
1561      Update the position for the next read (this may be overridden
1562      if filters or clauses are being used).
1563      */
1564     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1565     tcs->result = buf;
1566     tcs->result_len = strlen(tcs->result) + 1;
1567     tcs->idx_id = entry.idx_id;
1568     tcs->ramresult = false;
1569
1570     return true;
1571 }
1572
1573 bool tagcache_get_next(struct tagcache_search *tcs)
1574 {
1575     while (get_next(tcs))
1576     {
1577         if (tcs->result_len > 1)
1578             return true;
1579     }
1580
1581     return false;
1582 }
1583
1584 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1585                        int tag, char *buf, long size)
1586 {
1587     struct index_entry idx;
1588
1589     *buf = '\0';
1590     if (!get_index(tcs->masterfd, idxid, &idx, true))
1591         return false;
1592
1593     return retrieve(tcs, &idx, tag, buf, size);
1594 }
1595
1596 static bool update_master_header(void)
1597 {
1598     struct master_header myhdr;
1599     int fd;
1600
1601     if (!tc_stat.ready)
1602         return false;
1603
1604     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1605         return false;
1606
1607     myhdr.serial = current_tcmh.serial;
1608     myhdr.commitid = current_tcmh.commitid;
1609     myhdr.dirty = current_tcmh.dirty;
1610
1611     /* Write it back */
1612     lseek(fd, 0, SEEK_SET);
1613     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1614     close(fd);
1615
1616     return true;
1617 }
1618
1619 void tagcache_search_finish(struct tagcache_search *tcs)
1620 {
1621     int i;
1622
1623     if (!tcs->initialized)
1624         return;
1625
1626     if (tcs->masterfd >= 0)
1627     {
1628         close(tcs->masterfd);
1629         tcs->masterfd = -1;
1630     }
1631
1632     for (i = 0; i < TAG_COUNT; i++)
1633     {
1634         if (tcs->idxfd[i] >= 0)
1635         {
1636             close(tcs->idxfd[i]);
1637             tcs->idxfd[i] = -1;
1638         }
1639     }
1640
1641     tcs->ramsearch = false;
1642     tcs->valid = false;
1643     tcs->initialized = 0;
1644     if (write_lock > 0)
1645         write_lock--;
1646 }
1647
1648 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1649 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1650 {
1651     return (struct tagfile_entry *)&ramcache_hdr->tags[tag][entry->tag_seek[tag]];
1652 }
1653
1654 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1655 {
1656     return check_virtual_tags(tag, idx_id, entry);
1657 }
1658
1659 static char* get_tag_string(const struct index_entry *entry, int tag)
1660 {
1661     char* s = get_tag(entry, tag)->tag_data;
1662     return strcmp(s, UNTAGGED) ? s : NULL;
1663 }
1664
1665 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1666 {
1667     struct index_entry *entry;
1668     int idx_id;
1669
1670     if (!tc_stat.ready || !tc_stat.ramcache)
1671         return false;
1672
1673     /* Find the corresponding entry in tagcache. */
1674     idx_id = find_entry_ram(filename, -1);
1675     if (idx_id < 0)
1676         return false;
1677
1678     entry = &ramcache_hdr->indices[idx_id];
1679
1680     memset(id3, 0, sizeof(struct mp3entry));
1681
1682     id3->title        = get_tag_string(entry, tag_title);
1683     id3->artist       = get_tag_string(entry, tag_artist);
1684     id3->album        = get_tag_string(entry, tag_album);
1685     id3->genre_string = get_tag_string(entry, tag_genre);
1686     id3->composer     = get_tag_string(entry, tag_composer);
1687     id3->comment      = get_tag_string(entry, tag_comment);
1688     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1689     id3->grouping     = get_tag_string(entry, tag_grouping);
1690
1691     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1692     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1693     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1694     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1695     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1696     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1697
1698     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1699     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1700     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1701     if (id3->bitrate == 0)
1702         id3->bitrate = 1;
1703
1704 #if CONFIG_CODEC == SWCODEC
1705     if (global_settings.autoresume_enable)
1706     {
1707         id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1708         logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1709              id3->title, id3->offset);
1710     }
1711 #endif
1712
1713     return true;
1714 }
1715 #endif
1716
1717 static inline void write_item(const char *item)
1718 {
1719     int len = strlen(item) + 1;
1720
1721     data_size += len;
1722     write(cachefd, item, len);
1723 }
1724
1725 static int check_if_empty(char **tag)
1726 {
1727     int length;
1728
1729     if (*tag == NULL || **tag == '\0')
1730     {
1731         *tag = UNTAGGED;
1732         return sizeof(UNTAGGED); /* Tag length */
1733     }
1734
1735     length = strlen(*tag);
1736     if (length > TAG_MAXLEN)
1737     {
1738         logf("over length tag: %s", *tag);
1739         length = TAG_MAXLEN;
1740         (*tag)[length] = '\0';
1741     }
1742
1743     return length + 1;
1744 }
1745
1746 #define ADD_TAG(entry,tag,data) \
1747     /* Adding tag */ \
1748     entry.tag_offset[tag] = offset; \
1749     entry.tag_length[tag] = check_if_empty(data); \
1750     offset += entry.tag_length[tag]
1751 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1752  * idea, as it uses lots of stack and is called from a recursive function
1753  * (check_dir).
1754  */
1755 static void __attribute__ ((noinline)) add_tagcache(char *path,
1756                                                     unsigned long mtime
1757 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1758                                                     ,int dc
1759 #endif
1760                                                    )
1761 {
1762     struct mp3entry id3;
1763     struct temp_file_entry entry;
1764     bool ret;
1765     int fd;
1766     int idx_id = -1;
1767     char tracknumfix[3];
1768     int offset = 0;
1769     int path_length = strlen(path);
1770     bool has_albumartist;
1771     bool has_grouping;
1772
1773 #ifdef SIMULATOR
1774     /* Crude logging for the sim - to aid in debugging */
1775     int logfd = open(ROCKBOX_DIR "/database.log",
1776                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1777     if (logfd >= 0) {
1778         write(logfd, path, strlen(path));
1779         write(logfd, "\n", 1);
1780         close(logfd);
1781     }
1782 #endif
1783
1784     if (cachefd < 0)
1785         return ;
1786
1787     /* Check for overlength file path. */
1788     if (path_length > TAG_MAXLEN)
1789     {
1790         /* Path can't be shortened. */
1791         logf("Too long path: %s", path);
1792         return ;
1793     }
1794
1795     /* Check if the file is supported. */
1796     if (probe_file_format(path) == AFMT_UNKNOWN)
1797         return ;
1798
1799     /* Check if the file is already cached. */
1800 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1801     idx_id = find_entry_ram(path, dc);
1802 #endif
1803
1804     /* Be sure the entry doesn't exist. */
1805     if (filenametag_fd >= 0 && idx_id < 0)
1806         idx_id = find_entry_disk(path, false);
1807
1808     /* Check if file has been modified. */
1809     if (idx_id >= 0)
1810     {
1811         struct index_entry idx;
1812
1813         /* TODO: Mark that the index exists (for fast reverse scan) */
1814         //found_idx[idx_id/8] |= idx_id%8;
1815
1816         if (!get_index(-1, idx_id, &idx, true))
1817         {
1818             logf("failed to retrieve index entry");
1819             return ;
1820         }
1821
1822         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1823         {
1824             /* No changes to file. */
1825             return ;
1826         }
1827
1828         /* Metadata might have been changed. Delete the entry. */
1829         logf("Re-adding: %s", path);
1830         if (!delete_entry(idx_id))
1831         {
1832             logf("delete_entry failed: %d", idx_id);
1833             return ;
1834         }
1835     }
1836
1837     fd = open(path, O_RDONLY);
1838     if (fd < 0)
1839     {
1840         logf("open fail: %s", path);
1841         return ;
1842     }
1843
1844     memset(&id3, 0, sizeof(struct mp3entry));
1845     memset(&entry, 0, sizeof(struct temp_file_entry));
1846     memset(&tracknumfix, 0, sizeof(tracknumfix));
1847     ret = get_metadata(&id3, fd, path);
1848     close(fd);
1849
1850     if (!ret)
1851         return ;
1852
1853     logf("-> %s", path);
1854
1855     if (id3.tracknum <= 0)              /* Track number missing? */
1856     {
1857         id3.tracknum = -1;
1858     }
1859
1860     /* Numeric tags */
1861     entry.tag_offset[tag_year] = id3.year;
1862     entry.tag_offset[tag_discnumber] = id3.discnum;
1863     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1864     entry.tag_offset[tag_length] = id3.length;
1865     entry.tag_offset[tag_bitrate] = id3.bitrate;
1866     entry.tag_offset[tag_mtime] = mtime;
1867
1868     /* String tags. */
1869     has_albumartist = id3.albumartist != NULL
1870         && strlen(id3.albumartist) > 0;
1871     has_grouping = id3.grouping != NULL
1872         && strlen(id3.grouping) > 0;
1873
1874     ADD_TAG(entry, tag_filename, &path);
1875     ADD_TAG(entry, tag_title, &id3.title);
1876     ADD_TAG(entry, tag_artist, &id3.artist);
1877     ADD_TAG(entry, tag_album, &id3.album);
1878     ADD_TAG(entry, tag_genre, &id3.genre_string);
1879     ADD_TAG(entry, tag_composer, &id3.composer);
1880     ADD_TAG(entry, tag_comment, &id3.comment);
1881     if (has_albumartist)
1882     {
1883         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1884     }
1885     else
1886     {
1887         ADD_TAG(entry, tag_albumartist, &id3.artist);
1888     }
1889     if (has_grouping)
1890     {
1891         ADD_TAG(entry, tag_grouping, &id3.grouping);
1892     }
1893     else
1894     {
1895         ADD_TAG(entry, tag_grouping, &id3.title);
1896     }
1897     entry.data_length = offset;
1898
1899     /* Write the header */
1900     write(cachefd, &entry, sizeof(struct temp_file_entry));
1901
1902     /* And tags also... Correct order is critical */
1903     write_item(path);
1904     write_item(id3.title);
1905     write_item(id3.artist);
1906     write_item(id3.album);
1907     write_item(id3.genre_string);
1908     write_item(id3.composer);
1909     write_item(id3.comment);
1910     if (has_albumartist)
1911     {
1912         write_item(id3.albumartist);
1913     }
1914     else
1915     {
1916         write_item(id3.artist);
1917     }
1918     if (has_grouping)
1919     {
1920         write_item(id3.grouping);
1921     }
1922     else
1923     {
1924         write_item(id3.title);
1925     }
1926     total_entry_count++;
1927 }
1928
1929 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1930 {
1931     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1932     int len = strlen(str)+1;
1933     int i;
1934     unsigned crc32;
1935     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1936     char buf[TAG_MAXLEN+32];
1937
1938     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1939         buf[i] = tolower(str[i]);
1940     buf[i] = '\0';
1941
1942     crc32 = crc_32(buf, i, 0xffffffff);
1943
1944     if (unique)
1945     {
1946         /* Check if the crc does not exist -> entry does not exist for sure. */
1947         for (i = 0; i < tempbufidx; i++)
1948         {
1949             if (crcbuf[-i] != crc32)
1950                 continue;
1951
1952             if (!strcasecmp(str, index[i].str))
1953             {
1954                 if (id < 0 || id >= lookup_buffer_depth)
1955                 {
1956                     logf("lookup buf overf.: %d", id);
1957                     return false;
1958                 }
1959
1960                 lookup[id] = &index[i];
1961                 return true;
1962             }
1963         }
1964     }
1965
1966     /* Insert to CRC buffer. */
1967     crcbuf[-tempbufidx] = crc32;
1968     tempbuf_left -= 4;
1969
1970     /* Insert it to the buffer. */
1971     tempbuf_left -= len;
1972     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1973         return false;
1974
1975     if (id >= lookup_buffer_depth)
1976     {
1977         logf("lookup buf overf. #2: %d", id);
1978         return false;
1979     }
1980
1981     if (id >= 0)
1982     {
1983         lookup[id] = &index[tempbufidx];
1984         index[tempbufidx].idlist.id = id;
1985     }
1986     else
1987         index[tempbufidx].idlist.id = -1;
1988
1989     index[tempbufidx].idlist.next = NULL;
1990     index[tempbufidx].idx_id = idx_id;
1991     index[tempbufidx].seek = -1;
1992     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1993     memcpy(index[tempbufidx].str, str, len);
1994     tempbuf_pos += len;
1995     tempbufidx++;
1996
1997     return true;
1998 }
1999
2000 static int compare(const void *p1, const void *p2)
2001 {
2002     do_timed_yield();
2003
2004     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2005     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2006
2007     if (strcmp(e1->str, UNTAGGED) == 0)
2008     {
2009         if (strcmp(e2->str, UNTAGGED) == 0)
2010             return 0;
2011         return -1;
2012     }
2013     else if (strcmp(e2->str, UNTAGGED) == 0)
2014         return 1;
2015
2016     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2017 }
2018
2019 static int tempbuf_sort(int fd)
2020 {
2021     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2022     struct tagfile_entry fe;
2023     int i;
2024     int length;
2025
2026     /* Generate reverse lookup entries. */
2027     for (i = 0; i < lookup_buffer_depth; i++)
2028     {
2029         struct tempbuf_id_list *idlist;
2030
2031         if (!lookup[i])
2032             continue;
2033
2034         if (lookup[i]->idlist.id == i)
2035             continue;
2036
2037         idlist = &lookup[i]->idlist;
2038         while (idlist->next != NULL)
2039             idlist = idlist->next;
2040
2041         tempbuf_left -= sizeof(struct tempbuf_id_list);
2042         if (tempbuf_left - 4 < 0)
2043             return -1;
2044
2045         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2046         if (tempbuf_pos & 0x03)
2047         {
2048             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2049             tempbuf_left -= 3;
2050             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2051         }
2052         tempbuf_pos += sizeof(struct tempbuf_id_list);
2053
2054         idlist = idlist->next;
2055         idlist->id = i;
2056         idlist->next = NULL;
2057
2058         do_timed_yield();
2059     }
2060
2061     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2062     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2063
2064     for (i = 0; i < tempbufidx; i++)
2065     {
2066         struct tempbuf_id_list *idlist = &index[i].idlist;
2067
2068         /* Fix the lookup list. */
2069         while (idlist != NULL)
2070         {
2071             if (idlist->id >= 0)
2072                 lookup[idlist->id] = &index[i];
2073             idlist = idlist->next;
2074         }
2075
2076         index[i].seek = lseek(fd, 0, SEEK_CUR);
2077         length = strlen(index[i].str) + 1;
2078         fe.tag_length = length;
2079         fe.idx_id = index[i].idx_id;
2080
2081         /* Check the chunk alignment. */
2082         if ((fe.tag_length + sizeof(struct tagfile_entry))
2083             % TAGFILE_ENTRY_CHUNK_LENGTH)
2084         {
2085             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2086                 ((fe.tag_length + sizeof(struct tagfile_entry))
2087                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2088         }
2089
2090 #ifdef TAGCACHE_STRICT_ALIGN
2091         /* Make sure the entry is long aligned. */
2092         if (index[i].seek & 0x03)
2093         {
2094             logf("tempbuf_sort: alignment error!");
2095             return -3;
2096         }
2097 #endif
2098
2099         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2100             sizeof(struct tagfile_entry))
2101         {
2102             logf("tempbuf_sort: write error #1");
2103             return -1;
2104         }
2105
2106         if (write(fd, index[i].str, length) != length)
2107         {
2108             logf("tempbuf_sort: write error #2");
2109             return -2;
2110         }
2111
2112         /* Write some padding. */
2113         if (fe.tag_length - length > 0)
2114             write(fd, "XXXXXXXX", fe.tag_length - length);
2115     }
2116
2117     return i;
2118 }
2119
2120 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2121 {
2122     if (id < 0 || id >= lookup_buffer_depth)
2123         return NULL;
2124
2125     return lookup[id];
2126 }
2127
2128
2129 inline static int tempbuf_find_location(int id)
2130 {
2131     struct tempbuf_searchidx *entry;
2132
2133     entry = tempbuf_locate(id);
2134     if (entry == NULL)
2135         return -1;
2136
2137     return entry->seek;
2138 }
2139
2140 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2141 {
2142     struct master_header tcmh;
2143     struct index_entry idx;
2144     int masterfd;
2145     int masterfd_pos;
2146     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2147     int max_entries;
2148     int entries_processed = 0;
2149     int i, j;
2150     char buf[TAG_MAXLEN];
2151
2152     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2153
2154     logf("Building numeric indices...");
2155     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2156
2157     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2158         return false;
2159
2160     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2161                          SEEK_CUR);
2162     if (masterfd_pos == filesize(masterfd))
2163     {
2164         logf("we can't append!");
2165         close(masterfd);
2166         return false;
2167     }
2168
2169     while (entries_processed < h->entry_count)
2170     {
2171         int count = MIN(h->entry_count - entries_processed, max_entries);
2172
2173         /* Read in as many entries as possible. */
2174         for (i = 0; i < count; i++)
2175         {
2176             struct temp_file_entry *tfe = &entrybuf[i];
2177             int datastart;
2178
2179             /* Read in numeric data. */
2180             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2181                 sizeof(struct temp_file_entry))
2182             {
2183                 logf("read fail #1");
2184                 close(masterfd);
2185                 return false;
2186             }
2187
2188             datastart = lseek(tmpfd, 0, SEEK_CUR);
2189
2190             /**
2191              * Read string data from the following tags:
2192              * - tag_filename
2193              * - tag_artist
2194              * - tag_album
2195              * - tag_title
2196              *
2197              * A crc32 hash is calculated from the read data
2198              * and stored back to the data offset field kept in memory.
2199              */
2200 #define tmpdb_read_string_tag(tag) \
2201     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2202     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2203     { \
2204         logf("read fail: buffer overflow"); \
2205         close(masterfd); \
2206         return false; \
2207     } \
2208     \
2209     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2210         tfe->tag_length[tag]) \
2211     { \
2212         logf("read fail #2"); \
2213         close(masterfd); \
2214         return false; \
2215     } \
2216     \
2217     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2218     lseek(tmpfd, datastart, SEEK_SET)
2219
2220             tmpdb_read_string_tag(tag_filename);
2221             tmpdb_read_string_tag(tag_artist);
2222             tmpdb_read_string_tag(tag_album);
2223             tmpdb_read_string_tag(tag_title);
2224
2225             /* Seek to the end of the string data. */
2226             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2227         }
2228
2229         /* Backup the master index position. */
2230         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2231         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2232
2233         /* Check if we can resurrect some deleted runtime statistics data. */
2234         for (i = 0; i < tcmh.tch.entry_count; i++)
2235         {
2236             /* Read the index entry. */
2237             if (ecread_index_entry(masterfd, &idx)
2238                 != sizeof(struct index_entry))
2239             {
2240                 logf("read fail #3");
2241                 close(masterfd);
2242                 return false;
2243             }
2244
2245             /**
2246              * Skip unless the entry is marked as being deleted
2247              * or the data has already been resurrected.
2248              */
2249             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2250                 continue;
2251
2252             /* Now try to match the entry. */
2253             /**
2254              * To succesfully match a song, the following conditions
2255              * must apply:
2256              *
2257              * For numeric fields: tag_length
2258              * - Full identical match is required
2259              *
2260              * If tag_filename matches, no further checking necessary.
2261              *
2262              * For string hashes: tag_artist, tag_album, tag_title
2263              * - All three of these must match
2264              */
2265             for (j = 0; j < count; j++)
2266             {
2267                 struct temp_file_entry *tfe = &entrybuf[j];
2268
2269                 /* Try to match numeric fields first. */
2270                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2271                     continue;
2272
2273                 /* Now it's time to do the hash matching. */
2274                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2275                 {
2276                     int match_count = 0;
2277
2278                     /* No filename match, check if we can match two other tags. */
2279 #define tmpdb_match(tag) \
2280     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2281         match_count++
2282
2283                     tmpdb_match(tag_artist);
2284                     tmpdb_match(tag_album);
2285                     tmpdb_match(tag_title);
2286
2287                     if (match_count < 3)
2288                     {
2289                         /* Still no match found, give up. */
2290                         continue;
2291                     }
2292                 }
2293
2294                 /* A match found, now copy & resurrect the statistical data. */
2295 #define tmpdb_copy_tag(tag) \
2296     tfe->tag_offset[tag] = idx.tag_seek[tag]
2297
2298                 tmpdb_copy_tag(tag_playcount);
2299                 tmpdb_copy_tag(tag_rating);
2300                 tmpdb_copy_tag(tag_playtime);
2301                 tmpdb_copy_tag(tag_lastplayed);
2302                 tmpdb_copy_tag(tag_commitid);
2303                 tmpdb_copy_tag(tag_lastoffset);
2304
2305                 /* Avoid processing this entry again. */
2306                 idx.flag |= FLAG_RESURRECTED;
2307
2308                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2309                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2310                 {
2311                     logf("masterfd writeback fail #1");
2312                     close(masterfd);
2313                     return false;
2314                 }
2315
2316                 logf("Entry resurrected");
2317             }
2318         }
2319
2320
2321         /* Restore the master index position. */
2322         lseek(masterfd, masterfd_pos, SEEK_SET);
2323
2324         /* Commit the data to the index. */
2325         for (i = 0; i < count; i++)
2326         {
2327             int loc = lseek(masterfd, 0, SEEK_CUR);
2328
2329             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2330             {
2331                 logf("read fail #3");
2332                 close(masterfd);
2333                 return false;
2334             }
2335
2336             for (j = 0; j < TAG_COUNT; j++)
2337             {
2338                 if (!TAGCACHE_IS_NUMERIC(j))
2339                     continue;
2340
2341                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2342             }
2343             idx.flag = entrybuf[i].flag;
2344
2345             if (idx.tag_seek[tag_commitid])
2346             {
2347                 /* Data has been resurrected. */
2348                 idx.flag |= FLAG_DIRTYNUM;
2349             }
2350             else if (tc_stat.ready && current_tcmh.commitid > 0)
2351             {
2352                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2353                 idx.flag |= FLAG_DIRTYNUM;
2354             }
2355
2356             /* Write back the updated index. */
2357             lseek(masterfd, loc, SEEK_SET);
2358             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2359             {
2360                 logf("write fail");
2361                 close(masterfd);
2362                 return false;
2363             }
2364         }
2365
2366         entries_processed += count;
2367         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2368     }
2369
2370     close(masterfd);
2371
2372     return true;
2373 }
2374
2375 /**
2376  * Return values:
2377  *     > 0   success
2378  *    == 0   temporary failure
2379  *     < 0   fatal error
2380  */
2381 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2382 {
2383     int i;
2384     struct tagcache_header tch;
2385     struct master_header   tcmh;
2386     struct index_entry idxbuf[IDX_BUF_DEPTH];
2387     int idxbuf_pos;
2388     char buf[TAG_MAXLEN+32];
2389     int fd = -1, masterfd;
2390     bool error = false;
2391     int init;
2392     int masterfd_pos;
2393
2394     logf("Building index: %d", index_type);
2395
2396     /* Check the number of entries we need to allocate ram for. */
2397     commit_entry_count = h->entry_count + 1;
2398
2399     masterfd = open_master_fd(&tcmh, false);
2400     if (masterfd >= 0)
2401     {
2402         commit_entry_count += tcmh.tch.entry_count;
2403         close(masterfd);
2404     }
2405     else
2406         remove_files(); /* Just to be sure we are clean. */
2407
2408     /* Open the index file, which contains the tag names. */
2409     fd = open_tag_fd(&tch, index_type, true);
2410     if (fd >= 0)
2411     {
2412         logf("tch.datasize=%ld", tch.datasize);
2413         lookup_buffer_depth = 1 +
2414         /* First part */ commit_entry_count +
2415         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2416     }
2417     else
2418     {
2419         lookup_buffer_depth = 1 +
2420         /* First part */ commit_entry_count +
2421         /* Second part */ 0;
2422     }
2423
2424     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2425     logf("commit_entry_count=%ld", commit_entry_count);
2426
2427     /* Allocate buffer for all index entries from both old and new
2428      * tag files. */
2429     tempbufidx = 0;
2430     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2431
2432     /* Allocate lookup buffer. The first portion of commit_entry_count
2433      * contains the new tags in the temporary file and the second
2434      * part for locating entries already in the db.
2435      *
2436      *  New tags  Old tags
2437      * +---------+---------------------------+
2438      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2439      * +---------+---------------------------+
2440      *
2441      * Old tags are inserted to a temporary buffer with position:
2442      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2443      * And new tags with index:
2444      *     tempbuf_insert(idx, ...);
2445      *
2446      * The buffer is sorted and written into tag file:
2447      *     tempbuf_sort(...);
2448      * leaving master index locations messed up.
2449      *
2450      * That is fixed using the lookup buffer for old tags:
2451      *     new_seek = tempbuf_find_location(old_seek, ...);
2452      * and for new tags:
2453      *     new_seek = tempbuf_find_location(idx);
2454      */
2455     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2456     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2457     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2458
2459     /* And calculate the remaining data space used mainly for storing
2460      * tag data (strings). */
2461     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2462     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2463     {
2464         logf("Buffer way too small!");
2465         return 0;
2466     }
2467
2468     if (fd >= 0)
2469     {
2470         /**
2471          * If tag file contains unique tags (sorted index), we will load
2472          * it entirely into memory so we can resort it later for use with
2473          * chunked browsing.
2474          */
2475         if (TAGCACHE_IS_SORTED(index_type))
2476         {
2477             logf("loading tags...");
2478             for (i = 0; i < tch.entry_count; i++)
2479             {
2480                 struct tagfile_entry entry;
2481                 int loc = lseek(fd, 0, SEEK_CUR);
2482                 bool ret;
2483
2484                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2485                 {
2486                     logf("read error #7");
2487                     close(fd);
2488                     return -2;
2489                 }
2490
2491                 if (entry.tag_length >= (int)sizeof(buf))
2492                 {
2493                     logf("too long tag #3");
2494                     close(fd);
2495                     return -2;
2496                 }
2497
2498                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2499                 {
2500                     logf("read error #8");
2501                     close(fd);
2502                     return -2;
2503                 }
2504
2505                 /* Skip deleted entries. */
2506                 if (buf[0] == '\0')
2507                     continue;
2508
2509                 /**
2510                  * Save the tag and tag id in the memory buffer. Tag id
2511                  * is saved so we can later reindex the master lookup
2512                  * table when the index gets resorted.
2513                  */
2514                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2515                                      + commit_entry_count, entry.idx_id,
2516                                      TAGCACHE_IS_UNIQUE(index_type));
2517                 if (!ret)
2518                 {
2519                     close(fd);
2520                     return -3;
2521                 }
2522                 do_timed_yield();
2523             }
2524             logf("done");
2525         }
2526         else
2527             tempbufidx = tch.entry_count;
2528     }
2529     else
2530     {
2531         /**
2532          * Creating new index file to store the tags. No need to preload
2533          * anything whether the index type is sorted or not.
2534          */
2535         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2536         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2537         if (fd < 0)
2538         {
2539             logf("%s open fail", buf);
2540             return -2;
2541         }
2542
2543         tch.magic = TAGCACHE_MAGIC;
2544         tch.entry_count = 0;
2545         tch.datasize = 0;
2546
2547         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2548             != sizeof(struct tagcache_header))
2549         {
2550             logf("header write failed");
2551             close(fd);
2552             return -2;
2553         }
2554     }
2555
2556     /* Loading the tag lookup file as "master file". */
2557     logf("Loading index file");
2558     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2559
2560     if (masterfd < 0)
2561     {
2562         logf("Creating new DB");
2563         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2564
2565         if (masterfd < 0)
2566         {
2567             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2568             close(fd);
2569             return -2;
2570         }
2571
2572         /* Write the header (write real values later). */
2573         memset(&tcmh, 0, sizeof(struct master_header));
2574         tcmh.tch = *h;
2575         tcmh.tch.entry_count = 0;
2576         tcmh.tch.datasize = 0;
2577         tcmh.dirty = true;
2578         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2579         init = true;
2580         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2581     }
2582     else
2583     {
2584         /**
2585          * Master file already exists so we need to process the current
2586          * file first.
2587          */
2588         init = false;
2589
2590         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2591             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2592         {
2593             logf("header error");
2594             close(fd);
2595             close(masterfd);
2596             return -2;
2597         }
2598
2599         /**
2600          * If we reach end of the master file, we need to expand it to
2601          * hold new tags. If the current index is not sorted, we can
2602          * simply append new data to end of the file.
2603          * However, if the index is sorted, we need to update all tag
2604          * pointers in the master file for the current index.
2605          */
2606         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2607             SEEK_CUR);
2608         if (masterfd_pos == filesize(masterfd))
2609         {
2610             logf("appending...");
2611             init = true;
2612         }
2613     }
2614
2615     /**
2616      * Load new unique tags in memory to be sorted later and added
2617      * to the master lookup file.
2618      */
2619     if (TAGCACHE_IS_SORTED(index_type))
2620     {
2621         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2622         /* h is the header of the temporary file containing new tags. */
2623         logf("inserting new tags...");
2624         for (i = 0; i < h->entry_count; i++)
2625         {
2626             struct temp_file_entry entry;
2627
2628             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2629                 sizeof(struct temp_file_entry))
2630             {
2631                 logf("read fail #3");
2632                 error = true;
2633                 goto error_exit;
2634             }
2635
2636             /* Read data. */
2637             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2638             {
2639                 logf("too long entry!");
2640                 error = true;
2641                 goto error_exit;
2642             }
2643
2644             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2645             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2646                 entry.tag_length[index_type])
2647             {
2648                 logf("read fail #4");
2649                 error = true;
2650                 goto error_exit;
2651             }
2652
2653             if (TAGCACHE_IS_UNIQUE(index_type))
2654                 error = !tempbuf_insert(buf, i, -1, true);
2655             else
2656                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2657
2658             if (error)
2659             {
2660                 logf("insert error");
2661                 goto error_exit;
2662             }
2663
2664             /* Skip to next. */
2665             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2666                     entry.tag_length[index_type], SEEK_CUR);
2667             do_timed_yield();
2668         }
2669         logf("done");
2670
2671         /* Sort the buffer data and write it to the index file. */
2672         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2673         /**
2674          * We need to truncate the index file now. There can be junk left
2675          * at the end of file (however, we _should_ always follow the
2676          * entry_count and don't crash with that).
2677          */
2678         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2679
2680         i = tempbuf_sort(fd);
2681         if (i < 0)
2682             goto error_exit;
2683         logf("sorted %d tags", i);
2684
2685         /**
2686          * Now update all indexes in the master lookup file.
2687          */
2688         logf("updating indices...");
2689         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2690         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2691         {
2692             int j;
2693             int loc = lseek(masterfd, 0, SEEK_CUR);
2694
2695             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2696
2697             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2698                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2699             {
2700                 logf("read fail #5");
2701                 error = true;
2702                 goto error_exit ;
2703             }
2704             lseek(masterfd, loc, SEEK_SET);
2705
2706             for (j = 0; j < idxbuf_pos; j++)
2707             {
2708                 if (idxbuf[j].flag & FLAG_DELETED)
2709                 {
2710                     /* We can just ignore deleted entries. */
2711                     // idxbuf[j].tag_seek[index_type] = 0;
2712                     continue;
2713                 }
2714
2715                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2716                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2717                     + commit_entry_count);
2718
2719                 if (idxbuf[j].tag_seek[index_type] < 0)
2720                 {
2721                     logf("update error: %ld/%d/%ld",
2722                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2723                     error = true;
2724                     goto error_exit;
2725                 }
2726
2727                 do_timed_yield();
2728             }
2729
2730             /* Write back the updated index. */
2731             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2732                         index_entry_ec, tc_stat.econ) !=
2733                 (int)sizeof(struct index_entry)*idxbuf_pos)
2734             {
2735                 logf("write fail");
2736                 error = true;
2737                 goto error_exit;
2738             }
2739         }
2740         logf("done");
2741     }
2742
2743     /**
2744      * Walk through the temporary file containing the new tags.
2745      */
2746     // build_normal_index(h, tmpfd, masterfd, idx);
2747     logf("updating new indices...");
2748     lseek(masterfd, masterfd_pos, SEEK_SET);
2749     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2750     lseek(fd, 0, SEEK_END);
2751     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2752     {
2753         int j;
2754
2755         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2756         if (init)
2757         {
2758             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2759         }
2760         else
2761         {
2762             int loc = lseek(masterfd, 0, SEEK_CUR);
2763
2764             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2765                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2766             {
2767                 logf("read fail #6");
2768                 error = true;
2769                 break ;
2770             }
2771             lseek(masterfd, loc, SEEK_SET);
2772         }
2773
2774         /* Read entry headers. */
2775         for (j = 0; j < idxbuf_pos; j++)
2776         {
2777             if (!TAGCACHE_IS_SORTED(index_type))
2778             {
2779                 struct temp_file_entry entry;
2780                 struct tagfile_entry fe;
2781
2782                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2783                     sizeof(struct temp_file_entry))
2784                 {
2785                     logf("read fail #7");
2786                     error = true;
2787                     break ;
2788                 }
2789
2790                 /* Read data. */
2791                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2792                 {
2793                     logf("too long entry!");
2794                     logf("length=%d", entry.tag_length[index_type]);
2795                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2796                     error = true;
2797                     break ;
2798                 }
2799
2800                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2801                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2802                     entry.tag_length[index_type])
2803                 {
2804                     logf("read fail #8");
2805                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2806                     logf("length=0x%02x", entry.tag_length[index_type]);
2807                     error = true;
2808                     break ;
2809                 }
2810
2811                 /* Write to index file. */
2812                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2813                 fe.tag_length = entry.tag_length[index_type];
2814                 fe.idx_id = tcmh.tch.entry_count + i + j;
2815                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2816                 write(fd, buf, fe.tag_length);
2817                 tempbufidx++;
2818
2819                 /* Skip to next. */
2820                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2821                       entry.tag_length[index_type], SEEK_CUR);
2822             }
2823             else
2824             {
2825                 /* Locate the correct entry from the sorted array. */
2826                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2827                 if (idxbuf[j].tag_seek[index_type] < 0)
2828                 {
2829                     logf("entry not found (%d)", j);
2830                     error = true;
2831                     break ;
2832                 }
2833             }
2834         }
2835
2836         /* Write index. */
2837         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2838                   index_entry_ec, tc_stat.econ) !=
2839             (int)sizeof(struct index_entry)*idxbuf_pos)
2840         {
2841             logf("tagcache: write fail #4");
2842             error = true;
2843             break ;
2844         }
2845
2846         do_timed_yield();
2847     }
2848     logf("done");
2849
2850     /* Finally write the header. */
2851     tch.magic = TAGCACHE_MAGIC;
2852     tch.entry_count = tempbufidx;
2853     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2854     lseek(fd, 0, SEEK_SET);
2855     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2856
2857     if (index_type != tag_filename)
2858         h->datasize += tch.datasize;
2859     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2860     error_exit:
2861
2862     close(fd);
2863     close(masterfd);
2864
2865     if (error)
2866         return -2;
2867
2868     return 1;
2869 }
2870
2871 static bool commit(void)
2872 {
2873     struct tagcache_header tch;
2874     struct master_header   tcmh;
2875     int i, len, rc;
2876     int tmpfd;
2877     int masterfd;
2878 #ifdef HAVE_DIRCACHE
2879     bool dircache_buffer_stolen = false;
2880 #endif
2881     bool local_allocation = false;
2882
2883     logf("committing tagcache");
2884
2885     while (write_lock)
2886         sleep(1);
2887
2888     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2889     if (tmpfd < 0)
2890     {
2891         logf("nothing to commit");
2892         return true;
2893     }
2894
2895
2896     /* Load the header. */
2897     len = sizeof(struct tagcache_header);
2898     rc = read(tmpfd, &tch, len);
2899
2900     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2901     {
2902         logf("incorrect tmpheader");
2903         close(tmpfd);
2904         remove(TAGCACHE_FILE_TEMP);
2905         return false;
2906     }
2907
2908     if (tch.entry_count == 0)
2909     {
2910         logf("nothing to commit");
2911         close(tmpfd);
2912         remove(TAGCACHE_FILE_TEMP);
2913         return true;
2914     }
2915
2916     /* Fully initialize existing headers (if any) before going further. */
2917     tc_stat.ready = check_all_headers();
2918
2919 #ifdef HAVE_EEPROM_SETTINGS
2920     remove(TAGCACHE_STATEFILE);
2921 #endif
2922
2923     /* At first be sure to unload the ramcache! */
2924 #ifdef HAVE_TC_RAMCACHE
2925     tc_stat.ramcache = false;
2926 #endif
2927
2928     read_lock++;
2929
2930     /* Try to steal every buffer we can :) */
2931     if (tempbuf_size == 0)
2932         local_allocation = true;
2933
2934 #ifdef HAVE_DIRCACHE
2935     if (tempbuf_size == 0)
2936     {
2937         /* Try to steal the dircache buffer. */
2938         tempbuf = dircache_steal_buffer(&tempbuf_size);
2939         tempbuf_size &= ~0x03;
2940
2941         if (tempbuf_size > 0)
2942         {
2943             dircache_buffer_stolen = true;
2944         }
2945     }
2946 #endif
2947
2948 #ifdef HAVE_TC_RAMCACHE
2949     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2950     {
2951         tempbuf = (char *)(ramcache_hdr + 1);
2952         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2953         tempbuf_size &= ~0x03;
2954     }
2955 #endif
2956
2957     /* And finally fail if there are no buffers available. */
2958     if (tempbuf_size == 0)
2959     {
2960         logf("delaying commit until next boot");
2961         tc_stat.commit_delayed = true;
2962         close(tmpfd);
2963         read_lock--;
2964         return false;
2965     }
2966
2967     logf("commit %ld entries...", tch.entry_count);
2968
2969     /* Mark DB dirty so it will stay disabled if commit fails. */
2970     current_tcmh.dirty = true;
2971     update_master_header();
2972
2973     /* Now create the index files. */
2974     tc_stat.commit_step = 0;
2975     tch.datasize = 0;
2976     tc_stat.commit_delayed = false;
2977
2978     for (i = 0; i < TAG_COUNT; i++)
2979     {
2980         int ret;
2981
2982         if (TAGCACHE_IS_NUMERIC(i))
2983             continue;
2984
2985         tc_stat.commit_step++;
2986         ret = build_index(i, &tch, tmpfd);
2987         if (ret <= 0)
2988         {
2989             close(tmpfd);
2990             logf("tagcache failed init");
2991             if (ret == 0)
2992                 tc_stat.commit_delayed = true;
2993
2994             tc_stat.commit_step = 0;
2995             read_lock--;
2996             return false;
2997         }
2998     }
2999
3000     if (!build_numeric_indices(&tch, tmpfd))
3001     {
3002         logf("Failure to commit numeric indices");
3003         close(tmpfd);
3004         tc_stat.commit_step = 0;
3005         read_lock--;
3006         return false;
3007     }
3008
3009     close(tmpfd);
3010
3011     tc_stat.commit_step = 0;
3012
3013     /* Update the master index headers. */
3014     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3015     {
3016         read_lock--;
3017         return false;
3018     }
3019
3020     remove(TAGCACHE_FILE_TEMP);
3021
3022     tcmh.tch.entry_count += tch.entry_count;
3023     tcmh.tch.datasize = sizeof(struct master_header)
3024         + sizeof(struct index_entry) * tcmh.tch.entry_count
3025         + tch.datasize;
3026     tcmh.dirty = false;
3027     tcmh.commitid++;
3028
3029     lseek(masterfd, 0, SEEK_SET);
3030     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3031     close(masterfd);
3032
3033     logf("tagcache committed");
3034     tc_stat.ready = check_all_headers();
3035     tc_stat.readyvalid = true;
3036
3037     if (local_allocation)
3038     {
3039         tempbuf = NULL;
3040         tempbuf_size = 0;
3041     }
3042
3043 #ifdef HAVE_DIRCACHE
3044     /* Rebuild the dircache, if we stole the buffer. */
3045     if (dircache_buffer_stolen)
3046         dircache_build(0);
3047 #endif
3048
3049 #ifdef HAVE_TC_RAMCACHE
3050     /* Reload tagcache. */
3051     if (tc_stat.ramcache_allocated > 0)
3052         tagcache_start_scan();
3053 #endif
3054
3055     read_lock--;
3056
3057     return true;
3058 }
3059
3060 static void allocate_tempbuf(void)
3061 {
3062     /* Yeah, malloc would be really nice now :) */
3063 #ifdef __PCTOOL__
3064     tempbuf_size = 32*1024*1024;
3065     tempbuf = malloc(tempbuf_size);
3066 #else
3067     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3068     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3069     audiobuf += tempbuf_size;
3070 #endif
3071 }
3072
3073 static void free_tempbuf(void)
3074 {
3075     if (tempbuf_size == 0)
3076         return ;
3077
3078 #ifdef __PCTOOL__
3079     free(tempbuf);
3080 #else
3081     audiobuf -= tempbuf_size;
3082 #endif
3083     tempbuf = NULL;
3084     tempbuf_size = 0;
3085 }
3086
3087 #ifndef __PCTOOL__
3088
3089 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3090 {
3091     struct index_entry idx;
3092
3093     if (!tc_stat.ready)
3094         return false;
3095
3096     if (!TAGCACHE_IS_NUMERIC(tag))
3097         return false;
3098
3099     if (!get_index(masterfd, idx_id, &idx, false))
3100         return false;
3101
3102     idx.tag_seek[tag] = data;
3103     idx.flag |= FLAG_DIRTYNUM;
3104
3105     return write_index(masterfd, idx_id, &idx);
3106 }
3107
3108 #if 0
3109 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3110                                    int tag, long data)
3111 {
3112     struct master_header myhdr;
3113
3114     if (tcs->masterfd < 0)
3115     {
3116         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3117             return false;
3118     }
3119
3120     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3121 }
3122 #endif
3123
3124 static bool command_queue_is_full(void)
3125 {
3126     int next;
3127
3128     next = command_queue_widx + 1;
3129     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3130         next = 0;
3131
3132     return (next == command_queue_ridx);
3133 }
3134
3135 static void command_queue_sync_callback(void *data)
3136 {
3137     (void)data;
3138     struct master_header myhdr;
3139     int masterfd;
3140
3141     mutex_lock(&command_queue_mutex);
3142
3143     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3144         return;
3145
3146     while (command_queue_ridx != command_queue_widx)
3147     {
3148         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3149
3150         switch (ce->command)
3151         {
3152             case CMD_UPDATE_MASTER_HEADER:
3153             {
3154                 close(masterfd);
3155                 update_master_header();
3156
3157                 /* Re-open the masterfd. */
3158                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3159                     return;
3160
3161                 break;
3162             }
3163             case CMD_UPDATE_NUMERIC:
3164             {
3165                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3166                 break;
3167             }
3168         }
3169
3170         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3171             command_queue_ridx = 0;
3172     }
3173
3174     close(masterfd);
3175
3176     tc_stat.queue_length = 0;
3177     mutex_unlock(&command_queue_mutex);
3178 }
3179
3180 static void run_command_queue(bool force)
3181 {
3182     if (COMMAND_QUEUE_IS_EMPTY)
3183         return;
3184
3185     if (force || command_queue_is_full())
3186         command_queue_sync_callback(NULL);
3187     else
3188         register_storage_idle_func(command_queue_sync_callback);
3189 }
3190
3191 static void queue_command(int cmd, long idx_id, int tag, long data)
3192 {
3193     while (1)
3194     {
3195         int next;
3196
3197         mutex_lock(&command_queue_mutex);
3198         next = command_queue_widx + 1;
3199         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3200             next = 0;
3201
3202         /* Make sure queue is not full. */
3203         if (next != command_queue_ridx)
3204         {
3205             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3206
3207             ce->command = cmd;
3208             ce->idx_id = idx_id;
3209             ce->tag = tag;
3210             ce->data = data;
3211
3212             command_queue_widx = next;
3213
3214             tc_stat.queue_length++;
3215
3216             mutex_unlock(&command_queue_mutex);
3217             break;
3218         }
3219
3220         /* Queue is full, try again later... */
3221         mutex_unlock(&command_queue_mutex);
3222         sleep(1);
3223     }
3224 }
3225
3226 long tagcache_increase_serial(void)
3227 {
3228     long old;
3229
3230     if (!tc_stat.ready)
3231         return -2;
3232
3233     while (read_lock)
3234         sleep(1);
3235
3236     old = current_tcmh.serial++;
3237     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3238
3239     return old;
3240 }
3241
3242 void tagcache_update_numeric(int idx_id, int tag, long data)
3243 {
3244     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3245 }
3246 #endif /* !__PCTOOL__ */
3247
3248 long tagcache_get_serial(void)
3249 {
3250     return current_tcmh.serial;
3251 }
3252
3253 long tagcache_get_commitid(void)
3254 {
3255     return current_tcmh.commitid;
3256 }
3257
3258 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3259 {
3260     char buf[512];
3261     int i;
3262
3263     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3264     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3265     {
3266         if (*datastr == '\0')
3267             break;
3268
3269         if (*datastr == '"' || *datastr == '\\')
3270             buf[i++] = '\\';
3271
3272         else if (*datastr == '\n')
3273         {
3274             buf[i++] = '\\';
3275             buf[i] = 'n';
3276             continue;
3277         }
3278
3279         buf[i] = *(datastr++);
3280     }
3281
3282     strcpy(&buf[i], "\" ");
3283
3284     write(fd, buf, i + 2);
3285
3286     return true;
3287 }
3288
3289 #ifndef __PCTOOL__
3290
3291 static bool read_tag(char *dest, long size,
3292                      const char *src, const char *tagstr)
3293 {
3294     int pos;
3295     char current_tag[32];
3296
3297     while (*src != '\0')
3298     {
3299         /* Skip all whitespace */
3300         while (*src == ' ')
3301             src++;
3302
3303         if (*src == '\0')
3304             break;
3305
3306         pos = 0;
3307         /* Read in tag name */
3308         while (*src != '=' && *src != ' ')
3309         {
3310             current_tag[pos] = *src;
3311             src++;
3312             pos++;
3313
3314             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3315                 return false;
3316         }
3317         current_tag[pos] = '\0';
3318
3319         /* Read in tag data */
3320
3321         /* Find the start. */
3322         while (*src != '"' && *src != '\0')
3323             src++;
3324
3325         if (*src == '\0' || *(++src) == '\0')
3326             return false;
3327
3328         /* Read the data. */
3329         for (pos = 0; pos < size; pos++)
3330         {
3331             if (*src == '\0')
3332                 break;
3333
3334             if (*src == '\\')
3335             {
3336                 src++;
3337                 if (*src == 'n')
3338                     dest[pos] = '\n';
3339                 else
3340                     dest[pos] = *src;
3341
3342                 src++;
3343                 continue;
3344             }
3345
3346             if (*src == '\0')
3347                 break;
3348
3349             if (*src == '"')
3350             {
3351                 src++;
3352                 break;
3353             }
3354
3355             dest[pos] = *(src++);
3356         }
3357
3358         dest[pos] = '\0';
3359
3360         if (!strcasecmp(tagstr, current_tag))
3361             return true;
3362     }
3363
3364     return false;
3365 }
3366
3367 static int parse_changelog_line(int line_n, char *buf, void *parameters)
3368 {
3369     struct index_entry idx;
3370     char tag_data[TAG_MAXLEN+32];
3371     int idx_id;
3372     long masterfd = (long)parameters;
3373     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3374                                 tag_lastplayed, tag_commitid, tag_lastoffset };
3375     int i;
3376     (void)line_n;
3377
3378     if (*buf == '#')
3379         return 0;
3380
3381     /* logf("%d/%s", line_n, buf); */
3382     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3383     {
3384         logf("%d/filename missing", line_n);
3385         logf("-> %s", buf);
3386         return 0;
3387     }
3388
3389     idx_id = find_index(tag_data);
3390     if (idx_id < 0)
3391     {
3392         logf("%d/entry not found", line_n);
3393         return 0;
3394     }
3395
3396     if (!get_index(masterfd, idx_id, &idx, false))
3397     {
3398         logf("%d/failed to retrieve index entry", line_n);
3399         return 0;
3400     }
3401
3402     /* Stop if tag has already been modified. */
3403     if (idx.flag & FLAG_DIRTYNUM)
3404         return 0;
3405
3406     logf("%d/import: %s", line_n, tag_data);
3407
3408     idx.flag |= FLAG_DIRTYNUM;
3409     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3410     {
3411         int data;
3412
3413         if (!read_tag(tag_data, sizeof tag_data, buf,
3414                       tagcache_tag_to_str(import_tags[i])))
3415         {
3416             continue;
3417         }
3418
3419         data = atoi(tag_data);
3420         if (data < 0)
3421             continue;
3422
3423         idx.tag_seek[import_tags[i]] = data;
3424
3425         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3426             current_tcmh.serial = data + 1;
3427         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3428             current_tcmh.commitid = data + 1;
3429     }
3430
3431     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3432 }
3433
3434 bool tagcache_import_changelog(void)
3435 {
3436     struct master_header myhdr;
3437     struct tagcache_header tch;
3438     int clfd;
3439     long masterfd;
3440     char buf[2048];
3441
3442     if (!tc_stat.ready)
3443         return false;
3444
3445     while (read_lock)
3446         sleep(1);
3447
3448     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3449     if (clfd < 0)
3450     {
3451         logf("failure to open changelog");
3452         return false;
3453     }
3454
3455     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3456     {
3457         close(clfd);
3458         return false;
3459     }
3460
3461     write_lock++;
3462
3463     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3464
3465     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3466                   parse_changelog_line);
3467
3468     close(clfd);
3469     close(masterfd);
3470
3471     if (filenametag_fd >= 0)
3472     {
3473         close(filenametag_fd);
3474         filenametag_fd = -1;
3475     }
3476
3477     write_lock--;
3478
3479     update_master_header();
3480
3481     return true;
3482 }
3483
3484 #endif /* !__PCTOOL__ */
3485
3486 bool tagcache_create_changelog(struct tagcache_search *tcs)
3487 {
3488     struct master_header myhdr;
3489     struct index_entry idx;
3490     char buf[TAG_MAXLEN+32];
3491     char temp[32];
3492     int clfd;
3493     int i, j;
3494
3495     if (!tc_stat.ready)
3496         return false;
3497
3498     if (!tagcache_search(tcs, tag_filename))
3499         return false;
3500
3501     /* Initialize the changelog */
3502     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3503     if (clfd < 0)
3504     {
3505         logf("failure to open changelog");
3506         return false;
3507     }
3508
3509     if (tcs->masterfd < 0)
3510     {
3511         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3512             return false;
3513     }
3514     else
3515     {
3516         lseek(tcs->masterfd, 0, SEEK_SET);
3517         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3518     }
3519
3520     write(clfd, "## Changelog version 1\n", 23);
3521
3522     for (i = 0; i < myhdr.tch.entry_count; i++)
3523     {
3524         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3525         {
3526             logf("read error #9");
3527             tagcache_search_finish(tcs);
3528             close(clfd);
3529             return false;
3530         }
3531
3532         /* Skip until the entry found has been modified. */
3533         if (! (idx.flag & FLAG_DIRTYNUM) )
3534             continue;
3535
3536         /* Skip deleted entries too. */
3537         if (idx.flag & FLAG_DELETED)
3538             continue;
3539
3540         /* Now retrieve all tags. */
3541         for (j = 0; j < TAG_COUNT; j++)
3542         {
3543             if (TAGCACHE_IS_NUMERIC(j))
3544             {
3545                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3546                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3547                 continue;
3548             }
3549
3550             tcs->type = j;
3551             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3552             write_tag(clfd, tagcache_tag_to_str(j), buf);
3553         }
3554
3555         write(clfd, "\n", 1);
3556         do_timed_yield();
3557     }
3558
3559     close(clfd);
3560
3561     tagcache_search_finish(tcs);
3562
3563     return true;
3564 }
3565
3566 static bool delete_entry(long idx_id)
3567 {
3568     int fd = -1;
3569     int masterfd = -1;
3570     int tag, i;
3571     struct index_entry idx, myidx;
3572     struct master_header myhdr;
3573     char buf[TAG_MAXLEN+32];
3574     int in_use[TAG_COUNT];
3575
3576     logf("delete_entry(): %ld", idx_id);
3577
3578 #ifdef HAVE_TC_RAMCACHE
3579     /* At first mark the entry removed from ram cache. */
3580     if (tc_stat.ramcache)
3581         ramcache_hdr->indices[idx_id].flag |= FLAG_DELETED;
3582 #endif
3583
3584     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3585         return false;
3586
3587     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3588     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3589     {
3590         logf("delete_entry(): read error");
3591         goto cleanup;
3592     }
3593
3594     if (myidx.flag & FLAG_DELETED)
3595     {
3596         logf("delete_entry(): already deleted!");
3597         goto cleanup;
3598     }
3599
3600     myidx.flag |= FLAG_DELETED;
3601     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3602     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3603     {
3604         logf("delete_entry(): write_error #1");
3605         goto cleanup;
3606     }
3607
3608     /* Now check which tags are no longer in use (if any) */
3609     for (tag = 0; tag < TAG_COUNT; tag++)
3610         in_use[tag] = 0;
3611
3612     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3613     for (i = 0; i < myhdr.tch.entry_count; i++)
3614     {
3615         struct index_entry *idxp;
3616
3617 #ifdef HAVE_TC_RAMCACHE
3618         /* Use RAM DB if available for greater speed */
3619         if (tc_stat.ramcache)
3620             idxp = &ramcache_hdr->indices[i];
3621         else
3622 #endif
3623         {
3624             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3625             {
3626                 logf("delete_entry(): read error #2");
3627                 goto cleanup;
3628             }
3629             idxp = &idx;
3630         }
3631
3632         if (idxp->flag & FLAG_DELETED)
3633             continue;
3634
3635         for (tag = 0; tag < TAG_COUNT; tag++)
3636         {
3637             if (TAGCACHE_IS_NUMERIC(tag))
3638                 continue;
3639
3640             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3641                 in_use[tag]++;
3642         }
3643     }
3644
3645     /* Now delete all tags no longer in use. */
3646     for (tag = 0; tag < TAG_COUNT; tag++)
3647     {
3648         struct tagcache_header tch;
3649         int oldseek = myidx.tag_seek[tag];
3650
3651         if (TAGCACHE_IS_NUMERIC(tag))
3652             continue;
3653
3654         /**
3655          * Replace tag seek with a hash value of the field string data.
3656          * That way runtime statistics of moved or altered files can be
3657          * resurrected.
3658          */
3659 #ifdef HAVE_TC_RAMCACHE
3660         if (tc_stat.ramcache && tag != tag_filename)
3661         {
3662             struct tagfile_entry *tfe;
3663             int32_t *seek = &ramcache_hdr->indices[idx_id].tag_seek[tag];
3664
3665             tfe = (struct tagfile_entry *)&ramcache_hdr->tags[tag][*seek];
3666             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3667             myidx.tag_seek[tag] = *seek;
3668         }
3669         else
3670 #endif
3671         {
3672             struct tagfile_entry tfe;
3673
3674             /* Open the index file, which contains the tag names. */
3675             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3676                 goto cleanup;
3677
3678             /* Skip the header block */
3679             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3680             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3681             {
3682                 logf("delete_entry(): read error #3");
3683                 goto cleanup;
3684             }
3685
3686             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3687             {
3688                 logf("delete_entry(): read error #4");
3689                 goto cleanup;
3690             }
3691
3692             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3693         }
3694
3695         if (in_use[tag])
3696         {
3697             logf("in use: %d/%d", tag, in_use[tag]);
3698             if (fd >= 0)
3699             {
3700                 close(fd);
3701                 fd = -1;
3702             }
3703             continue;
3704         }
3705
3706 #ifdef HAVE_TC_RAMCACHE
3707         /* Delete from ram. */
3708         if (tc_stat.ramcache && tag != tag_filename)
3709         {
3710             struct tagfile_entry *tagentry =
3711                     (struct tagfile_entry *)&ramcache_hdr->tags[tag][oldseek];
3712             tagentry->tag_data[0] = '\0';
3713         }
3714 #endif
3715
3716         /* Open the index file, which contains the tag names. */
3717         if (fd < 0)
3718         {
3719             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3720                 goto cleanup;
3721         }
3722
3723         /* Skip the header block */
3724         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3725
3726         /* Debug, print 10 first characters of the tag
3727         read(fd, buf, 10);
3728         buf[10]='\0';
3729         logf("TAG:%s", buf);
3730         lseek(fd, -10, SEEK_CUR);
3731         */
3732
3733         /* Write first data byte in tag as \0 */
3734         write(fd, "", 1);
3735
3736         /* Now tag data has been removed */
3737         close(fd);
3738         fd = -1;
3739     }
3740
3741     /* Write index entry back into master index. */
3742     lseek(masterfd, sizeof(struct master_header) +
3743           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3744     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3745     {
3746         logf("delete_entry(): write_error #2");
3747         goto cleanup;
3748     }
3749
3750     close(masterfd);
3751
3752     return true;
3753
3754     cleanup:
3755     if (fd >= 0)
3756         close(fd);
3757     if (masterfd >= 0)
3758         close(masterfd);
3759
3760     return false;
3761 }
3762
3763 #ifndef __PCTOOL__
3764 /**
3765  * Returns true if there is an event waiting in the queue
3766  * that requires the current operation to be aborted.
3767  */
3768 static bool check_event_queue(void)
3769 {
3770     struct queue_event ev;
3771
3772     if(!queue_peek(&tagcache_queue, &ev))
3773         return false;
3774
3775     switch (ev.id)
3776     {
3777         case Q_STOP_SCAN:
3778         case SYS_POWEROFF:
3779         case SYS_USB_CONNECTED:
3780             return true;
3781     }
3782
3783     return false;
3784 }
3785 #endif
3786
3787 #ifdef HAVE_TC_RAMCACHE
3788 static bool allocate_tagcache(void)
3789 {
3790     struct master_header tcmh;
3791     int fd;
3792
3793     /* Load the header. */
3794     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3795     {
3796         ramcache_hdr = NULL;
3797         return false;
3798     }
3799
3800     close(fd);
3801
3802     /**
3803      * Now calculate the required cache size plus
3804      * some extra space for alignment fixes.
3805      */
3806     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3807         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3808     ramcache_hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3809     memset(ramcache_hdr, 0, sizeof(struct ramcache_header));
3810     memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3811     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3812
3813     return true;
3814 }
3815
3816 # ifdef HAVE_EEPROM_SETTINGS
3817 static bool tagcache_dumpload(void)
3818 {
3819     struct statefile_header shdr;
3820     int fd, rc;
3821     long offpos;
3822     int i;
3823
3824     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3825     if (fd < 0)
3826     {
3827         logf("no tagcache statedump");
3828         return false;
3829     }
3830
3831     /* Check the statefile memory placement */
3832     ramcache_hdr = buffer_alloc(0);
3833     rc = read(fd, &shdr, sizeof(struct statefile_header));
3834     if (rc != sizeof(struct statefile_header)
3835         || shdr.magic != TAGCACHE_STATEFILE_MAGIC
3836         || shdr.mh.tch.magic != TAGCACHE_MAGIC)
3837     {
3838         logf("incorrect statefile");
3839         ramcache_hdr = NULL;
3840         close(fd);
3841         return false;
3842     }
3843
3844     offpos = (long)ramcache_hdr - (long)shdr.hdr;
3845
3846     /* Lets allocate real memory and load it */
3847     ramcache_hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3848     rc = read(fd, ramcache_hdr, shdr.tc_stat.ramcache_allocated);
3849     close(fd);
3850
3851     if (rc != shdr.tc_stat.ramcache_allocated)
3852     {
3853         logf("read failure!");
3854         ramcache_hdr = NULL;
3855         return false;
3856     }
3857
3858     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3859
3860     /* Now fix the pointers */
3861     for (i = 0; i < TAG_COUNT; i++)
3862         ramcache_hdr->tags[i] += offpos;
3863
3864     /* Load the tagcache master header (should match the actual DB file header). */
3865     memcpy(&current_tcmh, &shdr.mh, sizeof current_tcmh);
3866
3867     return true;
3868 }
3869
3870 static bool tagcache_dumpsave(void)
3871 {
3872     struct statefile_header shdr;
3873     int fd;
3874
3875     if (!tc_stat.ramcache)
3876         return false;
3877
3878     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3879     if (fd < 0)
3880     {
3881         logf("failed to create a statedump");
3882         return false;
3883     }
3884
3885     /* Create the header */
3886     shdr.magic = TAGCACHE_STATEFILE_MAGIC;
3887     shdr.hdr = ramcache_hdr;
3888     memcpy(&shdr.mh, &current_tcmh, sizeof current_tcmh);
3889     memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat);
3890     write(fd, &shdr, sizeof shdr);
3891
3892     /* And dump the data too */
3893     write(fd, ramcache_hdr, tc_stat.ramcache_allocated);
3894     close(fd);
3895
3896     return true;
3897 }
3898 # endif
3899
3900 static bool load_tagcache(void)
3901 {
3902     struct tagcache_header *tch;
3903     struct master_header tcmh;
3904     long bytesleft = tc_stat.ramcache_allocated - sizeof(struct ramcache_header);
3905     struct index_entry *idx;
3906     int rc, fd;
3907     char *p;
3908     int i, tag;
3909
3910 # ifdef HAVE_DIRCACHE
3911     while (dircache_is_initializing())
3912         sleep(1);
3913
3914     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3915 # endif
3916
3917     logf("loading tagcache to ram...");
3918
3919     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3920     if (fd < 0)
3921     {
3922         logf("tagcache open failed");
3923         return false;
3924     }
3925
3926     if (ecread(fd, &tcmh, 1, master_header_ec, tc_stat.econ)
3927         != sizeof(struct master_header)
3928         || tcmh.tch.magic != TAGCACHE_MAGIC)
3929     {
3930         logf("incorrect header");
3931         return false;
3932     }
3933
3934     /* Master header copy should already match, this can be redundant to do. */
3935     memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3936
3937     idx = ramcache_hdr->indices;
3938
3939     /* Load the master index table. */
3940     for (i = 0; i < tcmh.tch.entry_count; i++)
3941     {
3942         bytesleft -= sizeof(struct index_entry);
3943         if (bytesleft < 0)
3944         {
3945             logf("too big tagcache.");
3946             close(fd);
3947             return false;
3948         }
3949
3950         /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
3951          * may become corrupt. */
3952         rc = ecread_index_entry(fd, idx);
3953         if (rc != sizeof(struct index_entry))
3954         {
3955             logf("read error #10");
3956             close(fd);
3957             return false;
3958         }
3959
3960         idx++;
3961     }
3962
3963     close(fd);
3964
3965     /* Load the tags. */
3966     p = (char *)idx;
3967     for (tag = 0; tag < TAG_COUNT; tag++)
3968     {
3969         struct tagfile_entry *fe;
3970         char buf[TAG_MAXLEN+32];
3971
3972         if (TAGCACHE_IS_NUMERIC(tag))
3973             continue ;
3974
3975         //p = ((void *)p+1);
3976         p = (char *)((long)p & ~0x03) + 0x04;
3977         ramcache_hdr->tags[tag] = p;
3978
3979         /* Check the header. */
3980         tch = (struct tagcache_header *)p;
3981         p += sizeof(struct tagcache_header);
3982
3983         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3984             return false;
3985
3986         for (ramcache_hdr->entry_count[tag] = 0;
3987              ramcache_hdr->entry_count[tag] < tch->entry_count;
3988              ramcache_hdr->entry_count[tag]++)
3989         {
3990             long pos;
3991
3992             if (do_timed_yield())
3993             {
3994                 /* Abort if we got a critical event in queue */
3995                 if (check_event_queue())
3996                     return false;
3997             }
3998
3999             fe = (struct tagfile_entry *)p;
4000             pos = lseek(fd, 0, SEEK_CUR);
4001             rc = ecread_tagfile_entry(fd, fe);
4002             if (rc != sizeof(struct tagfile_entry))
4003             {
4004                 /* End of lookup table. */
4005                 logf("read error #11");
4006                 close(fd);
4007                 return false;
4008             }
4009
4010             /* We have a special handling for the filename tags. */
4011             if (tag == tag_filename)
4012             {
4013 # ifdef HAVE_DIRCACHE
4014                 int dc;
4015 # endif
4016
4017                 idx = &ramcache_hdr->indices[fe->idx_id];
4018
4019                 if (fe->tag_length >= (long)sizeof(buf)-1)
4020                 {
4021                     read(fd, buf, 10);
4022                     buf[10] = '\0';
4023                     logf("TAG:%s", buf);
4024                     logf("too long filename");
4025                     close(fd);
4026                     return false;
4027                 }
4028
4029                 rc = read(fd, buf, fe->tag_length);
4030                 if (rc != fe->tag_length)
4031                 {
4032                     logf("read error #12");
4033                     close(fd);
4034                     return false;
4035                 }
4036
4037                 /* Check if the entry has already been removed */
4038                 if (idx->flag & FLAG_DELETED)
4039                     continue;
4040
4041                 /* This flag must not be used yet. */
4042                 if (idx->flag & FLAG_DIRCACHE)
4043                 {
4044                     logf("internal error!");
4045                     close(fd);
4046                     return false;
4047                 }
4048
4049                 if (idx->tag_seek[tag] != pos)
4050                 {
4051                     logf("corrupt data structures!");
4052                     close(fd);
4053                     return false;
4054                 }
4055
4056 # ifdef HAVE_DIRCACHE
4057                 if (dircache_is_enabled())
4058                 {
4059                     dc = dircache_get_entry_id(buf);
4060                     if (dc < 0)
4061                     {
4062                         logf("Entry no longer valid.");
4063                         logf("-> %s", buf);
4064                         if (global_settings.tagcache_autoupdate)
4065                             delete_entry(fe->idx_id);
4066                         continue ;
4067                     }
4068
4069                     idx->flag |= FLAG_DIRCACHE;
4070                     idx->tag_seek[tag_filename] = dc;
4071                 }
4072                 else
4073 # endif
4074                 {
4075                     /* This will be very slow unless dircache is enabled
4076                        or target is flash based, but do it anyway for
4077                        consistency. */
4078                     /* Check if entry has been removed. */
4079                     if (global_settings.tagcache_autoupdate)
4080                     {
4081                         if (!file_exists(buf))
4082                         {
4083                             logf("Entry no longer valid.");
4084                             logf("-> %s", buf);
4085                             delete_entry(fe->idx_id);
4086                             continue;
4087                         }
4088                     }
4089                 }
4090
4091                 continue ;
4092             }
4093
4094             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4095             if (bytesleft < 0)
4096             {
4097                 logf("too big tagcache #2");
4098                 logf("tl: %ld", fe->tag_length);
4099                 logf("bl: %ld", bytesleft);
4100                 close(fd);
4101                 return false;
4102             }
4103
4104             p = fe->tag_data;
4105             rc = read(fd, fe->tag_data, fe->tag_length);
4106             p += rc;
4107
4108             if (rc != fe->tag_length)
4109             {
4110                 logf("read error #13");
4111                 logf("rc=0x%04x", rc); // 0x431
4112                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4113                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4114                 logf("tag=0x%02x", tag); // 0x00
4115                 close(fd);
4116                 return false;
4117             }
4118         }
4119         close(fd);
4120     }
4121
4122     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4123     logf("tagcache loaded into ram!");
4124
4125     return true;
4126 }
4127 #endif /* HAVE_TC_RAMCACHE */
4128
4129 static bool check_deleted_files(void)
4130 {
4131     int fd;
4132     char buf[TAG_MAXLEN+32];
4133     struct tagfile_entry tfe;
4134
4135     logf("reverse scan...");
4136     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4137     fd = open(buf, O_RDONLY);
4138
4139     if (fd < 0)
4140     {
4141         logf("%s open fail", buf);
4142         return false;
4143     }
4144
4145     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4146     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4147 #ifndef __PCTOOL__
4148            && !check_event_queue()
4149 #endif
4150            )
4151     {
4152         if (tfe.tag_length >= (long)sizeof(buf)-1)
4153         {
4154             logf("too long tag");
4155             close(fd);
4156             return false;
4157         }
4158
4159         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4160         {
4161             logf("read error #14");
4162             close(fd);
4163             return false;
4164         }
4165
4166         /* Check if the file has already deleted from the db. */
4167         if (*buf == '\0')
4168             continue;
4169
4170         /* Now check if the file exists. */
4171         if (!file_exists(buf))
4172         {
4173             logf("Entry no longer valid.");
4174             logf("-> %s / %ld", buf, tfe.tag_length);
4175             delete_entry(tfe.idx_id);
4176         }
4177     }
4178
4179     close(fd);
4180
4181     logf("done");
4182
4183     return true;
4184 }
4185
4186
4187 /* Note that this function must not be inlined, otherwise the whole point
4188  * of having the code in a separate function is lost.
4189  */
4190 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4191     int *ignore, int *unignore)
4192 {
4193     char newpath[MAX_PATH];
4194
4195     /* check for a database.ignore file */
4196     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4197     *ignore = file_exists(newpath);
4198     /* check for a database.unignore file */
4199     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4200     *unignore = file_exists(newpath);
4201 }
4202
4203 static struct search_roots_ll {
4204     const char *path;
4205     struct search_roots_ll * next;
4206 } roots_ll;
4207
4208 #ifdef APPLICATION
4209 /*
4210  * This adds a path to the search roots, possibly during traveling through
4211  * the filesystem. It only adds if the path is not inside an already existing
4212  * search root.
4213  *
4214  * Returns true if it added the path to the search roots
4215  *
4216  * Windows 2000 and greater supports symlinks, but they don't provide
4217  * realpath() or readlink(), and symlinks are rarely used on them so
4218  * ignore this for windows for now
4219  **/
4220 static bool add_search_root(const char *name)
4221 {
4222     (void)name;
4223 #ifndef WIN32
4224     struct search_roots_ll *this, *prev = NULL;
4225     char target[MAX_PATH];
4226     /* Okay, realpath() is almost completely broken on android
4227      *
4228      * It doesn't accept NULL for resolved_name to dynamically allocate
4229      * the resulting path; and it assumes resolved_name to be PATH_MAX
4230      * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4231      * and blindly writes to the end if it
4232      *
4233      * therefore use sufficiently large static storage here
4234      * Note that PATH_MAX != MAX_PATH
4235      **/
4236     static char abs_target[PATH_MAX];
4237     ssize_t len;
4238
4239     len = readlink(name, target, sizeof(target));
4240     if (len < 0)
4241         return false;
4242
4243     target[len] = '\0';
4244     if (realpath(target, abs_target) == NULL)
4245         return false;
4246
4247     for(this = &roots_ll; this; prev = this, this = this->next)
4248     {
4249         size_t root_len = strlen(this->path);
4250         /* check if the link target is inside of an existing search root
4251          * don't add if target is inside, we'll scan it later */
4252         if (!strncmp(this->path, abs_target, root_len))
4253             return false;
4254     }
4255
4256     if (prev)
4257     {
4258         size_t len = strlen(abs_target) + 1; /* count \0 */
4259         this = malloc(sizeof(struct search_roots_ll) + len );
4260         if (!this || len > MAX_PATH)
4261         {
4262             logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4263             free(this);
4264             prev->next = NULL;
4265             return false;
4266         }
4267         this->path = ((char*)this) + sizeof(struct search_roots_ll);
4268         strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4269         this->next = NULL;
4270         prev->next = this;
4271         logf("Added %s to the search roots\n", abs_target);
4272         return true;
4273     }
4274 #endif
4275     return false;
4276 }
4277
4278 static int free_search_roots(struct search_roots_ll * start)
4279 {
4280     int ret = 0;
4281     if (start->next)
4282     {
4283         ret += free_search_roots(start->next);
4284         ret += sizeof(struct search_roots_ll);
4285         free(start->next);
4286     }
4287     return ret;
4288 }
4289 #else /* native, simulator */
4290 #define add_search_root(a) do {} while(0)
4291 #define free_search_roots(a) do {} while(0)
4292 #endif
4293
4294 static bool check_dir(const char *dirname, int add_files)
4295 {
4296     DIR *dir;
4297     int len;
4298     int success = false;
4299     int ignore, unignore;
4300
4301     dir = opendir(dirname);
4302     if (!dir)
4303     {
4304         logf("tagcache: opendir(%s) failed", dirname);
4305         return false;
4306     }
4307     /* check for a database.ignore and database.unignore */
4308     check_ignore(dirname, &ignore, &unignore);
4309
4310     /* don't do anything if both ignore and unignore are there */
4311     if (ignore != unignore)
4312         add_files = unignore;
4313
4314     /* Recursively scan the dir. */
4315 #ifdef __PCTOOL__
4316     while (1)
4317 #else
4318     while (!check_event_queue())
4319 #endif
4320     {
4321         struct dirent *entry = readdir(dir);
4322         if (entry == NULL)
4323         {
4324             success = true;
4325             break;
4326         }
4327
4328         if (!strcmp((char *)entry->d_name, ".") ||
4329             !strcmp((char *)entry->d_name, ".."))
4330             continue;
4331
4332         struct dirinfo info = dir_get_info(dir, entry);
4333
4334         yield();
4335
4336         len = strlen(curpath);
4337         /* don't add an extra / for curpath == / */
4338         if (len <= 1) len = 0;
4339         snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4340
4341         processed_dir_count++;
4342         if (info.attribute & ATTR_DIRECTORY)
4343 #ifndef SIMULATOR
4344         {   /* don't follow symlinks to dirs, but try to add it as a search root
4345              * this makes able to avoid looping in recursive symlinks */
4346             if (info.attribute & ATTR_LINK)
4347                 add_search_root(curpath);
4348             else
4349                 check_dir(curpath, add_files);
4350         }
4351 #else
4352             check_dir(curpath, add_files);
4353 #endif
4354         else if (add_files)
4355         {
4356             tc_stat.curentry = curpath;
4357
4358             /* Add a new entry to the temporary db file. */
4359             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4360 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4361                          , dir->internal_entry
4362 #endif
4363                          );
4364
4365             /* Wait until current path for debug screen is read and unset. */
4366             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4367                 yield();
4368
4369             tc_stat.curentry = NULL;
4370         }
4371
4372         curpath[len] = '\0';
4373     }
4374
4375     closedir(dir);
4376
4377     return success;
4378 }
4379
4380 void tagcache_screensync_event(void)
4381 {
4382     tc_stat.curentry = NULL;
4383 }
4384
4385 void tagcache_screensync_enable(bool state)
4386 {
4387     tc_stat.syncscreen = state;
4388 }
4389
4390 void tagcache_build(const char *path)
4391 {
4392     struct tagcache_header header;
4393     bool ret;
4394
4395     curpath[0] = '\0';
4396     data_size = 0;
4397     total_entry_count = 0;
4398     processed_dir_count = 0;
4399
4400 #ifdef HAVE_DIRCACHE
4401     while (dircache_is_initializing())
4402         sleep(1);
4403 #endif
4404
4405     logf("updating tagcache");
4406
4407     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4408     if (cachefd >= 0)
4409     {
4410         logf("skipping, cache already waiting for commit");
4411         close(cachefd);
4412         return ;
4413     }
4414
4415     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4416     if (cachefd < 0)
4417     {
4418         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4419         return ;
4420     }
4421
4422     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4423
4424     cpu_boost(true);
4425
4426     logf("Scanning files...");
4427     /* Scan for new files. */
4428     memset(&header, 0, sizeof(struct tagcache_header));
4429     write(cachefd, &header, sizeof(struct tagcache_header));
4430
4431     ret = true;
4432     roots_ll.path = path;
4433     roots_ll.next = NULL;
4434     struct search_roots_ll * this;
4435     /* check_dir might add new roots */
4436     for(this = &roots_ll; this; this = this->next)
4437     {
4438         strcpy(curpath, this->path);
4439         ret = ret && check_dir(this->path, true);
4440     }
4441     if (roots_ll.next)
4442         free_search_roots(roots_ll.next);
4443
4444     /* Write the header. */
4445     header.magic = TAGCACHE_MAGIC;
4446     header.datasize = data_size;
4447     header.entry_count = total_entry_count;
4448     lseek(cachefd, 0, SEEK_SET);
4449     write(cachefd, &header, sizeof(struct tagcache_header));
4450     close(cachefd);
4451
4452     if (filenametag_fd >= 0)
4453     {
4454         close(filenametag_fd);
4455         filenametag_fd = -1;
4456     }
4457
4458     if (!ret)
4459     {
4460         logf("Aborted.");
4461         cpu_boost(false);
4462         return ;
4463     }
4464
4465     /* Commit changes to the database. */
4466 #ifdef __PCTOOL__
4467     allocate_tempbuf();
4468 #endif
4469     if (commit())
4470     {
4471         logf("tagcache built!");
4472     }
4473 #ifdef __PCTOOL__
4474     free_tempbuf();
4475 #endif
4476
4477 #ifdef HAVE_TC_RAMCACHE
4478     if (ramcache_hdr)
4479     {
4480         /* Import runtime statistics if we just initialized the db. */
4481         if (current_tcmh.serial == 0)
4482             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4483     }
4484 #endif
4485
4486     cpu_boost(false);
4487 }
4488
4489 #ifdef HAVE_TC_RAMCACHE
4490 static void load_ramcache(void)
4491 {
4492     if (!ramcache_hdr)
4493         return ;
4494
4495     cpu_boost(true);
4496
4497     /* At first we should load the cache (if exists). */
4498     tc_stat.ramcache = load_tagcache();
4499
4500     if (!tc_stat.ramcache)
4501     {
4502         /* If loading failed, it must indicate some problem with the db
4503          * so disable it entirely to prevent further issues. */
4504         tc_stat.ready = false;
4505         ramcache_hdr = NULL;
4506     }
4507
4508     cpu_boost(false);
4509 }
4510
4511 void tagcache_unload_ramcache(void)
4512 {
4513     tc_stat.ramcache = false;
4514     /* Just to make sure there is no statefile present. */
4515     // remove(TAGCACHE_STATEFILE);
4516 }
4517 #endif
4518
4519 #ifndef __PCTOOL__
4520 static void tagcache_thread(void)
4521 {
4522     struct queue_event ev;
4523     bool check_done = false;
4524
4525     /* If the previous cache build/update was interrupted, commit
4526      * the changes first in foreground. */
4527     cpu_boost(true);
4528     allocate_tempbuf();
4529     commit();
4530     free_tempbuf();
4531
4532 #ifdef HAVE_TC_RAMCACHE
4533 # ifdef HAVE_EEPROM_SETTINGS
4534     if (firmware_settings.initialized && firmware_settings.disk_clean
4535         && global_settings.tagcache_ram)
4536     {
4537         check_done = tagcache_dumpload();
4538     }
4539
4540     remove(TAGCACHE_STATEFILE);
4541 # endif
4542
4543     /* Allocate space for the tagcache if found on disk. */
4544     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4545         allocate_tagcache();
4546 #endif
4547
4548     cpu_boost(false);
4549     tc_stat.initialized = true;
4550
4551     /* Don't delay bootup with the header check but do it on background. */
4552     if (!tc_stat.ready)
4553     {
4554         sleep(HZ);
4555         tc_stat.ready = check_all_headers();
4556         tc_stat.readyvalid = true;
4557     }
4558
4559     while (1)
4560     {
4561         run_command_queue(false);
4562
4563         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4564
4565         switch (ev.id)
4566         {
4567             case Q_IMPORT_CHANGELOG:
4568                 tagcache_import_changelog();
4569                 break;
4570
4571             case Q_REBUILD:
4572                 remove_files();
4573                 remove(TAGCACHE_FILE_TEMP);
4574                 tagcache_build("/");
4575                 break;
4576
4577             case Q_UPDATE:
4578                 tagcache_build("/");
4579 #ifdef HAVE_TC_RAMCACHE
4580                 load_ramcache();
4581 #endif
4582                 check_deleted_files();
4583                 break ;
4584
4585             case Q_START_SCAN:
4586                 check_done = false;
4587             case SYS_TIMEOUT:
4588                 if (check_done || !tc_stat.ready)
4589                     break ;
4590
4591 #ifdef HAVE_TC_RAMCACHE
4592                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4593                 {
4594                     load_ramcache();
4595                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4596                         tagcache_build("/");
4597                 }
4598                 else
4599 #endif
4600                 if (global_settings.tagcache_autoupdate)
4601                 {
4602                     tagcache_build("/");
4603
4604                     /* This will be very slow unless dircache is enabled
4605                        or target is flash based, but do it anyway for
4606                        consistency. */
4607                     check_deleted_files();
4608                 }
4609
4610                 logf("tagcache check done");
4611
4612                 check_done = true;
4613                 break ;
4614
4615             case Q_STOP_SCAN:
4616                 break ;
4617
4618             case SYS_POWEROFF:
4619                 break ;
4620
4621 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4622             case SYS_USB_CONNECTED:
4623                 logf("USB: TagCache");
4624                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4625                 usb_wait_for_disconnect(&tagcache_queue);
4626                 break ;
4627 #endif
4628         }
4629     }
4630 }
4631
4632 bool tagcache_prepare_shutdown(void)
4633 {
4634     if (tagcache_get_commit_step() > 0)
4635         return false;
4636
4637     tagcache_stop_scan();
4638     while (read_lock || write_lock)
4639         sleep(1);
4640
4641     return true;
4642 }
4643
4644 void tagcache_shutdown(void)
4645 {
4646     /* Flush the command queue. */
4647     run_command_queue(true);
4648
4649 #ifdef HAVE_EEPROM_SETTINGS
4650     if (tc_stat.ramcache)
4651         tagcache_dumpsave();
4652 #endif
4653 }
4654
4655 static int get_progress(void)
4656 {
4657     int total_count = -1;
4658
4659 #ifdef HAVE_DIRCACHE
4660     if (dircache_is_enabled())
4661     {
4662         total_count = dircache_get_entry_count();
4663     }
4664     else
4665 #endif
4666 #ifdef HAVE_TC_RAMCACHE
4667     {
4668         if (ramcache_hdr && tc_stat.ramcache)
4669             total_count = current_tcmh.tch.entry_count;
4670     }
4671 #endif
4672
4673     if (total_count < 0)
4674         return -1;
4675
4676     return processed_dir_count * 100 / total_count;
4677 }
4678
4679 struct tagcache_stat* tagcache_get_stat(void)
4680 {
4681     tc_stat.progress = get_progress();
4682     tc_stat.processed_entries = processed_dir_count;
4683
4684     return &tc_stat;
4685 }
4686
4687 void tagcache_start_scan(void)
4688 {
4689     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4690 }
4691
4692 bool tagcache_update(void)
4693 {
4694     if (!tc_stat.ready)
4695         return false;
4696
4697     queue_post(&tagcache_queue, Q_UPDATE, 0);
4698     return false;
4699 }
4700
4701 bool tagcache_rebuild()
4702 {
4703     queue_post(&tagcache_queue, Q_REBUILD, 0);
4704     return false;
4705 }
4706
4707 void tagcache_stop_scan(void)
4708 {
4709     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4710 }
4711
4712 #ifdef HAVE_TC_RAMCACHE
4713 bool tagcache_is_ramcache(void)
4714 {
4715     return tc_stat.ramcache;
4716 }
4717 #endif
4718
4719 #endif /* !__PCTOOL__ */
4720
4721
4722 void tagcache_init(void)
4723 {
4724     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4725     memset(&current_tcmh, 0, sizeof(struct master_header));
4726     filenametag_fd = -1;
4727     write_lock = read_lock = 0;
4728
4729 #ifndef __PCTOOL__
4730     mutex_init(&command_queue_mutex);
4731     queue_init(&tagcache_queue, true);
4732     create_thread(tagcache_thread, tagcache_stack,
4733                   sizeof(tagcache_stack), 0, tagcache_thread_name
4734                   IF_PRIO(, PRIORITY_BACKGROUND)
4735                   IF_COP(, CPU));
4736 #else
4737     tc_stat.initialized = true;
4738     allocate_tempbuf();
4739     commit();
4740     free_tempbuf();
4741     tc_stat.ready = check_all_headers();
4742 #endif
4743 }
4744
4745 #ifdef __PCTOOL__
4746 void tagcache_reverse_scan(void)
4747 {
4748     logf("Checking for deleted files");
4749     check_deleted_files();
4750 }
4751 #endif
4752
4753 bool tagcache_is_initialized(void)
4754 {
4755     return tc_stat.initialized;
4756 }
4757 bool tagcache_is_fully_initialized(void)
4758 {
4759     return tc_stat.readyvalid;
4760 }
4761 bool tagcache_is_usable(void)
4762 {
4763     return tc_stat.initialized && tc_stat.ready;
4764 }
4765 int tagcache_get_commit_step(void)
4766 {
4767     return tc_stat.commit_step;
4768 }
4769 int tagcache_get_max_commit_step(void)
4770 {
4771     return (int)(SORTED_TAGS_COUNT)+1;
4772 }
4773