apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "filefuncs.h"
  79 #include "structec.h"
  80
  81 #ifndef __PCTOOL__
  82 #include "lang.h"
  83 #include "eeprom_settings.h"
  84 #endif
  85
  86 #ifdef __PCTOOL__
  87 #define yield() do { } while(0)
  88 #define sim_sleep(timeout) do { } while(0)
  89 #define do_timed_yield() do { } while(0)
  90 #endif
  91
  92 #ifndef __PCTOOL__
  93 /* Tag Cache thread. */
  94 static struct event_queue tagcache_queue;
  95 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  96 static const char tagcache_thread_name[] = "tagcache";
  97 #endif
  98
  99 /* Previous path when scanning directory tree recursively. */
 100 static char curpath[TAG_MAXLEN+32];
 101
 102 /* Used when removing duplicates. */
 103 static char *tempbuf;     /* Allocated when needed. */
 104 static long tempbufidx;   /* Current location in buffer. */
 105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 106 static long tempbuf_left; /* Buffer space left. */
 107 static long tempbuf_pos;
 108
 109 #define SORTED_TAGS_COUNT 8
 110 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 111 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 112 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 113     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 114 /* Tags we want to get sorted (loaded to the tempbuf). */
 115 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 116     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 117     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 118
 119 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 120 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping))
 123
 124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 125 static const char *tags_str[] = { "artist", "album", "genre", "title",
 126     "filename", "composer", "comment", "albumartist", "grouping", "year",
 127     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 128     "playtime", "lastplayed", "commitid", "mtime" };
 129
 130 /* Status information of the tagcache. */
 131 static struct tagcache_stat tc_stat;
 132
 133 /* Queue commands. */
 134 enum tagcache_queue {
 135     Q_STOP_SCAN = 0,
 136     Q_START_SCAN,
 137     Q_IMPORT_CHANGELOG,
 138     Q_UPDATE,
 139     Q_REBUILD,
 140
 141     /* Internal tagcache command queue. */
 142     CMD_UPDATE_MASTER_HEADER,
 143     CMD_UPDATE_NUMERIC,
 144 };
 145
 146 struct tagcache_command_entry {
 147     int32_t command;
 148     int32_t idx_id;
 149     int32_t tag;
 150     int32_t data;
 151 };
 152
 153 #ifndef __PCTOOL__
 154 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 155 static volatile int command_queue_widx = 0;
 156 static volatile int command_queue_ridx = 0;
 157 static struct mutex command_queue_mutex;
 158 #endif
 159
 160 /* Tag database structures. */
 161
 162 /* Variable-length tag entry in tag files. */
 163 struct tagfile_entry {
 164     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 165     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 166     char tag_data[0];  /* Begin of the tag data */
 167 };
 168
 169 /* Fixed-size tag entry in master db index. */
 170 struct index_entry {
 171     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 172     int32_t flag;                /* Status flags */
 173 };
 174
 175 /* Header is the same in every file. */
 176 struct tagcache_header {
 177     int32_t magic;       /* Header version number */
 178     int32_t datasize;    /* Data size in bytes */
 179     int32_t entry_count; /* Number of entries in this file */
 180 };
 181
 182 struct master_header {
 183     struct tagcache_header tch;
 184     int32_t serial; /* Increasing counting number */
 185     int32_t commitid; /* Number of commits so far */
 186     int32_t dirty;
 187 };
 188
 189 /* For the endianess correction */
 190 static const char *tagfile_entry_ec   = "ll";
 191 /**
 192  Note: This should be (1 + TAG_COUNT) amount of l's.
 193  */
 194 static const char *index_entry_ec     = "lllllllllllllllllllll";
 195
 196 static const char *tagcache_header_ec = "lll";
 197 static const char *master_header_ec   = "llllll";
 198
 199 static struct master_header current_tcmh;
 200
 201 #ifdef HAVE_TC_RAMCACHE
 202 /* Header is created when loading database to ram. */
 203 struct ramcache_header {
 204     struct master_header h;      /* Header from the master index */
 205     struct index_entry *indices; /* Master index file content */
 206     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 207     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 208 };
 209
 210 # ifdef HAVE_EEPROM_SETTINGS
 211 struct statefile_header {
 212     struct ramcache_header *hdr;
 213     struct tagcache_stat tc_stat;
 214 };
 215 # endif
 216
 217 /* Pointer to allocated ramcache_header */
 218 static struct ramcache_header *hdr;
 219 #endif
 220
 221 /**
 222  * Full tag entries stored in a temporary file waiting
 223  * for commit to the cache. */
 224 struct temp_file_entry {
 225     long tag_offset[TAG_COUNT];
 226     short tag_length[TAG_COUNT];
 227     long flag;
 228
 229     long data_length;
 230 };
 231
 232 struct tempbuf_id_list {
 233     long id;
 234     struct tempbuf_id_list *next;
 235 };
 236
 237 struct tempbuf_searchidx {
 238     long idx_id;
 239     char *str;
 240     int seek;
 241     struct tempbuf_id_list idlist;
 242 };
 243
 244 /* Lookup buffer for fixing messed up index while after sorting. */
 245 static long commit_entry_count;
 246 static long lookup_buffer_depth;
 247 static struct tempbuf_searchidx **lookup;
 248
 249 /* Used when building the temporary file. */
 250 static int cachefd = -1, filenametag_fd;
 251 static int total_entry_count = 0;
 252 static int data_size = 0;
 253 static int processed_dir_count;
 254
 255 /* Thread safe locking */
 256 static volatile int write_lock;
 257 static volatile int read_lock;
 258
 259 static bool delete_entry(long idx_id);
 260
 261 const char* tagcache_tag_to_str(int tag)
 262 {
 263     return tags_str[tag];
 264 }
 265
 266 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 267 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 268 {
 269     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 270 }
 271
 272 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 #ifdef HAVE_DIRCACHE
 283 /**
 284  * Returns true if specified flag is still present, i.e., dircache
 285  * has not been reloaded.
 286  */
 287 static bool is_dircache_intact(void)
 288 {
 289     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 290 }
 291 #endif
 292
 293 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 294 {
 295     int fd;
 296     char buf[MAX_PATH];
 297     int rc;
 298
 299     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 300         return -1;
 301
 302     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 303
 304     fd = open(buf, write ? O_RDWR : O_RDONLY);
 305     if (fd < 0)
 306     {
 307         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 308         tc_stat.ready = false;
 309         return fd;
 310     }
 311
 312     /* Check the header. */
 313     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 314     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 315     {
 316         logf("header error");
 317         tc_stat.ready = false;
 318         close(fd);
 319         return -2;
 320     }
 321
 322     return fd;
 323 }
 324
 325 static int open_master_fd(struct master_header *hdr, bool write)
 326 {
 327     int fd;
 328     int rc;
 329
 330     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 331     if (fd < 0)
 332     {
 333         logf("master file open failed for R/W");
 334         tc_stat.ready = false;
 335         return fd;
 336     }
 337
 338     tc_stat.econ = false;
 339
 340     /* Check the header. */
 341     rc = read(fd, hdr, sizeof(struct master_header));
 342     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 343     {
 344         /* Success. */
 345         return fd;
 346     }
 347
 348     /* Trying to read again, this time with endianess correction enabled. */
 349     lseek(fd, 0, SEEK_SET);
 350
 351     rc = ecread(fd, hdr, 1, master_header_ec, true);
 352     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 353     {
 354         logf("header error");
 355         tc_stat.ready = false;
 356         close(fd);
 357         return -2;
 358     }
 359
 360     tc_stat.econ = true;
 361
 362     return fd;
 363 }
 364
 365 #ifndef __PCTOOL__
 366 static bool do_timed_yield(void)
 367 {
 368     /* Sorting can lock up for quite a while, so yield occasionally */
 369     static long wakeup_tick = 0;
 370     if (TIME_AFTER(current_tick, wakeup_tick))
 371     {
 372         wakeup_tick = current_tick + (HZ/4);
 373         yield();
 374         return true;
 375     }
 376     return false;
 377 }
 378 #endif
 379
 380 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 381 static long find_entry_ram(const char *filename,
 382                            const struct dircache_entry *dc)
 383 {
 384     static long last_pos = 0;
 385     int i;
 386
 387     /* Check if tagcache is loaded into ram. */
 388     if (!tc_stat.ramcache)
 389         return -1;
 390
 391     if (dc == NULL)
 392         dc = dircache_get_entry_ptr(filename);
 393
 394     if (dc == NULL)
 395     {
 396         logf("tagcache: file not found.");
 397         return -1;
 398     }
 399
 400     try_again:
 401
 402     if (last_pos > 0)
 403         i = last_pos;
 404     else
 405         i = 0;
 406
 407     for (; i < hdr->h.tch.entry_count; i++)
 408     {
 409         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 410         {
 411             last_pos = MAX(0, i - 3);
 412             return i;
 413         }
 414
 415         do_timed_yield();
 416     }
 417
 418     if (last_pos > 0)
 419     {
 420         last_pos = 0;
 421         goto try_again;
 422     }
 423
 424     return -1;
 425 }
 426 #endif
 427
 428 static long find_entry_disk(const char *filename, bool localfd)
 429 {
 430     struct tagcache_header tch;
 431     static long last_pos = -1;
 432     long pos_history[POS_HISTORY_COUNT];
 433     long pos_history_idx = 0;
 434     bool found = false;
 435     struct tagfile_entry tfe;
 436     int fd;
 437     char buf[TAG_MAXLEN+32];
 438     int i;
 439     int pos = -1;
 440
 441     if (!tc_stat.ready)
 442         return -2;
 443
 444     fd = filenametag_fd;
 445     if (fd < 0 || localfd)
 446     {
 447         last_pos = -1;
 448         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 449             return -1;
 450     }
 451
 452     check_again:
 453
 454     if (last_pos > 0)
 455         lseek(fd, last_pos, SEEK_SET);
 456     else
 457         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 458
 459     while (true)
 460     {
 461         pos = lseek(fd, 0, SEEK_CUR);
 462         for (i = pos_history_idx-1; i >= 0; i--)
 463             pos_history[i+1] = pos_history[i];
 464         pos_history[0] = pos;
 465
 466         if (ecread_tagfile_entry(fd, &tfe)
 467             != sizeof(struct tagfile_entry))
 468         {
 469             break ;
 470         }
 471
 472         if (tfe.tag_length >= (long)sizeof(buf))
 473         {
 474             logf("too long tag #1");
 475             close(fd);
 476             if (!localfd)
 477                 filenametag_fd = -1;
 478             last_pos = -1;
 479             return -2;
 480         }
 481
 482         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 483         {
 484             logf("read error #2");
 485             close(fd);
 486             if (!localfd)
 487                 filenametag_fd = -1;
 488             last_pos = -1;
 489             return -3;
 490         }
 491
 492         if (!strcasecmp(filename, buf))
 493         {
 494             last_pos = pos_history[pos_history_idx];
 495             found = true;
 496             break ;
 497         }
 498
 499         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 500             pos_history_idx++;
 501     }
 502
 503     /* Not found? */
 504     if (!found)
 505     {
 506         if (last_pos > 0)
 507         {
 508             last_pos = -1;
 509             logf("seek again");
 510             goto check_again;
 511         }
 512
 513         if (fd != filenametag_fd || localfd)
 514             close(fd);
 515         return -4;
 516     }
 517
 518     if (fd != filenametag_fd || localfd)
 519         close(fd);
 520
 521     return tfe.idx_id;
 522 }
 523
 524 static int find_index(const char *filename)
 525 {
 526     long idx_id = -1;
 527
 528 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 529     if (tc_stat.ramcache && is_dircache_intact())
 530         idx_id = find_entry_ram(filename, NULL);
 531 #endif
 532
 533     if (idx_id < 0)
 534         idx_id = find_entry_disk(filename, true);
 535
 536     return idx_id;
 537 }
 538
 539 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 540 {
 541     int idx_id;
 542
 543     if (!tc_stat.ready)
 544         return false;
 545
 546     idx_id = find_index(filename);
 547     if (idx_id < 0)
 548         return false;
 549
 550     if (!tagcache_search(tcs, tag_filename))
 551         return false;
 552
 553     tcs->entry_count = 0;
 554     tcs->idx_id = idx_id;
 555
 556     return true;
 557 }
 558
 559 static bool get_index(int masterfd, int idxid,
 560                       struct index_entry *idx, bool use_ram)
 561 {
 562     bool localfd = false;
 563
 564     if (idxid < 0)
 565     {
 566         logf("Incorrect idxid: %d", idxid);
 567         return false;
 568     }
 569
 570 #ifdef HAVE_TC_RAMCACHE
 571     if (tc_stat.ramcache && use_ram)
 572     {
 573         if (hdr->indices[idxid].flag & FLAG_DELETED)
 574             return false;
 575
 576 # ifdef HAVE_DIRCACHE
 577         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 578             || is_dircache_intact())
 579 #endif
 580         {
 581             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 582             return true;
 583         }
 584     }
 585 #else
 586     (void)use_ram;
 587 #endif
 588
 589     if (masterfd < 0)
 590     {
 591         struct master_header tcmh;
 592
 593         localfd = true;
 594         masterfd = open_master_fd(&tcmh, false);
 595         if (masterfd < 0)
 596             return false;
 597     }
 598
 599     lseek(masterfd, idxid * sizeof(struct index_entry)
 600           + sizeof(struct master_header), SEEK_SET);
 601     if (ecread_index_entry(masterfd, idx)
 602         != sizeof(struct index_entry))
 603     {
 604         logf("read error #3");
 605         if (localfd)
 606             close(masterfd);
 607
 608         return false;
 609     }
 610
 611     if (localfd)
 612         close(masterfd);
 613
 614     if (idx->flag & FLAG_DELETED)
 615         return false;
 616
 617     return true;
 618 }
 619
 620 #ifndef __PCTOOL__
 621
 622 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 623 {
 624     /* We need to exclude all memory only flags & tags when writing to disk. */
 625     if (idx->flag & FLAG_DIRCACHE)
 626     {
 627         logf("memory only flags!");
 628         return false;
 629     }
 630
 631 #ifdef HAVE_TC_RAMCACHE
 632     /* Only update numeric data. Writing the whole index to RAM by memcpy
 633      * destroys dircache pointers!
 634      */
 635     if (tc_stat.ramcache)
 636     {
 637         int tag;
 638         struct index_entry *idx_ram = &hdr->indices[idxid];
 639
 640         for (tag = 0; tag < TAG_COUNT; tag++)
 641         {
 642             if (TAGCACHE_IS_NUMERIC(tag))
 643             {
 644                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 645             }
 646         }
 647
 648         /* Don't touch the dircache flag or attributes. */
 649         idx_ram->flag = (idx->flag & 0x0000ffff)
 650             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 651     }
 652 #endif
 653
 654     lseek(masterfd, idxid * sizeof(struct index_entry)
 655           + sizeof(struct master_header), SEEK_SET);
 656     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 657     {
 658         logf("write error #3");
 659         logf("idxid: %d", idxid);
 660         return false;
 661     }
 662
 663     return true;
 664 }
 665
 666 #endif /* !__PCTOOL__ */
 667
 668 static bool open_files(struct tagcache_search *tcs, int tag)
 669 {
 670     if (tcs->idxfd[tag] < 0)
 671     {
 672         char fn[MAX_PATH];
 673
 674         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 675         tcs->idxfd[tag] = open(fn, O_RDONLY);
 676     }
 677
 678     if (tcs->idxfd[tag] < 0)
 679     {
 680         logf("File not open!");
 681         return false;
 682     }
 683
 684     return true;
 685 }
 686
 687 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 688                      int tag, char *buf, long size)
 689 {
 690     struct tagfile_entry tfe;
 691     long seek;
 692
 693     *buf = '\0';
 694
 695     if (TAGCACHE_IS_NUMERIC(tag))
 696         return false;
 697
 698     seek = idx->tag_seek[tag];
 699     if (seek < 0)
 700     {
 701         logf("Retrieve failed");
 702         return false;
 703     }
 704
 705 #ifdef HAVE_TC_RAMCACHE
 706     if (tcs->ramsearch)
 707     {
 708         struct tagfile_entry *ep;
 709
 710 # ifdef HAVE_DIRCACHE
 711         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 712             && is_dircache_intact())
 713         {
 714             dircache_copy_path((struct dircache_entry *)seek,
 715                                buf, size);
 716             return true;
 717         }
 718         else
 719 # endif
 720         if (tag != tag_filename)
 721         {
 722             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 723             strlcpy(buf, ep->tag_data, size);
 724
 725             return true;
 726         }
 727     }
 728 #endif
 729
 730     if (!open_files(tcs, tag))
 731         return false;
 732
 733     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 734     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 735         != sizeof(struct tagfile_entry))
 736     {
 737         logf("read error #5");
 738         return false;
 739     }
 740
 741     if (tfe.tag_length >= size)
 742     {
 743         logf("too small buffer");
 744         return false;
 745     }
 746
 747     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 748         tfe.tag_length)
 749     {
 750         logf("read error #6");
 751         return false;
 752     }
 753
 754     buf[tfe.tag_length] = '\0';
 755
 756     return true;
 757 }
 758
 759 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 760
 761 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 762 {
 763 #ifndef __PCTOOL__
 764     if (! COMMAND_QUEUE_IS_EMPTY)
 765     {
 766         /* Attempt to find tag data through store-to-load forwarding in
 767            command queue */
 768         long result = -1;
 769
 770         mutex_lock(&command_queue_mutex);
 771
 772         int ridx = command_queue_widx;
 773
 774         while (ridx != command_queue_ridx)
 775         {
 776             if (--ridx < 0)
 777                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 778
 779             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 780                 && command_queue[ridx].idx_id == idx_id
 781                 && command_queue[ridx].tag == tag)
 782             {
 783                 result = command_queue[ridx].data;
 784                 break;
 785             }
 786         }
 787
 788         mutex_unlock(&command_queue_mutex);
 789
 790         if (result >= 0)
 791         {
 792             logf("read_numeric_tag: "
 793                  "Recovered tag %d value %lX from write queue",
 794                  tag, result);
 795             return result;
 796         }
 797     }
 798 #endif
 799
 800     return idx->tag_seek[tag];
 801 }
 802
 803
 804 static long check_virtual_tags(int tag, int idx_id,
 805                                const struct index_entry *idx)
 806 {
 807     long data = 0;
 808
 809     switch (tag)
 810     {
 811         case tag_virt_length_sec:
 812             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 813             break;
 814
 815         case tag_virt_length_min:
 816             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 817             break;
 818
 819         case tag_virt_playtime_sec:
 820             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 821             break;
 822
 823         case tag_virt_playtime_min:
 824             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 825             break;
 826
 827         case tag_virt_autoscore:
 828             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 829                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 830             {
 831                 data = 0;
 832             }
 833             else
 834             {
 835                 /* A straight calculus gives:
 836                      autoscore = 100 * playtime / length / playcout (1)
 837                    Now, consider the euclidian division of playtime by length:
 838                      playtime = alpha * length + beta
 839                    With:
 840                      0 <= beta < length
 841                    Now, (1) becomes:
 842                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 843                    Both terms should be small enough to avoid any overflow
 844                 */
 845                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 846                               / read_numeric_tag(tag_length, idx_id, idx))
 847                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 848                                  % read_numeric_tag(tag_length, idx_id, idx)))
 849                          / read_numeric_tag(tag_length, idx_id, idx);
 850                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 851             }
 852             break;
 853
 854         /* How many commits before the file has been added to the DB. */
 855         case tag_virt_entryage:
 856             data = current_tcmh.commitid
 857                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 858             break;
 859
 860         default:
 861             data = read_numeric_tag(tag, idx_id, idx);
 862     }
 863
 864     return data;
 865 }
 866
 867 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 868 {
 869     struct index_entry idx;
 870
 871     if (!tc_stat.ready)
 872         return false;
 873
 874     if (!TAGCACHE_IS_NUMERIC(tag))
 875         return -1;
 876
 877     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 878         return -2;
 879
 880     return check_virtual_tags(tag, tcs->idx_id, &idx);
 881 }
 882
 883 inline static bool str_ends_with(const char *str1, const char *str2)
 884 {
 885     int str_len = strlen(str1);
 886     int clause_len = strlen(str2);
 887
 888     if (clause_len > str_len)
 889         return false;
 890
 891     return !strcasecmp(&str1[str_len - clause_len], str2);
 892 }
 893
 894 inline static bool str_oneof(const char *str, const char *list)
 895 {
 896     const char *sep;
 897     int l, len = strlen(str);
 898
 899     while (*list)
 900     {
 901         sep = strchr(list, '|');
 902         l = sep ? (long)sep - (long)list : (int)strlen(list);
 903         if ((l==len) && !strncasecmp(str, list, len))
 904             return true;
 905         list += sep ? l + 1 : l;
 906     }
 907
 908     return false;
 909 }
 910
 911 static bool check_against_clause(long numeric, const char *str,
 912                                  const struct tagcache_search_clause *clause)
 913 {
 914     if (clause->numeric)
 915     {
 916         switch (clause->type)
 917         {
 918             case clause_is:
 919                 return numeric == clause->numeric_data;
 920             case clause_is_not:
 921                 return numeric != clause->numeric_data;
 922             case clause_gt:
 923                 return numeric > clause->numeric_data;
 924             case clause_gteq:
 925                 return numeric >= clause->numeric_data;
 926             case clause_lt:
 927                 return numeric < clause->numeric_data;
 928             case clause_lteq:
 929                 return numeric <= clause->numeric_data;
 930             default:
 931                 logf("Incorrect numeric tag: %d", clause->type);
 932         }
 933     }
 934     else
 935     {
 936         switch (clause->type)
 937         {
 938             case clause_is:
 939                 return !strcasecmp(clause->str, str);
 940             case clause_is_not:
 941                 return strcasecmp(clause->str, str);
 942             case clause_gt:
 943                 return 0>strcasecmp(clause->str, str);
 944             case clause_gteq:
 945                 return 0>=strcasecmp(clause->str, str);
 946             case clause_lt:
 947                 return 0<strcasecmp(clause->str, str);
 948             case clause_lteq:
 949                 return 0<=strcasecmp(clause->str, str);
 950             case clause_contains:
 951                 return (strcasestr(str, clause->str) != NULL);
 952             case clause_not_contains:
 953                 return (strcasestr(str, clause->str) == NULL);
 954             case clause_begins_with:
 955                 return (strcasestr(str, clause->str) == str);
 956             case clause_not_begins_with:
 957                 return (strcasestr(str, clause->str) != str);
 958             case clause_ends_with:
 959                 return str_ends_with(str, clause->str);
 960             case clause_not_ends_with:
 961                 return !str_ends_with(str, clause->str);
 962             case clause_oneof:
 963                 return str_oneof(str, clause->str);
 964
 965             default:
 966                 logf("Incorrect tag: %d", clause->type);
 967         }
 968     }
 969
 970     return false;
 971 }
 972
 973 static bool check_clauses(struct tagcache_search *tcs,
 974                           struct index_entry *idx,
 975                           struct tagcache_search_clause **clause, int count)
 976 {
 977     int i;
 978
 979 #ifdef HAVE_TC_RAMCACHE
 980     if (tcs->ramsearch)
 981     {
 982         /* Go through all conditional clauses. */
 983         for (i = 0; i < count; i++)
 984         {
 985             struct tagfile_entry *tfe;
 986             int seek;
 987             char buf[256];
 988             char *str = NULL;
 989
 990             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
 991
 992             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 993             {
 994                 if (clause[i]->tag == tag_filename)
 995                 {
 996                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 997                     str = buf;
 998                 }
 999                 else
1000                 {
1001                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1002                     str = tfe->tag_data;
1003                 }
1004             }
1005
1006             if (!check_against_clause(seek, str, clause[i]))
1007                 return false;
1008         }
1009     }
1010     else
1011 #endif
1012     {
1013         /* Check for conditions. */
1014         for (i = 0; i < count; i++)
1015         {
1016             struct tagfile_entry tfe;
1017             int seek;
1018             char str[256];
1019
1020             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1021
1022             memset(str, 0, sizeof str);
1023             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1024             {
1025                 int fd = tcs->idxfd[clause[i]->tag];
1026                 lseek(fd, seek, SEEK_SET);
1027                 ecread_tagfile_entry(fd, &tfe);
1028                 if (tfe.tag_length >= (int)sizeof(str))
1029                 {
1030                     logf("Too long tag read!");
1031                     break ;
1032                 }
1033
1034                 read(fd, str, tfe.tag_length);
1035
1036                 /* Check if entry has been deleted. */
1037                 if (str[0] == '\0')
1038                     break;
1039             }
1040
1041             if (!check_against_clause(seek, str, clause[i]))
1042                 return false;
1043         }
1044     }
1045
1046     return true;
1047 }
1048
1049 bool tagcache_check_clauses(struct tagcache_search *tcs,
1050                             struct tagcache_search_clause **clause, int count)
1051 {
1052     struct index_entry idx;
1053
1054     if (count == 0)
1055         return true;
1056
1057     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1058         return false;
1059
1060     return check_clauses(tcs, &idx, clause, count);
1061 }
1062
1063 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1064 {
1065     int i;
1066
1067     /* If uniq buffer is not defined we must return true for search to work. */
1068     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1069         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1070     {
1071         return true;
1072     }
1073
1074     for (i = 0; i < tcs->unique_list_count; i++)
1075     {
1076         /* Return false if entry is found. */
1077         if (tcs->unique_list[i] == id)
1078             return false;
1079     }
1080
1081     if (tcs->unique_list_count < tcs->unique_list_capacity)
1082     {
1083         tcs->unique_list[i] = id;
1084         tcs->unique_list_count++;
1085     }
1086
1087     return true;
1088 }
1089
1090 static bool build_lookup_list(struct tagcache_search *tcs)
1091 {
1092     struct index_entry entry;
1093     int i, j;
1094
1095     tcs->seek_list_count = 0;
1096
1097 #ifdef HAVE_TC_RAMCACHE
1098     if (tcs->ramsearch
1099 # ifdef HAVE_DIRCACHE
1100         && (tcs->type != tag_filename || is_dircache_intact())
1101 # endif
1102         )
1103     {
1104         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1105         {
1106             struct tagcache_seeklist_entry *seeklist;
1107             struct index_entry *idx = &hdr->indices[i];
1108             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1109                 break ;
1110
1111             /* Skip deleted files. */
1112             if (idx->flag & FLAG_DELETED)
1113                 continue;
1114
1115             /* Go through all filters.. */
1116             for (j = 0; j < tcs->filter_count; j++)
1117             {
1118                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1119                 {
1120                     break ;
1121                 }
1122             }
1123
1124             if (j < tcs->filter_count)
1125                 continue ;
1126
1127             /* Check for conditions. */
1128             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1129                 continue;
1130
1131             /* Add to the seek list if not already in uniq buffer. */
1132             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1133                 continue;
1134
1135             /* Lets add it. */
1136             seeklist = &tcs->seeklist[tcs->seek_list_count];
1137             seeklist->seek = idx->tag_seek[tcs->type];
1138             seeklist->flag = idx->flag;
1139             seeklist->idx_id = i;
1140             tcs->seek_list_count++;
1141         }
1142
1143         tcs->seek_pos = i;
1144
1145         return tcs->seek_list_count > 0;
1146     }
1147 #endif
1148
1149     if (tcs->masterfd < 0)
1150     {
1151         struct master_header tcmh;
1152         tcs->masterfd = open_master_fd(&tcmh, false);
1153     }
1154
1155     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1156             sizeof(struct master_header), SEEK_SET);
1157
1158     while (ecread_index_entry(tcs->masterfd, &entry)
1159            == sizeof(struct index_entry))
1160     {
1161         struct tagcache_seeklist_entry *seeklist;
1162
1163         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1164             break ;
1165
1166         i = tcs->seek_pos;
1167         tcs->seek_pos++;
1168
1169         /* Check if entry has been deleted. */
1170         if (entry.flag & FLAG_DELETED)
1171             continue;
1172
1173         /* Go through all filters.. */
1174         for (j = 0; j < tcs->filter_count; j++)
1175         {
1176             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1177                 break ;
1178         }
1179
1180         if (j < tcs->filter_count)
1181             continue ;
1182
1183         /* Check for conditions. */
1184         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1185             continue;
1186
1187         /* Add to the seek list if not already in uniq buffer. */
1188         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1189             continue;
1190
1191         /* Lets add it. */
1192         seeklist = &tcs->seeklist[tcs->seek_list_count];
1193         seeklist->seek = entry.tag_seek[tcs->type];
1194         seeklist->flag = entry.flag;
1195         seeklist->idx_id = i;
1196         tcs->seek_list_count++;
1197
1198         yield();
1199     }
1200
1201     return tcs->seek_list_count > 0;
1202 }
1203
1204
1205 static void remove_files(void)
1206 {
1207     int i;
1208     char buf[MAX_PATH];
1209
1210     tc_stat.ready = false;
1211     tc_stat.ramcache = false;
1212     tc_stat.econ = false;
1213     remove(TAGCACHE_FILE_MASTER);
1214     for (i = 0; i < TAG_COUNT; i++)
1215     {
1216         if (TAGCACHE_IS_NUMERIC(i))
1217             continue;
1218
1219         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1220         remove(buf);
1221     }
1222 }
1223
1224
1225 static bool check_all_headers(void)
1226 {
1227     struct master_header myhdr;
1228     struct tagcache_header tch;
1229     int tag;
1230     int fd;
1231
1232     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1233         return false;
1234
1235     close(fd);
1236     if (myhdr.dirty)
1237     {
1238         logf("tagcache is dirty!");
1239         return false;
1240     }
1241
1242     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1243
1244     for (tag = 0; tag < TAG_COUNT; tag++)
1245     {
1246         if (TAGCACHE_IS_NUMERIC(tag))
1247             continue;
1248
1249         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1250             return false;
1251
1252         close(fd);
1253     }
1254
1255     return true;
1256 }
1257
1258 bool tagcache_is_busy(void)
1259 {
1260     return read_lock || write_lock;
1261 }
1262
1263 bool tagcache_search(struct tagcache_search *tcs, int tag)
1264 {
1265     struct tagcache_header tag_hdr;
1266     struct master_header   master_hdr;
1267     int i;
1268
1269     while (read_lock)
1270         sleep(1);
1271
1272     memset(tcs, 0, sizeof(struct tagcache_search));
1273     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1274         return false;
1275
1276     tcs->position = sizeof(struct tagcache_header);
1277     tcs->type = tag;
1278     tcs->seek_pos = 0;
1279     tcs->list_position = 0;
1280     tcs->seek_list_count = 0;
1281     tcs->filter_count = 0;
1282     tcs->masterfd = -1;
1283
1284     for (i = 0; i < TAG_COUNT; i++)
1285         tcs->idxfd[i] = -1;
1286
1287 #ifndef HAVE_TC_RAMCACHE
1288     tcs->ramsearch = false;
1289 #else
1290     tcs->ramsearch = tc_stat.ramcache;
1291     if (tcs->ramsearch)
1292     {
1293         tcs->entry_count = hdr->entry_count[tcs->type];
1294     }
1295     else
1296 #endif
1297     {
1298         /* Always open as R/W so we can pass tcs to functions that modify data also
1299          * without failing. */
1300         tcs->masterfd = open_master_fd(&master_hdr, true);
1301         if (tcs->masterfd < 0)
1302             return false;
1303
1304         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1305         {
1306             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1307             if (tcs->idxfd[tcs->type] < 0)
1308                 return false;
1309
1310             tcs->entry_count = tag_hdr.entry_count;
1311         }
1312         else
1313         {
1314             tcs->entry_count = master_hdr.tch.entry_count;
1315         }
1316     }
1317
1318     tcs->valid = true;
1319     tcs->initialized = true;
1320     write_lock++;
1321
1322     return true;
1323 }
1324
1325 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1326                                  void *buffer, long length)
1327 {
1328     tcs->unique_list = (unsigned long *)buffer;
1329     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1330     tcs->unique_list_count = 0;
1331 }
1332
1333 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1334                                 int tag, int seek)
1335 {
1336     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1337         return false;
1338
1339     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1340         return false;
1341
1342     tcs->filter_tag[tcs->filter_count] = tag;
1343     tcs->filter_seek[tcs->filter_count] = seek;
1344     tcs->filter_count++;
1345
1346     return true;
1347 }
1348
1349 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1350                                 struct tagcache_search_clause *clause)
1351 {
1352     int i;
1353
1354     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1355     {
1356         logf("Too many clauses");
1357         return false;
1358     }
1359
1360     /* Check if there is already a similar filter in present (filters are
1361      * much faster than clauses).
1362      */
1363     for (i = 0; i < tcs->filter_count; i++)
1364     {
1365         if (tcs->filter_tag[i] == clause->tag)
1366             return true;
1367     }
1368
1369     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1370     {
1371         char buf[MAX_PATH];
1372
1373         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1374         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1375     }
1376
1377     tcs->clause[tcs->clause_count] = clause;
1378     tcs->clause_count++;
1379
1380     return true;
1381 }
1382
1383 static bool get_next(struct tagcache_search *tcs)
1384 {
1385     static char buf[TAG_MAXLEN+32];
1386     struct tagfile_entry entry;
1387     long flag = 0;
1388
1389     if (!tcs->valid || !tc_stat.ready)
1390         return false;
1391
1392     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1393 #ifdef HAVE_TC_RAMCACHE
1394         && !tcs->ramsearch
1395 #endif
1396         )
1397         return false;
1398
1399     /* Relative fetch. */
1400     if (tcs->filter_count > 0 || tcs->clause_count > 0
1401         || TAGCACHE_IS_NUMERIC(tcs->type)
1402 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1403         /* We need to retrieve flag status for dircache. */
1404         || (tcs->ramsearch && tcs->type == tag_filename)
1405 #endif
1406         )
1407     {
1408         struct tagcache_seeklist_entry *seeklist;
1409
1410         /* Check for end of list. */
1411         if (tcs->list_position == tcs->seek_list_count)
1412         {
1413             tcs->list_position = 0;
1414
1415             /* Try to fetch more. */
1416             if (!build_lookup_list(tcs))
1417             {
1418                 tcs->valid = false;
1419                 return false;
1420             }
1421         }
1422
1423         seeklist = &tcs->seeklist[tcs->list_position];
1424         flag = seeklist->flag;
1425         tcs->position = seeklist->seek;
1426         tcs->idx_id = seeklist->idx_id;
1427         tcs->list_position++;
1428     }
1429     else
1430     {
1431         if (tcs->entry_count == 0)
1432         {
1433             tcs->valid = false;
1434             return false;
1435         }
1436
1437         tcs->entry_count--;
1438     }
1439
1440     tcs->result_seek = tcs->position;
1441
1442     if (TAGCACHE_IS_NUMERIC(tcs->type))
1443     {
1444         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1445         tcs->result = buf;
1446         tcs->result_len = strlen(buf) + 1;
1447         return true;
1448     }
1449
1450     /* Direct fetch. */
1451 #ifdef HAVE_TC_RAMCACHE
1452     if (tcs->ramsearch)
1453     {
1454
1455 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1456         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1457             && is_dircache_intact())
1458         {
1459             dircache_copy_path((struct dircache_entry *)tcs->position,
1460                                buf, sizeof buf);
1461             tcs->result = buf;
1462             tcs->result_len = strlen(buf) + 1;
1463             tcs->ramresult = false;
1464
1465             return true;
1466         }
1467         else
1468 #endif
1469         if (tcs->type != tag_filename)
1470         {
1471             struct tagfile_entry *ep;
1472
1473             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1474             tcs->result = ep->tag_data;
1475             tcs->result_len = strlen(tcs->result) + 1;
1476             tcs->idx_id = ep->idx_id;
1477             tcs->ramresult = true;
1478
1479             /* Increase position for the next run. This may get overwritten. */
1480             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1481
1482             return true;
1483         }
1484     }
1485 #endif
1486
1487     if (!open_files(tcs, tcs->type))
1488     {
1489         tcs->valid = false;
1490         return false;
1491     }
1492
1493     /* Seek stream to the correct position and continue to direct fetch. */
1494     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1495
1496     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1497     {
1498         logf("read error #5");
1499         tcs->valid = false;
1500         return false;
1501     }
1502
1503     if (entry.tag_length > (long)sizeof(buf))
1504     {
1505         tcs->valid = false;
1506         logf("too long tag #2");
1507         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1508         return false;
1509     }
1510
1511     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1512     {
1513         tcs->valid = false;
1514         logf("read error #4");
1515         return false;
1516     }
1517
1518     /**
1519      Update the position for the next read (this may be overridden
1520      if filters or clauses are being used).
1521      */
1522     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1523     tcs->result = buf;
1524     tcs->result_len = strlen(tcs->result) + 1;
1525     tcs->idx_id = entry.idx_id;
1526     tcs->ramresult = false;
1527
1528     return true;
1529 }
1530
1531 bool tagcache_get_next(struct tagcache_search *tcs)
1532 {
1533     while (get_next(tcs))
1534     {
1535         if (tcs->result_len > 1)
1536             return true;
1537     }
1538
1539     return false;
1540 }
1541
1542 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1543                        int tag, char *buf, long size)
1544 {
1545     struct index_entry idx;
1546
1547     *buf = '\0';
1548     if (!get_index(tcs->masterfd, idxid, &idx, true))
1549         return false;
1550
1551     return retrieve(tcs, &idx, tag, buf, size);
1552 }
1553
1554 static bool update_master_header(void)
1555 {
1556     struct master_header myhdr;
1557     int fd;
1558
1559     if (!tc_stat.ready)
1560         return false;
1561
1562     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1563         return false;
1564
1565     myhdr.serial = current_tcmh.serial;
1566     myhdr.commitid = current_tcmh.commitid;
1567     myhdr.dirty = current_tcmh.dirty;
1568
1569     /* Write it back */
1570     lseek(fd, 0, SEEK_SET);
1571     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1572     close(fd);
1573
1574 #ifdef HAVE_TC_RAMCACHE
1575     if (hdr)
1576     {
1577         hdr->h.serial = current_tcmh.serial;
1578         hdr->h.commitid = current_tcmh.commitid;
1579         hdr->h.dirty = current_tcmh.dirty;
1580     }
1581 #endif
1582
1583     return true;
1584 }
1585
1586 #if 0
1587
1588 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1589 {
1590     struct tagentry *entry;
1591
1592     if (tcs->type != tag_title)
1593         return ;
1594
1595     /* We will need reserve buffer for this. */
1596     if (tcs->ramcache)
1597     {
1598         struct tagfile_entry *ep;
1599
1600         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1601         tcs->seek_list[tcs->seek_list_count];
1602     }
1603
1604     entry = find_entry_ram();
1605
1606 }
1607 #endif
1608
1609 void tagcache_search_finish(struct tagcache_search *tcs)
1610 {
1611     int i;
1612
1613     if (!tcs->initialized)
1614         return;
1615
1616     if (tcs->masterfd >= 0)
1617     {
1618         close(tcs->masterfd);
1619         tcs->masterfd = -1;
1620     }
1621
1622     for (i = 0; i < TAG_COUNT; i++)
1623     {
1624         if (tcs->idxfd[i] >= 0)
1625         {
1626             close(tcs->idxfd[i]);
1627             tcs->idxfd[i] = -1;
1628         }
1629     }
1630
1631     tcs->ramsearch = false;
1632     tcs->valid = false;
1633     tcs->initialized = 0;
1634     if (write_lock > 0)
1635         write_lock--;
1636 }
1637
1638 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1639 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1640 {
1641     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1642 }
1643
1644 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1645 {
1646     return check_virtual_tags(tag, idx_id, entry);
1647 }
1648
1649 static char* get_tag_string(const struct index_entry *entry, int tag)
1650 {
1651     char* s = get_tag(entry, tag)->tag_data;
1652     return strcmp(s, UNTAGGED) ? s : NULL;
1653 }
1654
1655 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1656 {
1657     struct index_entry *entry;
1658     int idx_id;
1659
1660     if (!tc_stat.ready || !tc_stat.ramcache)
1661         return false;
1662
1663     /* Find the corresponding entry in tagcache. */
1664     idx_id = find_entry_ram(filename, NULL);
1665     if (idx_id < 0)
1666         return false;
1667
1668     entry = &hdr->indices[idx_id];
1669
1670     memset(id3, 0, sizeof(struct mp3entry));
1671
1672     id3->title        = get_tag_string(entry, tag_title);
1673     id3->artist       = get_tag_string(entry, tag_artist);
1674     id3->album        = get_tag_string(entry, tag_album);
1675     id3->genre_string = get_tag_string(entry, tag_genre);
1676     id3->composer     = get_tag_string(entry, tag_composer);
1677     id3->comment      = get_tag_string(entry, tag_comment);
1678     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1679     id3->grouping     = get_tag_string(entry, tag_grouping);
1680
1681     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1682     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1683     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1684     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1685     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1686     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1687
1688     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1689     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1690     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1691     if (id3->bitrate == 0)
1692         id3->bitrate = 1;
1693
1694     return true;
1695 }
1696 #endif
1697
1698 static inline void write_item(const char *item)
1699 {
1700     int len = strlen(item) + 1;
1701
1702     data_size += len;
1703     write(cachefd, item, len);
1704 }
1705
1706 static int check_if_empty(char **tag)
1707 {
1708     int length;
1709
1710     if (*tag == NULL || **tag == '\0')
1711     {
1712         *tag = UNTAGGED;
1713         return sizeof(UNTAGGED); /* Tag length */
1714     }
1715
1716     length = strlen(*tag);
1717     if (length > TAG_MAXLEN)
1718     {
1719         logf("over length tag: %s", *tag);
1720         length = TAG_MAXLEN;
1721         (*tag)[length] = '\0';
1722     }
1723
1724     return length + 1;
1725 }
1726
1727 #define ADD_TAG(entry,tag,data) \
1728     /* Adding tag */ \
1729     entry.tag_offset[tag] = offset; \
1730     entry.tag_length[tag] = check_if_empty(data); \
1731     offset += entry.tag_length[tag]
1732 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1733  * idea, as it uses lots of stack and is called from a recursive function
1734  * (check_dir).
1735  */
1736 static void __attribute__ ((noinline)) add_tagcache(char *path,
1737                                                     unsigned long mtime
1738 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1739                                                     ,const struct dircache_entry *dc
1740 #endif
1741                                                    )
1742 {
1743     struct mp3entry id3;
1744     struct temp_file_entry entry;
1745     bool ret;
1746     int fd;
1747     int idx_id = -1;
1748     char tracknumfix[3];
1749     int offset = 0;
1750     int path_length = strlen(path);
1751     bool has_albumartist;
1752     bool has_grouping;
1753
1754 #ifdef SIMULATOR
1755     /* Crude logging for the sim - to aid in debugging */
1756     int logfd = open(ROCKBOX_DIR "/database.log",
1757                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1758     if (logfd >= 0) {
1759         write(logfd, path, strlen(path));
1760         write(logfd, "\n", 1);
1761         close(logfd);
1762     }
1763 #endif
1764
1765     if (cachefd < 0)
1766         return ;
1767
1768     /* Check for overlength file path. */
1769     if (path_length > TAG_MAXLEN)
1770     {
1771         /* Path can't be shortened. */
1772         logf("Too long path: %s", path);
1773         return ;
1774     }
1775
1776     /* Check if the file is supported. */
1777     if (probe_file_format(path) == AFMT_UNKNOWN)
1778         return ;
1779
1780     /* Check if the file is already cached. */
1781 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1782     if (tc_stat.ramcache && is_dircache_intact())
1783     {
1784         idx_id = find_entry_ram(path, dc);
1785     }
1786 #endif
1787
1788     /* Be sure the entry doesn't exist. */
1789     if (filenametag_fd >= 0 && idx_id < 0)
1790         idx_id = find_entry_disk(path, false);
1791
1792     /* Check if file has been modified. */
1793     if (idx_id >= 0)
1794     {
1795         struct index_entry idx;
1796
1797         /* TODO: Mark that the index exists (for fast reverse scan) */
1798         //found_idx[idx_id/8] |= idx_id%8;
1799
1800         if (!get_index(-1, idx_id, &idx, true))
1801         {
1802             logf("failed to retrieve index entry");
1803             return ;
1804         }
1805
1806         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1807         {
1808             /* No changes to file. */
1809             return ;
1810         }
1811
1812         /* Metadata might have been changed. Delete the entry. */
1813         logf("Re-adding: %s", path);
1814         if (!delete_entry(idx_id))
1815         {
1816             logf("delete_entry failed: %d", idx_id);
1817             return ;
1818         }
1819     }
1820
1821     fd = open(path, O_RDONLY);
1822     if (fd < 0)
1823     {
1824         logf("open fail: %s", path);
1825         return ;
1826     }
1827
1828     memset(&id3, 0, sizeof(struct mp3entry));
1829     memset(&entry, 0, sizeof(struct temp_file_entry));
1830     memset(&tracknumfix, 0, sizeof(tracknumfix));
1831     ret = get_metadata(&id3, fd, path);
1832     close(fd);
1833
1834     if (!ret)
1835         return ;
1836
1837     logf("-> %s", path);
1838
1839     /* Generate track number if missing. */
1840     if (id3.tracknum <= 0)
1841     {
1842         const char *p = strrchr(path, '.');
1843
1844         if (p == NULL)
1845             p = &path[strlen(path)-1];
1846
1847         while (*p != '/')
1848         {
1849             if (isdigit(*p) && isdigit(*(p-1)))
1850             {
1851                 tracknumfix[1] = *p--;
1852                 tracknumfix[0] = *p;
1853                 break;
1854             }
1855             p--;
1856         }
1857
1858         if (tracknumfix[0] != '\0')
1859         {
1860             id3.tracknum = atoi(tracknumfix);
1861             /* Set a flag to indicate track number has been generated. */
1862             entry.flag |= FLAG_TRKNUMGEN;
1863         }
1864         else
1865         {
1866             /* Unable to generate track number. */
1867             id3.tracknum = -1;
1868         }
1869     }
1870
1871     /* Numeric tags */
1872     entry.tag_offset[tag_year] = id3.year;
1873     entry.tag_offset[tag_discnumber] = id3.discnum;
1874     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1875     entry.tag_offset[tag_length] = id3.length;
1876     entry.tag_offset[tag_bitrate] = id3.bitrate;
1877     entry.tag_offset[tag_mtime] = mtime;
1878
1879     /* String tags. */
1880     has_albumartist = id3.albumartist != NULL
1881         && strlen(id3.albumartist) > 0;
1882     has_grouping = id3.grouping != NULL
1883         && strlen(id3.grouping) > 0;
1884
1885     ADD_TAG(entry, tag_filename, &path);
1886     ADD_TAG(entry, tag_title, &id3.title);
1887     ADD_TAG(entry, tag_artist, &id3.artist);
1888     ADD_TAG(entry, tag_album, &id3.album);
1889     ADD_TAG(entry, tag_genre, &id3.genre_string);
1890     ADD_TAG(entry, tag_composer, &id3.composer);
1891     ADD_TAG(entry, tag_comment, &id3.comment);
1892     if (has_albumartist)
1893     {
1894         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1895     }
1896     else
1897     {
1898         ADD_TAG(entry, tag_albumartist, &id3.artist);
1899     }
1900     if (has_grouping)
1901     {
1902         ADD_TAG(entry, tag_grouping, &id3.grouping);
1903     }
1904     else
1905     {
1906         ADD_TAG(entry, tag_grouping, &id3.title);
1907     }
1908     entry.data_length = offset;
1909
1910     /* Write the header */
1911     write(cachefd, &entry, sizeof(struct temp_file_entry));
1912
1913     /* And tags also... Correct order is critical */
1914     write_item(path);
1915     write_item(id3.title);
1916     write_item(id3.artist);
1917     write_item(id3.album);
1918     write_item(id3.genre_string);
1919     write_item(id3.composer);
1920     write_item(id3.comment);
1921     if (has_albumartist)
1922     {
1923         write_item(id3.albumartist);
1924     }
1925     else
1926     {
1927         write_item(id3.artist);
1928     }
1929     if (has_grouping)
1930     {
1931         write_item(id3.grouping);
1932     }
1933     else
1934     {
1935         write_item(id3.title);
1936     }
1937     total_entry_count++;
1938 }
1939
1940 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1941 {
1942     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1943     int len = strlen(str)+1;
1944     int i;
1945     unsigned crc32;
1946     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1947     char buf[TAG_MAXLEN+32];
1948
1949     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1950         buf[i] = tolower(str[i]);
1951     buf[i] = '\0';
1952
1953     crc32 = crc_32(buf, i, 0xffffffff);
1954
1955     if (unique)
1956     {
1957         /* Check if the crc does not exist -> entry does not exist for sure. */
1958         for (i = 0; i < tempbufidx; i++)
1959         {
1960             if (crcbuf[-i] != crc32)
1961                 continue;
1962
1963             if (!strcasecmp(str, index[i].str))
1964             {
1965                 if (id < 0 || id >= lookup_buffer_depth)
1966                 {
1967                     logf("lookup buf overf.: %d", id);
1968                     return false;
1969                 }
1970
1971                 lookup[id] = &index[i];
1972                 return true;
1973             }
1974         }
1975     }
1976
1977     /* Insert to CRC buffer. */
1978     crcbuf[-tempbufidx] = crc32;
1979     tempbuf_left -= 4;
1980
1981     /* Insert it to the buffer. */
1982     tempbuf_left -= len;
1983     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1984         return false;
1985
1986     if (id >= lookup_buffer_depth)
1987     {
1988         logf("lookup buf overf. #2: %d", id);
1989         return false;
1990     }
1991
1992     if (id >= 0)
1993     {
1994         lookup[id] = &index[tempbufidx];
1995         index[tempbufidx].idlist.id = id;
1996     }
1997     else
1998         index[tempbufidx].idlist.id = -1;
1999
2000     index[tempbufidx].idlist.next = NULL;
2001     index[tempbufidx].idx_id = idx_id;
2002     index[tempbufidx].seek = -1;
2003     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2004     memcpy(index[tempbufidx].str, str, len);
2005     tempbuf_pos += len;
2006     tempbufidx++;
2007
2008     return true;
2009 }
2010
2011 static int compare(const void *p1, const void *p2)
2012 {
2013     do_timed_yield();
2014
2015     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2016     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2017
2018     if (strcmp(e1->str, UNTAGGED) == 0)
2019     {
2020         if (strcmp(e2->str, UNTAGGED) == 0)
2021             return 0;
2022         return -1;
2023     }
2024     else if (strcmp(e2->str, UNTAGGED) == 0)
2025         return 1;
2026
2027     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2028 }
2029
2030 static int tempbuf_sort(int fd)
2031 {
2032     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2033     struct tagfile_entry fe;
2034     int i;
2035     int length;
2036
2037     /* Generate reverse lookup entries. */
2038     for (i = 0; i < lookup_buffer_depth; i++)
2039     {
2040         struct tempbuf_id_list *idlist;
2041
2042         if (!lookup[i])
2043             continue;
2044
2045         if (lookup[i]->idlist.id == i)
2046             continue;
2047
2048         idlist = &lookup[i]->idlist;
2049         while (idlist->next != NULL)
2050             idlist = idlist->next;
2051
2052         tempbuf_left -= sizeof(struct tempbuf_id_list);
2053         if (tempbuf_left - 4 < 0)
2054             return -1;
2055
2056         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2057         if (tempbuf_pos & 0x03)
2058         {
2059             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2060             tempbuf_left -= 3;
2061             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2062         }
2063         tempbuf_pos += sizeof(struct tempbuf_id_list);
2064
2065         idlist = idlist->next;
2066         idlist->id = i;
2067         idlist->next = NULL;
2068
2069         do_timed_yield();
2070     }
2071
2072     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2073     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2074
2075     for (i = 0; i < tempbufidx; i++)
2076     {
2077         struct tempbuf_id_list *idlist = &index[i].idlist;
2078
2079         /* Fix the lookup list. */
2080         while (idlist != NULL)
2081         {
2082             if (idlist->id >= 0)
2083                 lookup[idlist->id] = &index[i];
2084             idlist = idlist->next;
2085         }
2086
2087         index[i].seek = lseek(fd, 0, SEEK_CUR);
2088         length = strlen(index[i].str) + 1;
2089         fe.tag_length = length;
2090         fe.idx_id = index[i].idx_id;
2091
2092         /* Check the chunk alignment. */
2093         if ((fe.tag_length + sizeof(struct tagfile_entry))
2094             % TAGFILE_ENTRY_CHUNK_LENGTH)
2095         {
2096             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2097                 ((fe.tag_length + sizeof(struct tagfile_entry))
2098                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2099         }
2100
2101 #ifdef TAGCACHE_STRICT_ALIGN
2102         /* Make sure the entry is long aligned. */
2103         if (index[i].seek & 0x03)
2104         {
2105             logf("tempbuf_sort: alignment error!");
2106             return -3;
2107         }
2108 #endif
2109
2110         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2111             sizeof(struct tagfile_entry))
2112         {
2113             logf("tempbuf_sort: write error #1");
2114             return -1;
2115         }
2116
2117         if (write(fd, index[i].str, length) != length)
2118         {
2119             logf("tempbuf_sort: write error #2");
2120             return -2;
2121         }
2122
2123         /* Write some padding. */
2124         if (fe.tag_length - length > 0)
2125             write(fd, "XXXXXXXX", fe.tag_length - length);
2126     }
2127
2128     return i;
2129 }
2130
2131 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2132 {
2133     if (id < 0 || id >= lookup_buffer_depth)
2134         return NULL;
2135
2136     return lookup[id];
2137 }
2138
2139
2140 inline static int tempbuf_find_location(int id)
2141 {
2142     struct tempbuf_searchidx *entry;
2143
2144     entry = tempbuf_locate(id);
2145     if (entry == NULL)
2146         return -1;
2147
2148     return entry->seek;
2149 }
2150
2151 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2152 {
2153     struct master_header tcmh;
2154     struct index_entry idx;
2155     int masterfd;
2156     int masterfd_pos;
2157     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2158     int max_entries;
2159     int entries_processed = 0;
2160     int i, j;
2161     char buf[TAG_MAXLEN];
2162
2163     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2164
2165     logf("Building numeric indices...");
2166     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2167
2168     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2169         return false;
2170
2171     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2172                          SEEK_CUR);
2173     if (masterfd_pos == filesize(masterfd))
2174     {
2175         logf("we can't append!");
2176         close(masterfd);
2177         return false;
2178     }
2179
2180     while (entries_processed < h->entry_count)
2181     {
2182         int count = MIN(h->entry_count - entries_processed, max_entries);
2183
2184         /* Read in as many entries as possible. */
2185         for (i = 0; i < count; i++)
2186         {
2187             struct temp_file_entry *tfe = &entrybuf[i];
2188             int datastart;
2189
2190             /* Read in numeric data. */
2191             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2192                 sizeof(struct temp_file_entry))
2193             {
2194                 logf("read fail #1");
2195                 close(masterfd);
2196                 return false;
2197             }
2198
2199             datastart = lseek(tmpfd, 0, SEEK_CUR);
2200
2201             /**
2202              * Read string data from the following tags:
2203              * - tag_filename
2204              * - tag_artist
2205              * - tag_album
2206              * - tag_title
2207              *
2208              * A crc32 hash is calculated from the read data
2209              * and stored back to the data offset field kept in memory.
2210              */
2211 #define tmpdb_read_string_tag(tag) \
2212     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2213     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2214     { \
2215         logf("read fail: buffer overflow"); \
2216         close(masterfd); \
2217         return false; \
2218     } \
2219     \
2220     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2221         tfe->tag_length[tag]) \
2222     { \
2223         logf("read fail #2"); \
2224         close(masterfd); \
2225         return false; \
2226     } \
2227     \
2228     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2229     lseek(tmpfd, datastart, SEEK_SET)
2230
2231             tmpdb_read_string_tag(tag_filename);
2232             tmpdb_read_string_tag(tag_artist);
2233             tmpdb_read_string_tag(tag_album);
2234             tmpdb_read_string_tag(tag_title);
2235
2236             /* Seek to the end of the string data. */
2237             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2238         }
2239
2240         /* Backup the master index position. */
2241         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2242         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2243
2244         /* Check if we can resurrect some deleted runtime statistics data. */
2245         for (i = 0; i < tcmh.tch.entry_count; i++)
2246         {
2247             /* Read the index entry. */
2248             if (ecread_index_entry(masterfd, &idx)
2249                 != sizeof(struct index_entry))
2250             {
2251                 logf("read fail #3");
2252                 close(masterfd);
2253                 return false;
2254             }
2255
2256             /**
2257              * Skip unless the entry is marked as being deleted
2258              * or the data has already been resurrected.
2259              */
2260             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2261                 continue;
2262
2263             /* Now try to match the entry. */
2264             /**
2265              * To succesfully match a song, the following conditions
2266              * must apply:
2267              *
2268              * For numeric fields: tag_length
2269              * - Full identical match is required
2270              *
2271              * If tag_filename matches, no further checking necessary.
2272              *
2273              * For string hashes: tag_artist, tag_album, tag_title
2274              * - Two of these must match
2275              */
2276             for (j = 0; j < count; j++)
2277             {
2278                 struct temp_file_entry *tfe = &entrybuf[j];
2279
2280                 /* Try to match numeric fields first. */
2281                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2282                     continue;
2283
2284                 /* Now it's time to do the hash matching. */
2285                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2286                 {
2287                     int match_count = 0;
2288
2289                     /* No filename match, check if we can match two other tags. */
2290 #define tmpdb_match(tag) \
2291     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2292         match_count++
2293
2294                     tmpdb_match(tag_artist);
2295                     tmpdb_match(tag_album);
2296                     tmpdb_match(tag_title);
2297
2298                     if (match_count < 2)
2299                     {
2300                         /* Still no match found, give up. */
2301                         continue;
2302                     }
2303                 }
2304
2305                 /* A match found, now copy & resurrect the statistical data. */
2306 #define tmpdb_copy_tag(tag) \
2307     tfe->tag_offset[tag] = idx.tag_seek[tag]
2308
2309                 tmpdb_copy_tag(tag_playcount);
2310                 tmpdb_copy_tag(tag_rating);
2311                 tmpdb_copy_tag(tag_playtime);
2312                 tmpdb_copy_tag(tag_lastplayed);
2313                 tmpdb_copy_tag(tag_commitid);
2314
2315                 /* Avoid processing this entry again. */
2316                 idx.flag |= FLAG_RESURRECTED;
2317
2318                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2319                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2320                 {
2321                     logf("masterfd writeback fail #1");
2322                     close(masterfd);
2323                     return false;
2324                 }
2325
2326                 logf("Entry resurrected");
2327             }
2328         }
2329
2330
2331         /* Restore the master index position. */
2332         lseek(masterfd, masterfd_pos, SEEK_SET);
2333
2334         /* Commit the data to the index. */
2335         for (i = 0; i < count; i++)
2336         {
2337             int loc = lseek(masterfd, 0, SEEK_CUR);
2338
2339             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2340             {
2341                 logf("read fail #3");
2342                 close(masterfd);
2343                 return false;
2344             }
2345
2346             for (j = 0; j < TAG_COUNT; j++)
2347             {
2348                 if (!TAGCACHE_IS_NUMERIC(j))
2349                     continue;
2350
2351                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2352             }
2353             idx.flag = entrybuf[i].flag;
2354
2355             if (idx.tag_seek[tag_commitid])
2356             {
2357                 /* Data has been resurrected. */
2358                 idx.flag |= FLAG_DIRTYNUM;
2359             }
2360             else if (tc_stat.ready && current_tcmh.commitid > 0)
2361             {
2362                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2363                 idx.flag |= FLAG_DIRTYNUM;
2364             }
2365
2366             /* Write back the updated index. */
2367             lseek(masterfd, loc, SEEK_SET);
2368             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2369             {
2370                 logf("write fail");
2371                 close(masterfd);
2372                 return false;
2373             }
2374         }
2375
2376         entries_processed += count;
2377         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2378     }
2379
2380     close(masterfd);
2381
2382     return true;
2383 }
2384
2385 /**
2386  * Return values:
2387  *     > 0   success
2388  *    == 0   temporary failure
2389  *     < 0   fatal error
2390  */
2391 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2392 {
2393     int i;
2394     struct tagcache_header tch;
2395     struct master_header   tcmh;
2396     struct index_entry idxbuf[IDX_BUF_DEPTH];
2397     int idxbuf_pos;
2398     char buf[TAG_MAXLEN+32];
2399     int fd = -1, masterfd;
2400     bool error = false;
2401     int init;
2402     int masterfd_pos;
2403
2404     logf("Building index: %d", index_type);
2405
2406     /* Check the number of entries we need to allocate ram for. */
2407     commit_entry_count = h->entry_count + 1;
2408
2409     masterfd = open_master_fd(&tcmh, false);
2410     if (masterfd >= 0)
2411     {
2412         commit_entry_count += tcmh.tch.entry_count;
2413         close(masterfd);
2414     }
2415     else
2416         remove_files(); /* Just to be sure we are clean. */
2417
2418     /* Open the index file, which contains the tag names. */
2419     fd = open_tag_fd(&tch, index_type, true);
2420     if (fd >= 0)
2421     {
2422         logf("tch.datasize=%ld", tch.datasize);
2423         lookup_buffer_depth = 1 +
2424         /* First part */ commit_entry_count +
2425         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2426     }
2427     else
2428     {
2429         lookup_buffer_depth = 1 +
2430         /* First part */ commit_entry_count +
2431         /* Second part */ 0;
2432     }
2433
2434     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2435     logf("commit_entry_count=%ld", commit_entry_count);
2436
2437     /* Allocate buffer for all index entries from both old and new
2438      * tag files. */
2439     tempbufidx = 0;
2440     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2441
2442     /* Allocate lookup buffer. The first portion of commit_entry_count
2443      * contains the new tags in the temporary file and the second
2444      * part for locating entries already in the db.
2445      *
2446      *  New tags  Old tags
2447      * +---------+---------------------------+
2448      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2449      * +---------+---------------------------+
2450      *
2451      * Old tags are inserted to a temporary buffer with position:
2452      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2453      * And new tags with index:
2454      *     tempbuf_insert(idx, ...);
2455      *
2456      * The buffer is sorted and written into tag file:
2457      *     tempbuf_sort(...);
2458      * leaving master index locations messed up.
2459      *
2460      * That is fixed using the lookup buffer for old tags:
2461      *     new_seek = tempbuf_find_location(old_seek, ...);
2462      * and for new tags:
2463      *     new_seek = tempbuf_find_location(idx);
2464      */
2465     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2466     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2467     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2468
2469     /* And calculate the remaining data space used mainly for storing
2470      * tag data (strings). */
2471     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2472     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2473     {
2474         logf("Buffer way too small!");
2475         return 0;
2476     }
2477
2478     if (fd >= 0)
2479     {
2480         /**
2481          * If tag file contains unique tags (sorted index), we will load
2482          * it entirely into memory so we can resort it later for use with
2483          * chunked browsing.
2484          */
2485         if (TAGCACHE_IS_SORTED(index_type))
2486         {
2487             logf("loading tags...");
2488             for (i = 0; i < tch.entry_count; i++)
2489             {
2490                 struct tagfile_entry entry;
2491                 int loc = lseek(fd, 0, SEEK_CUR);
2492                 bool ret;
2493
2494                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2495                 {
2496                     logf("read error #7");
2497                     close(fd);
2498                     return -2;
2499                 }
2500
2501                 if (entry.tag_length >= (int)sizeof(buf))
2502                 {
2503                     logf("too long tag #3");
2504                     close(fd);
2505                     return -2;
2506                 }
2507
2508                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2509                 {
2510                     logf("read error #8");
2511                     close(fd);
2512                     return -2;
2513                 }
2514
2515                 /* Skip deleted entries. */
2516                 if (buf[0] == '\0')
2517                     continue;
2518
2519                 /**
2520                  * Save the tag and tag id in the memory buffer. Tag id
2521                  * is saved so we can later reindex the master lookup
2522                  * table when the index gets resorted.
2523                  */
2524                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2525                                      + commit_entry_count, entry.idx_id,
2526                                      TAGCACHE_IS_UNIQUE(index_type));
2527                 if (!ret)
2528                 {
2529                     close(fd);
2530                     return -3;
2531                 }
2532                 do_timed_yield();
2533             }
2534             logf("done");
2535         }
2536         else
2537             tempbufidx = tch.entry_count;
2538     }
2539     else
2540     {
2541         /**
2542          * Creating new index file to store the tags. No need to preload
2543          * anything whether the index type is sorted or not.
2544          */
2545         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2546         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2547         if (fd < 0)
2548         {
2549             logf("%s open fail", buf);
2550             return -2;
2551         }
2552
2553         tch.magic = TAGCACHE_MAGIC;
2554         tch.entry_count = 0;
2555         tch.datasize = 0;
2556
2557         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2558             != sizeof(struct tagcache_header))
2559         {
2560             logf("header write failed");
2561             close(fd);
2562             return -2;
2563         }
2564     }
2565
2566     /* Loading the tag lookup file as "master file". */
2567     logf("Loading index file");
2568     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2569
2570     if (masterfd < 0)
2571     {
2572         logf("Creating new DB");
2573         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2574
2575         if (masterfd < 0)
2576         {
2577             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2578             close(fd);
2579             return -2;
2580         }
2581
2582         /* Write the header (write real values later). */
2583         memset(&tcmh, 0, sizeof(struct master_header));
2584         tcmh.tch = *h;
2585         tcmh.tch.entry_count = 0;
2586         tcmh.tch.datasize = 0;
2587         tcmh.dirty = true;
2588         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2589         init = true;
2590         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2591     }
2592     else
2593     {
2594         /**
2595          * Master file already exists so we need to process the current
2596          * file first.
2597          */
2598         init = false;
2599
2600         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2601             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2602         {
2603             logf("header error");
2604             close(fd);
2605             close(masterfd);
2606             return -2;
2607         }
2608
2609         /**
2610          * If we reach end of the master file, we need to expand it to
2611          * hold new tags. If the current index is not sorted, we can
2612          * simply append new data to end of the file.
2613          * However, if the index is sorted, we need to update all tag
2614          * pointers in the master file for the current index.
2615          */
2616         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2617             SEEK_CUR);
2618         if (masterfd_pos == filesize(masterfd))
2619         {
2620             logf("appending...");
2621             init = true;
2622         }
2623     }
2624
2625     /**
2626      * Load new unique tags in memory to be sorted later and added
2627      * to the master lookup file.
2628      */
2629     if (TAGCACHE_IS_SORTED(index_type))
2630     {
2631         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2632         /* h is the header of the temporary file containing new tags. */
2633         logf("inserting new tags...");
2634         for (i = 0; i < h->entry_count; i++)
2635         {
2636             struct temp_file_entry entry;
2637
2638             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2639                 sizeof(struct temp_file_entry))
2640             {
2641                 logf("read fail #3");
2642                 error = true;
2643                 goto error_exit;
2644             }
2645
2646             /* Read data. */
2647             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2648             {
2649                 logf("too long entry!");
2650                 error = true;
2651                 goto error_exit;
2652             }
2653
2654             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2655             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2656                 entry.tag_length[index_type])
2657             {
2658                 logf("read fail #4");
2659                 error = true;
2660                 goto error_exit;
2661             }
2662
2663             if (TAGCACHE_IS_UNIQUE(index_type))
2664                 error = !tempbuf_insert(buf, i, -1, true);
2665             else
2666                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2667
2668             if (error)
2669             {
2670                 logf("insert error");
2671                 goto error_exit;
2672             }
2673
2674             /* Skip to next. */
2675             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2676                     entry.tag_length[index_type], SEEK_CUR);
2677             do_timed_yield();
2678         }
2679         logf("done");
2680
2681         /* Sort the buffer data and write it to the index file. */
2682         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2683         /**
2684          * We need to truncate the index file now. There can be junk left
2685          * at the end of file (however, we _should_ always follow the
2686          * entry_count and don't crash with that).
2687          */
2688         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2689
2690         i = tempbuf_sort(fd);
2691         if (i < 0)
2692             goto error_exit;
2693         logf("sorted %d tags", i);
2694
2695         /**
2696          * Now update all indexes in the master lookup file.
2697          */
2698         logf("updating indices...");
2699         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2700         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2701         {
2702             int j;
2703             int loc = lseek(masterfd, 0, SEEK_CUR);
2704
2705             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2706
2707             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2708                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2709             {
2710                 logf("read fail #5");
2711                 error = true;
2712                 goto error_exit ;
2713             }
2714             lseek(masterfd, loc, SEEK_SET);
2715
2716             for (j = 0; j < idxbuf_pos; j++)
2717             {
2718                 if (idxbuf[j].flag & FLAG_DELETED)
2719                 {
2720                     /* We can just ignore deleted entries. */
2721                     // idxbuf[j].tag_seek[index_type] = 0;
2722                     continue;
2723                 }
2724
2725                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2726                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2727                     + commit_entry_count);
2728
2729                 if (idxbuf[j].tag_seek[index_type] < 0)
2730                 {
2731                     logf("update error: %ld/%d/%ld",
2732                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2733                     error = true;
2734                     goto error_exit;
2735                 }
2736
2737                 do_timed_yield();
2738             }
2739
2740             /* Write back the updated index. */
2741             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2742                         index_entry_ec, tc_stat.econ) !=
2743                 (int)sizeof(struct index_entry)*idxbuf_pos)
2744             {
2745                 logf("write fail");
2746                 error = true;
2747                 goto error_exit;
2748             }
2749         }
2750         logf("done");
2751     }
2752
2753     /**
2754      * Walk through the temporary file containing the new tags.
2755      */
2756     // build_normal_index(h, tmpfd, masterfd, idx);
2757     logf("updating new indices...");
2758     lseek(masterfd, masterfd_pos, SEEK_SET);
2759     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2760     lseek(fd, 0, SEEK_END);
2761     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2762     {
2763         int j;
2764
2765         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2766         if (init)
2767         {
2768             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2769         }
2770         else
2771         {
2772             int loc = lseek(masterfd, 0, SEEK_CUR);
2773
2774             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2775                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2776             {
2777                 logf("read fail #6");
2778                 error = true;
2779                 break ;
2780             }
2781             lseek(masterfd, loc, SEEK_SET);
2782         }
2783
2784         /* Read entry headers. */
2785         for (j = 0; j < idxbuf_pos; j++)
2786         {
2787             if (!TAGCACHE_IS_SORTED(index_type))
2788             {
2789                 struct temp_file_entry entry;
2790                 struct tagfile_entry fe;
2791
2792                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2793                     sizeof(struct temp_file_entry))
2794                 {
2795                     logf("read fail #7");
2796                     error = true;
2797                     break ;
2798                 }
2799
2800                 /* Read data. */
2801                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2802                 {
2803                     logf("too long entry!");
2804                     logf("length=%d", entry.tag_length[index_type]);
2805                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2806                     error = true;
2807                     break ;
2808                 }
2809
2810                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2811                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2812                     entry.tag_length[index_type])
2813                 {
2814                     logf("read fail #8");
2815                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2816                     logf("length=0x%02x", entry.tag_length[index_type]);
2817                     error = true;
2818                     break ;
2819                 }
2820
2821                 /* Write to index file. */
2822                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2823                 fe.tag_length = entry.tag_length[index_type];
2824                 fe.idx_id = tcmh.tch.entry_count + i + j;
2825                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2826                 write(fd, buf, fe.tag_length);
2827                 tempbufidx++;
2828
2829                 /* Skip to next. */
2830                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2831                       entry.tag_length[index_type], SEEK_CUR);
2832             }
2833             else
2834             {
2835                 /* Locate the correct entry from the sorted array. */
2836                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2837                 if (idxbuf[j].tag_seek[index_type] < 0)
2838                 {
2839                     logf("entry not found (%d)", j);
2840                     error = true;
2841                     break ;
2842                 }
2843             }
2844         }
2845
2846         /* Write index. */
2847         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2848                   index_entry_ec, tc_stat.econ) !=
2849             (int)sizeof(struct index_entry)*idxbuf_pos)
2850         {
2851             logf("tagcache: write fail #4");
2852             error = true;
2853             break ;
2854         }
2855
2856         do_timed_yield();
2857     }
2858     logf("done");
2859
2860     /* Finally write the header. */
2861     tch.magic = TAGCACHE_MAGIC;
2862     tch.entry_count = tempbufidx;
2863     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2864     lseek(fd, 0, SEEK_SET);
2865     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2866
2867     if (index_type != tag_filename)
2868         h->datasize += tch.datasize;
2869     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2870     error_exit:
2871
2872     close(fd);
2873     close(masterfd);
2874
2875     if (error)
2876         return -2;
2877
2878     return 1;
2879 }
2880
2881 static bool commit(void)
2882 {
2883     struct tagcache_header tch;
2884     struct master_header   tcmh;
2885     int i, len, rc;
2886     int tmpfd;
2887     int masterfd;
2888 #ifdef HAVE_DIRCACHE
2889     bool dircache_buffer_stolen = false;
2890 #endif
2891     bool local_allocation = false;
2892
2893     logf("committing tagcache");
2894
2895     while (write_lock)
2896         sleep(1);
2897
2898     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2899     if (tmpfd < 0)
2900     {
2901         logf("nothing to commit");
2902         return true;
2903     }
2904
2905
2906     /* Load the header. */
2907     len = sizeof(struct tagcache_header);
2908     rc = read(tmpfd, &tch, len);
2909
2910     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2911     {
2912         logf("incorrect tmpheader");
2913         close(tmpfd);
2914         remove(TAGCACHE_FILE_TEMP);
2915         return false;
2916     }
2917
2918     if (tch.entry_count == 0)
2919     {
2920         logf("nothing to commit");
2921         close(tmpfd);
2922         remove(TAGCACHE_FILE_TEMP);
2923         return true;
2924     }
2925
2926     /* Fully initialize existing headers (if any) before going further. */
2927     tc_stat.ready = check_all_headers();
2928
2929 #ifdef HAVE_EEPROM_SETTINGS
2930     remove(TAGCACHE_STATEFILE);
2931 #endif
2932
2933     /* At first be sure to unload the ramcache! */
2934 #ifdef HAVE_TC_RAMCACHE
2935     tc_stat.ramcache = false;
2936 #endif
2937
2938     read_lock++;
2939
2940     /* Try to steal every buffer we can :) */
2941     if (tempbuf_size == 0)
2942         local_allocation = true;
2943
2944 #ifdef HAVE_DIRCACHE
2945     if (tempbuf_size == 0)
2946     {
2947         /* Try to steal the dircache buffer. */
2948         tempbuf = dircache_steal_buffer(&tempbuf_size);
2949         tempbuf_size &= ~0x03;
2950
2951         if (tempbuf_size > 0)
2952         {
2953             dircache_buffer_stolen = true;
2954         }
2955     }
2956 #endif
2957
2958 #ifdef HAVE_TC_RAMCACHE
2959     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2960     {
2961         tempbuf = (char *)(hdr + 1);
2962         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2963         tempbuf_size &= ~0x03;
2964     }
2965 #endif
2966
2967     /* And finally fail if there are no buffers available. */
2968     if (tempbuf_size == 0)
2969     {
2970         logf("delaying commit until next boot");
2971         tc_stat.commit_delayed = true;
2972         close(tmpfd);
2973         read_lock--;
2974         return false;
2975     }
2976
2977     logf("commit %ld entries...", tch.entry_count);
2978
2979     /* Mark DB dirty so it will stay disabled if commit fails. */
2980     current_tcmh.dirty = true;
2981     update_master_header();
2982
2983     /* Now create the index files. */
2984     tc_stat.commit_step = 0;
2985     tch.datasize = 0;
2986     tc_stat.commit_delayed = false;
2987
2988     for (i = 0; i < TAG_COUNT; i++)
2989     {
2990         int ret;
2991
2992         if (TAGCACHE_IS_NUMERIC(i))
2993             continue;
2994
2995         tc_stat.commit_step++;
2996         ret = build_index(i, &tch, tmpfd);
2997         if (ret <= 0)
2998         {
2999             close(tmpfd);
3000             logf("tagcache failed init");
3001             if (ret == 0)
3002                 tc_stat.commit_delayed = true;
3003
3004             tc_stat.commit_step = 0;
3005             read_lock--;
3006             return false;
3007         }
3008     }
3009
3010     if (!build_numeric_indices(&tch, tmpfd))
3011     {
3012         logf("Failure to commit numeric indices");
3013         close(tmpfd);
3014         tc_stat.commit_step = 0;
3015         read_lock--;
3016         return false;
3017     }
3018
3019     close(tmpfd);
3020     remove(TAGCACHE_FILE_TEMP);
3021
3022     tc_stat.commit_step = 0;
3023
3024     /* Update the master index headers. */
3025     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3026     {
3027         read_lock--;
3028         return false;
3029     }
3030
3031     tcmh.tch.entry_count += tch.entry_count;
3032     tcmh.tch.datasize = sizeof(struct master_header)
3033         + sizeof(struct index_entry) * tcmh.tch.entry_count
3034         + tch.datasize;
3035     tcmh.dirty = false;
3036     tcmh.commitid++;
3037
3038     lseek(masterfd, 0, SEEK_SET);
3039     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3040     close(masterfd);
3041
3042     logf("tagcache committed");
3043     tc_stat.ready = check_all_headers();
3044     tc_stat.readyvalid = true;
3045
3046     if (local_allocation)
3047     {
3048         tempbuf = NULL;
3049         tempbuf_size = 0;
3050     }
3051
3052 #ifdef HAVE_DIRCACHE
3053     /* Rebuild the dircache, if we stole the buffer. */
3054     if (dircache_buffer_stolen)
3055         dircache_build(0);
3056 #endif
3057
3058 #ifdef HAVE_TC_RAMCACHE
3059     /* Reload tagcache. */
3060     if (tc_stat.ramcache_allocated > 0)
3061         tagcache_start_scan();
3062 #endif
3063
3064     read_lock--;
3065
3066     return true;
3067 }
3068
3069 static void allocate_tempbuf(void)
3070 {
3071     /* Yeah, malloc would be really nice now :) */
3072 #ifdef __PCTOOL__
3073     tempbuf_size = 32*1024*1024;
3074     tempbuf = malloc(tempbuf_size);
3075 #else
3076     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3077     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3078     audiobuf += tempbuf_size;
3079 #endif
3080 }
3081
3082 static void free_tempbuf(void)
3083 {
3084     if (tempbuf_size == 0)
3085         return ;
3086
3087 #ifdef __PCTOOL__
3088     free(tempbuf);
3089 #else
3090     audiobuf -= tempbuf_size;
3091 #endif
3092     tempbuf = NULL;
3093     tempbuf_size = 0;
3094 }
3095
3096 #ifndef __PCTOOL__
3097
3098 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3099 {
3100     struct index_entry idx;
3101
3102     if (!tc_stat.ready)
3103         return false;
3104
3105     if (!TAGCACHE_IS_NUMERIC(tag))
3106         return false;
3107
3108     if (!get_index(masterfd, idx_id, &idx, false))
3109         return false;
3110
3111     idx.tag_seek[tag] = data;
3112     idx.flag |= FLAG_DIRTYNUM;
3113
3114     return write_index(masterfd, idx_id, &idx);
3115 }
3116
3117 #if 0
3118 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3119                                    int tag, long data)
3120 {
3121     struct master_header myhdr;
3122
3123     if (tcs->masterfd < 0)
3124     {
3125         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3126             return false;
3127     }
3128
3129     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3130 }
3131 #endif
3132
3133 static bool command_queue_is_full(void)
3134 {
3135     int next;
3136
3137     next = command_queue_widx + 1;
3138     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3139         next = 0;
3140
3141     return (next == command_queue_ridx);
3142 }
3143
3144 static void command_queue_sync_callback(void *data)
3145 {
3146     (void)data;
3147     struct master_header myhdr;
3148     int masterfd;
3149
3150     mutex_lock(&command_queue_mutex);
3151
3152     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3153         return;
3154
3155     while (command_queue_ridx != command_queue_widx)
3156     {
3157         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3158
3159         switch (ce->command)
3160         {
3161             case CMD_UPDATE_MASTER_HEADER:
3162             {
3163                 close(masterfd);
3164                 update_master_header();
3165
3166                 /* Re-open the masterfd. */
3167                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3168                     return;
3169
3170                 break;
3171             }
3172             case CMD_UPDATE_NUMERIC:
3173             {
3174                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3175                 break;
3176             }
3177         }
3178
3179         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3180             command_queue_ridx = 0;
3181     }
3182
3183     close(masterfd);
3184
3185     tc_stat.queue_length = 0;
3186     mutex_unlock(&command_queue_mutex);
3187 }
3188
3189 static void run_command_queue(bool force)
3190 {
3191     if (COMMAND_QUEUE_IS_EMPTY)
3192         return;
3193
3194     if (force || command_queue_is_full())
3195         command_queue_sync_callback(NULL);
3196     else
3197         register_storage_idle_func(command_queue_sync_callback);
3198 }
3199
3200 static void queue_command(int cmd, long idx_id, int tag, long data)
3201 {
3202     while (1)
3203     {
3204         int next;
3205
3206         mutex_lock(&command_queue_mutex);
3207         next = command_queue_widx + 1;
3208         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3209             next = 0;
3210
3211         /* Make sure queue is not full. */
3212         if (next != command_queue_ridx)
3213         {
3214             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3215
3216             ce->command = cmd;
3217             ce->idx_id = idx_id;
3218             ce->tag = tag;
3219             ce->data = data;
3220
3221             command_queue_widx = next;
3222
3223             tc_stat.queue_length++;
3224
3225             mutex_unlock(&command_queue_mutex);
3226             break;
3227         }
3228
3229         /* Queue is full, try again later... */
3230         mutex_unlock(&command_queue_mutex);
3231         sleep(1);
3232     }
3233 }
3234
3235 long tagcache_increase_serial(void)
3236 {
3237     long old;
3238
3239     if (!tc_stat.ready)
3240         return -2;
3241
3242     while (read_lock)
3243         sleep(1);
3244
3245     old = current_tcmh.serial++;
3246     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3247
3248     return old;
3249 }
3250
3251 void tagcache_update_numeric(int idx_id, int tag, long data)
3252 {
3253     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3254 }
3255 #endif /* !__PCTOOL__ */
3256
3257 long tagcache_get_serial(void)
3258 {
3259     return current_tcmh.serial;
3260 }
3261
3262 long tagcache_get_commitid(void)
3263 {
3264     return current_tcmh.commitid;
3265 }
3266
3267 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3268 {
3269     char buf[512];
3270     int i;
3271
3272     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3273     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3274     {
3275         if (*datastr == '\0')
3276             break;
3277
3278         if (*datastr == '"' || *datastr == '\\')
3279             buf[i++] = '\\';
3280
3281         buf[i] = *(datastr++);
3282     }
3283
3284     strcpy(&buf[i], "\" ");
3285
3286     write(fd, buf, i + 2);
3287
3288     return true;
3289 }
3290
3291 #ifndef __PCTOOL__
3292
3293 static bool read_tag(char *dest, long size,
3294                      const char *src, const char *tagstr)
3295 {
3296     int pos;
3297     char current_tag[32];
3298
3299     while (*src != '\0')
3300     {
3301         /* Skip all whitespace */
3302         while (*src == ' ')
3303             src++;
3304
3305         if (*src == '\0')
3306             break;
3307
3308         pos = 0;
3309         /* Read in tag name */
3310         while (*src != '=' && *src != ' ')
3311         {
3312             current_tag[pos] = *src;
3313             src++;
3314             pos++;
3315
3316             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3317                 return false;
3318         }
3319         current_tag[pos] = '\0';
3320
3321         /* Read in tag data */
3322
3323         /* Find the start. */
3324         while (*src != '"' && *src != '\0')
3325             src++;
3326
3327         if (*src == '\0' || *(++src) == '\0')
3328             return false;
3329
3330         /* Read the data. */
3331         for (pos = 0; pos < size; pos++)
3332         {
3333             if (*src == '\0')
3334                 break;
3335
3336             if (*src == '\\')
3337             {
3338                 dest[pos] = *(src+1);
3339                 src += 2;
3340                 continue;
3341             }
3342
3343             dest[pos] = *src;
3344
3345             if (*src == '"')
3346             {
3347                 src++;
3348                 break;
3349             }
3350
3351             if (*src == '\0')
3352                 break;
3353
3354             src++;
3355         }
3356         dest[pos] = '\0';
3357
3358         if (!strcasecmp(tagstr, current_tag))
3359             return true;
3360     }
3361
3362     return false;
3363 }
3364
3365 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3366 {
3367     struct index_entry idx;
3368     char tag_data[TAG_MAXLEN+32];
3369     int idx_id;
3370     long masterfd = (long)parameters;
3371     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3372         tag_commitid };
3373     int i;
3374     (void)line_n;
3375
3376     if (*buf == '#')
3377         return 0;
3378
3379     logf("%d/%s", line_n, buf);
3380     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3381     {
3382         logf("filename missing");
3383         logf("-> %s", buf);
3384         return 0;
3385     }
3386
3387     idx_id = find_index(tag_data);
3388     if (idx_id < 0)
3389     {
3390         logf("entry not found");
3391         return 0;
3392     }
3393
3394     if (!get_index(masterfd, idx_id, &idx, false))
3395     {
3396         logf("failed to retrieve index entry");
3397         return 0;
3398     }
3399
3400     /* Stop if tag has already been modified. */
3401     if (idx.flag & FLAG_DIRTYNUM)
3402         return 0;
3403
3404     logf("import: %s", tag_data);
3405
3406     idx.flag |= FLAG_DIRTYNUM;
3407     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3408     {
3409         int data;
3410
3411         if (!read_tag(tag_data, sizeof tag_data, buf,
3412                       tagcache_tag_to_str(import_tags[i])))
3413         {
3414             continue;
3415         }
3416
3417         data = atoi(tag_data);
3418         if (data < 0)
3419             continue;
3420
3421         idx.tag_seek[import_tags[i]] = data;
3422
3423         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3424             current_tcmh.serial = data + 1;
3425         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3426             current_tcmh.commitid = data + 1;
3427     }
3428
3429     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3430 }
3431
3432 bool tagcache_import_changelog(void)
3433 {
3434     struct master_header myhdr;
3435     struct tagcache_header tch;
3436     int clfd;
3437     long masterfd;
3438     char buf[2048];
3439
3440     if (!tc_stat.ready)
3441         return false;
3442
3443     while (read_lock)
3444         sleep(1);
3445
3446     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3447     if (clfd < 0)
3448     {
3449         logf("failure to open changelog");
3450         return false;
3451     }
3452
3453     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3454     {
3455         close(clfd);
3456         return false;
3457     }
3458
3459     write_lock++;
3460
3461     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3462
3463     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3464                   parse_changelog_line);
3465
3466     close(clfd);
3467     close(masterfd);
3468
3469     if (filenametag_fd >= 0)
3470     {
3471         close(filenametag_fd);
3472         filenametag_fd = -1;
3473     }
3474
3475     write_lock--;
3476
3477     update_master_header();
3478
3479     return true;
3480 }
3481
3482 #endif /* !__PCTOOL__ */
3483
3484 bool tagcache_create_changelog(struct tagcache_search *tcs)
3485 {
3486     struct master_header myhdr;
3487     struct index_entry idx;
3488     char buf[TAG_MAXLEN+32];
3489     char temp[32];
3490     int clfd;
3491     int i, j;
3492
3493     if (!tc_stat.ready)
3494         return false;
3495
3496     if (!tagcache_search(tcs, tag_filename))
3497         return false;
3498
3499     /* Initialize the changelog */
3500     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3501     if (clfd < 0)
3502     {
3503         logf("failure to open changelog");
3504         return false;
3505     }
3506
3507     if (tcs->masterfd < 0)
3508     {
3509         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3510             return false;
3511     }
3512     else
3513     {
3514         lseek(tcs->masterfd, 0, SEEK_SET);
3515         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3516     }
3517
3518     write(clfd, "## Changelog version 1\n", 23);
3519
3520     for (i = 0; i < myhdr.tch.entry_count; i++)
3521     {
3522         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3523         {
3524             logf("read error #9");
3525             tagcache_search_finish(tcs);
3526             close(clfd);
3527             return false;
3528         }
3529
3530         /* Skip until the entry found has been modified. */
3531         if (! (idx.flag & FLAG_DIRTYNUM) )
3532             continue;
3533
3534         /* Skip deleted entries too. */
3535         if (idx.flag & FLAG_DELETED)
3536             continue;
3537
3538         /* Now retrieve all tags. */
3539         for (j = 0; j < TAG_COUNT; j++)
3540         {
3541             if (TAGCACHE_IS_NUMERIC(j))
3542             {
3543                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3544                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3545                 continue;
3546             }
3547
3548             tcs->type = j;
3549             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3550             write_tag(clfd, tagcache_tag_to_str(j), buf);
3551         }
3552
3553         write(clfd, "\n", 1);
3554         do_timed_yield();
3555     }
3556
3557     close(clfd);
3558
3559     tagcache_search_finish(tcs);
3560
3561     return true;
3562 }
3563
3564 static bool delete_entry(long idx_id)
3565 {
3566     int fd = -1;
3567     int masterfd = -1;
3568     int tag, i;
3569     struct index_entry idx, myidx;
3570     struct master_header myhdr;
3571     char buf[TAG_MAXLEN+32];
3572     int in_use[TAG_COUNT];
3573
3574     logf("delete_entry(): %ld", idx_id);
3575
3576 #ifdef HAVE_TC_RAMCACHE
3577     /* At first mark the entry removed from ram cache. */
3578     if (tc_stat.ramcache)
3579         hdr->indices[idx_id].flag |= FLAG_DELETED;
3580 #endif
3581
3582     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3583         return false;
3584
3585     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3586     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3587     {
3588         logf("delete_entry(): read error");
3589         goto cleanup;
3590     }
3591
3592     if (myidx.flag & FLAG_DELETED)
3593     {
3594         logf("delete_entry(): already deleted!");
3595         goto cleanup;
3596     }
3597
3598     myidx.flag |= FLAG_DELETED;
3599     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3600     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3601     {
3602         logf("delete_entry(): write_error #1");
3603         goto cleanup;
3604     }
3605
3606     /* Now check which tags are no longer in use (if any) */
3607     for (tag = 0; tag < TAG_COUNT; tag++)
3608         in_use[tag] = 0;
3609
3610     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3611     for (i = 0; i < myhdr.tch.entry_count; i++)
3612     {
3613         struct index_entry *idxp;
3614
3615 #ifdef HAVE_TC_RAMCACHE
3616         /* Use RAM DB if available for greater speed */
3617         if (tc_stat.ramcache)
3618             idxp = &hdr->indices[i];
3619         else
3620 #endif
3621         {
3622             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3623             {
3624                 logf("delete_entry(): read error #2");
3625                 goto cleanup;
3626             }
3627             idxp = &idx;
3628         }
3629
3630         if (idxp->flag & FLAG_DELETED)
3631             continue;
3632
3633         for (tag = 0; tag < TAG_COUNT; tag++)
3634         {
3635             if (TAGCACHE_IS_NUMERIC(tag))
3636                 continue;
3637
3638             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3639                 in_use[tag]++;
3640         }
3641     }
3642
3643     /* Now delete all tags no longer in use. */
3644     for (tag = 0; tag < TAG_COUNT; tag++)
3645     {
3646         struct tagcache_header tch;
3647         int oldseek = myidx.tag_seek[tag];
3648
3649         if (TAGCACHE_IS_NUMERIC(tag))
3650             continue;
3651
3652         /**
3653          * Replace tag seek with a hash value of the field string data.
3654          * That way runtime statistics of moved or altered files can be
3655          * resurrected.
3656          */
3657 #ifdef HAVE_TC_RAMCACHE
3658         if (tc_stat.ramcache && tag != tag_filename)
3659         {
3660             struct tagfile_entry *tfe;
3661             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3662
3663             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3664             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3665             myidx.tag_seek[tag] = *seek;
3666         }
3667         else
3668 #endif
3669         {
3670             struct tagfile_entry tfe;
3671
3672             /* Open the index file, which contains the tag names. */
3673             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3674                 goto cleanup;
3675
3676             /* Skip the header block */
3677             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3678             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3679             {
3680                 logf("delete_entry(): read error #3");
3681                 goto cleanup;
3682             }
3683
3684             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3685             {
3686                 logf("delete_entry(): read error #4");
3687                 goto cleanup;
3688             }
3689
3690             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3691         }
3692
3693         if (in_use[tag])
3694         {
3695             logf("in use: %d/%d", tag, in_use[tag]);
3696             if (fd >= 0)
3697             {
3698                 close(fd);
3699                 fd = -1;
3700             }
3701             continue;
3702         }
3703
3704 #ifdef HAVE_TC_RAMCACHE
3705         /* Delete from ram. */
3706         if (tc_stat.ramcache && tag != tag_filename)
3707         {
3708             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3709             tagentry->tag_data[0] = '\0';
3710         }
3711 #endif
3712
3713         /* Open the index file, which contains the tag names. */
3714         if (fd < 0)
3715         {
3716             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3717                 goto cleanup;
3718         }
3719
3720         /* Skip the header block */
3721         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3722
3723         /* Debug, print 10 first characters of the tag
3724         read(fd, buf, 10);
3725         buf[10]='\0';
3726         logf("TAG:%s", buf);
3727         lseek(fd, -10, SEEK_CUR);
3728         */
3729
3730         /* Write first data byte in tag as \0 */
3731         write(fd, "", 1);
3732
3733         /* Now tag data has been removed */
3734         close(fd);
3735         fd = -1;
3736     }
3737
3738     /* Write index entry back into master index. */
3739     lseek(masterfd, sizeof(struct master_header) +
3740           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3741     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3742     {
3743         logf("delete_entry(): write_error #2");
3744         goto cleanup;
3745     }
3746
3747     close(masterfd);
3748
3749     return true;
3750
3751     cleanup:
3752     if (fd >= 0)
3753         close(fd);
3754     if (masterfd >= 0)
3755         close(masterfd);
3756
3757     return false;
3758 }
3759
3760 #ifndef __PCTOOL__
3761 /**
3762  * Returns true if there is an event waiting in the queue
3763  * that requires the current operation to be aborted.
3764  */
3765 static bool check_event_queue(void)
3766 {
3767     struct queue_event ev;
3768
3769     if(!queue_peek(&tagcache_queue, &ev))
3770         return false;
3771
3772     switch (ev.id)
3773     {
3774         case Q_STOP_SCAN:
3775         case SYS_POWEROFF:
3776         case SYS_USB_CONNECTED:
3777             return true;
3778     }
3779
3780     return false;
3781 }
3782 #endif
3783
3784 #ifdef HAVE_TC_RAMCACHE
3785 static bool allocate_tagcache(void)
3786 {
3787     struct master_header tcmh;
3788     int fd;
3789
3790     /* Load the header. */
3791     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3792     {
3793         hdr = NULL;
3794         return false;
3795     }
3796
3797     close(fd);
3798
3799     /**
3800      * Now calculate the required cache size plus
3801      * some extra space for alignment fixes.
3802      */
3803     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3804         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3805     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3806     memset(hdr, 0, sizeof(struct ramcache_header));
3807     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3808     hdr->indices = (struct index_entry *)(hdr + 1);
3809     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3810
3811     return true;
3812 }
3813
3814 # ifdef HAVE_EEPROM_SETTINGS
3815 static bool tagcache_dumpload(void)
3816 {
3817     struct statefile_header shdr;
3818     int fd, rc;
3819     long offpos;
3820     int i;
3821
3822     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3823     if (fd < 0)
3824     {
3825         logf("no tagcache statedump");
3826         return false;
3827     }
3828
3829     /* Check the statefile memory placement */
3830     hdr = buffer_alloc(0);
3831     rc = read(fd, &shdr, sizeof(struct statefile_header));
3832     if (rc != sizeof(struct statefile_header)
3833         /* || (long)hdr != (long)shdr.hdr */)
3834     {
3835         logf("incorrect statefile");
3836         hdr = NULL;
3837         close(fd);
3838         return false;
3839     }
3840
3841     offpos = (long)hdr - (long)shdr.hdr;
3842
3843     /* Lets allocate real memory and load it */
3844     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3845     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3846     close(fd);
3847
3848     if (rc != shdr.tc_stat.ramcache_allocated)
3849     {
3850         logf("read failure!");
3851         hdr = NULL;
3852         return false;
3853     }
3854
3855     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3856
3857     /* Now fix the pointers */
3858     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3859     for (i = 0; i < TAG_COUNT; i++)
3860         hdr->tags[i] += offpos;
3861
3862     return true;
3863 }
3864
3865 static bool tagcache_dumpsave(void)
3866 {
3867     struct statefile_header shdr;
3868     int fd;
3869
3870     if (!tc_stat.ramcache)
3871         return false;
3872
3873     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3874     if (fd < 0)
3875     {
3876         logf("failed to create a statedump");
3877         return false;
3878     }
3879
3880     /* Create the header */
3881     shdr.hdr = hdr;
3882     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3883     write(fd, &shdr, sizeof(struct statefile_header));
3884
3885     /* And dump the data too */
3886     write(fd, hdr, tc_stat.ramcache_allocated);
3887     close(fd);
3888
3889     return true;
3890 }
3891 # endif
3892
3893 static bool load_tagcache(void)
3894 {
3895     struct tagcache_header *tch;
3896     long bytesleft = tc_stat.ramcache_allocated;
3897     struct index_entry *idx;
3898     int rc, fd;
3899     char *p;
3900     int i, tag;
3901
3902 # ifdef HAVE_DIRCACHE
3903     while (dircache_is_initializing())
3904         sleep(1);
3905
3906     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3907 # endif
3908
3909     logf("loading tagcache to ram...");
3910
3911     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3912     if (fd < 0)
3913     {
3914         logf("tagcache open failed");
3915         return false;
3916     }
3917
3918     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3919         != sizeof(struct master_header)
3920         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3921     {
3922         logf("incorrect header");
3923         return false;
3924     }
3925
3926     idx = hdr->indices;
3927
3928     /* Load the master index table. */
3929     for (i = 0; i < hdr->h.tch.entry_count; i++)
3930     {
3931         rc = ecread_index_entry(fd, idx);
3932         if (rc != sizeof(struct index_entry))
3933         {
3934             logf("read error #10");
3935             close(fd);
3936             return false;
3937         }
3938
3939         bytesleft -= sizeof(struct index_entry);
3940         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3941         {
3942             logf("too big tagcache.");
3943             close(fd);
3944             return false;
3945         }
3946
3947         idx++;
3948     }
3949
3950     close(fd);
3951
3952     /* Load the tags. */
3953     p = (char *)idx;
3954     for (tag = 0; tag < TAG_COUNT; tag++)
3955     {
3956         struct tagfile_entry *fe;
3957         char buf[TAG_MAXLEN+32];
3958
3959         if (TAGCACHE_IS_NUMERIC(tag))
3960             continue ;
3961
3962         //p = ((void *)p+1);
3963         p = (char *)((long)p & ~0x03) + 0x04;
3964         hdr->tags[tag] = p;
3965
3966         /* Check the header. */
3967         tch = (struct tagcache_header *)p;
3968         p += sizeof(struct tagcache_header);
3969
3970         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3971             return false;
3972
3973         for (hdr->entry_count[tag] = 0;
3974              hdr->entry_count[tag] < tch->entry_count;
3975              hdr->entry_count[tag]++)
3976         {
3977             long pos;
3978
3979             if (do_timed_yield())
3980             {
3981                 /* Abort if we got a critical event in queue */
3982                 if (check_event_queue())
3983                     return false;
3984             }
3985
3986             fe = (struct tagfile_entry *)p;
3987             pos = lseek(fd, 0, SEEK_CUR);
3988             rc = ecread_tagfile_entry(fd, fe);
3989             if (rc != sizeof(struct tagfile_entry))
3990             {
3991                 /* End of lookup table. */
3992                 logf("read error #11");
3993                 close(fd);
3994                 return false;
3995             }
3996
3997             /* We have a special handling for the filename tags. */
3998             if (tag == tag_filename)
3999             {
4000 # ifdef HAVE_DIRCACHE
4001                 const struct dircache_entry *dc;
4002 # endif
4003
4004                 // FIXME: This is wrong!
4005                 // idx = &hdr->indices[hdr->entry_count[i]];
4006                 idx = &hdr->indices[fe->idx_id];
4007
4008                 if (fe->tag_length >= (long)sizeof(buf)-1)
4009                 {
4010                     read(fd, buf, 10);
4011                     buf[10] = '\0';
4012                     logf("TAG:%s", buf);
4013                     logf("too long filename");
4014                     close(fd);
4015                     return false;
4016                 }
4017
4018                 rc = read(fd, buf, fe->tag_length);
4019                 if (rc != fe->tag_length)
4020                 {
4021                     logf("read error #12");
4022                     close(fd);
4023                     return false;
4024                 }
4025
4026                 /* Check if the entry has already been removed */
4027                 if (idx->flag & FLAG_DELETED)
4028                     continue;
4029
4030                 /* This flag must not be used yet. */
4031                 if (idx->flag & FLAG_DIRCACHE)
4032                 {
4033                     logf("internal error!");
4034                     close(fd);
4035                     return false;
4036                 }
4037
4038                 if (idx->tag_seek[tag] != pos)
4039                 {
4040                     logf("corrupt data structures!");
4041                     close(fd);
4042                     return false;
4043                 }
4044
4045 # ifdef HAVE_DIRCACHE
4046                 if (dircache_is_enabled())
4047                 {
4048                     dc = dircache_get_entry_ptr(buf);
4049                     if (dc == NULL)
4050                     {
4051                         logf("Entry no longer valid.");
4052                         logf("-> %s", buf);
4053                         if (global_settings.tagcache_autoupdate)
4054                             delete_entry(fe->idx_id);
4055                         continue ;
4056                     }
4057
4058                     idx->flag |= FLAG_DIRCACHE;
4059                     idx->tag_seek[tag_filename] = (long)dc;
4060                 }
4061                 else
4062 # endif
4063                 {
4064                     /* This will be very slow unless dircache is enabled
4065                        or target is flash based, but do it anyway for
4066                        consistency. */
4067                     /* Check if entry has been removed. */
4068                     if (global_settings.tagcache_autoupdate)
4069                     {
4070                         if (!file_exists(buf))
4071                         {
4072                             logf("Entry no longer valid.");
4073                             logf("-> %s", buf);
4074                             delete_entry(fe->idx_id);
4075                             continue;
4076                         }
4077                     }
4078                 }
4079
4080                 continue ;
4081             }
4082
4083             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4084             if (bytesleft < 0)
4085             {
4086                 logf("too big tagcache #2");
4087                 logf("tl: %ld", fe->tag_length);
4088                 logf("bl: %ld", bytesleft);
4089                 close(fd);
4090                 return false;
4091             }
4092
4093             p = fe->tag_data;
4094             rc = read(fd, fe->tag_data, fe->tag_length);
4095             p += rc;
4096
4097             if (rc != fe->tag_length)
4098             {
4099                 logf("read error #13");
4100                 logf("rc=0x%04x", rc); // 0x431
4101                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4102                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4103                 logf("tag=0x%02x", tag); // 0x00
4104                 close(fd);
4105                 return false;
4106             }
4107         }
4108         close(fd);
4109     }
4110
4111     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4112     logf("tagcache loaded into ram!");
4113
4114     return true;
4115 }
4116 #endif /* HAVE_TC_RAMCACHE */
4117
4118 static bool check_deleted_files(void)
4119 {
4120     int fd;
4121     char buf[TAG_MAXLEN+32];
4122     struct tagfile_entry tfe;
4123
4124     logf("reverse scan...");
4125     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4126     fd = open(buf, O_RDONLY);
4127
4128     if (fd < 0)
4129     {
4130         logf("%s open fail", buf);
4131         return false;
4132     }
4133
4134     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4135     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4136 #ifndef __PCTOOL__
4137            && !check_event_queue()
4138 #endif
4139            )
4140     {
4141         if (tfe.tag_length >= (long)sizeof(buf)-1)
4142         {
4143             logf("too long tag");
4144             close(fd);
4145             return false;
4146         }
4147
4148         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4149         {
4150             logf("read error #14");
4151             close(fd);
4152             return false;
4153         }
4154
4155         /* Check if the file has already deleted from the db. */
4156         if (*buf == '\0')
4157             continue;
4158
4159         /* Now check if the file exists. */
4160         if (!file_exists(buf))
4161         {
4162             logf("Entry no longer valid.");
4163             logf("-> %s / %ld", buf, tfe.tag_length);
4164             delete_entry(tfe.idx_id);
4165         }
4166     }
4167
4168     close(fd);
4169
4170     logf("done");
4171
4172     return true;
4173 }
4174
4175
4176 /* Note that this function must not be inlined, otherwise the whole point
4177  * of having the code in a separate function is lost.
4178  */
4179 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4180     int *ignore, int *unignore)
4181 {
4182     char newpath[MAX_PATH];
4183
4184     /* check for a database.ignore file */
4185     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4186     *ignore = file_exists(newpath);
4187     /* check for a database.unignore file */
4188     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4189     *unignore = file_exists(newpath);
4190 }
4191
4192
4193 static bool check_dir(const char *dirname, int add_files)
4194 {
4195     DIR *dir;
4196     int len;
4197     int success = false;
4198     int ignore, unignore;
4199
4200     dir = opendir(dirname);
4201     if (!dir)
4202     {
4203         logf("tagcache: opendir(%s) failed", dirname);
4204         return false;
4205     }
4206
4207     /* check for a database.ignore and database.unignore */
4208     check_ignore(dirname, &ignore, &unignore);
4209
4210     /* don't do anything if both ignore and unignore are there */
4211     if (ignore != unignore)
4212         add_files = unignore;
4213
4214     /* Recursively scan the dir. */
4215 #ifdef __PCTOOL__
4216     while (1)
4217 #else
4218     while (!check_event_queue())
4219 #endif
4220     {
4221         struct dirent *entry;
4222
4223         entry = readdir(dir);
4224
4225         if (entry == NULL)
4226         {
4227             success = true;
4228             break ;
4229         }
4230
4231         struct dirinfo info = dir_get_info(dir, entry);
4232
4233         if (!strcmp((char *)entry->d_name, ".") ||
4234             !strcmp((char *)entry->d_name, ".."))
4235             continue;
4236
4237         yield();
4238
4239         len = strlen(curpath);
4240         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4241                  entry->d_name);
4242
4243         processed_dir_count++;
4244         if (info.attribute & ATTR_DIRECTORY)
4245             check_dir(curpath, add_files);
4246         else if (add_files)
4247         {
4248             tc_stat.curentry = curpath;
4249
4250             /* Add a new entry to the temporary db file. */
4251             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4252 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4253                          , dir->internal_entry
4254 #endif
4255                          );
4256
4257             /* Wait until current path for debug screen is read and unset. */
4258             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4259                 yield();
4260
4261             tc_stat.curentry = NULL;
4262         }
4263
4264         curpath[len] = '\0';
4265     }
4266
4267     closedir(dir);
4268
4269     return success;
4270 }
4271
4272 void tagcache_screensync_event(void)
4273 {
4274     tc_stat.curentry = NULL;
4275 }
4276
4277 void tagcache_screensync_enable(bool state)
4278 {
4279     tc_stat.syncscreen = state;
4280 }
4281
4282 void tagcache_build(const char *path)
4283 {
4284     struct tagcache_header header;
4285     bool ret;
4286
4287     curpath[0] = '\0';
4288     data_size = 0;
4289     total_entry_count = 0;
4290     processed_dir_count = 0;
4291
4292 #ifdef HAVE_DIRCACHE
4293     while (dircache_is_initializing())
4294         sleep(1);
4295 #endif
4296
4297     logf("updating tagcache");
4298
4299     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4300     if (cachefd >= 0)
4301     {
4302         logf("skipping, cache already waiting for commit");
4303         close(cachefd);
4304         return ;
4305     }
4306
4307     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4308     if (cachefd < 0)
4309     {
4310         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4311         return ;
4312     }
4313
4314     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4315
4316     cpu_boost(true);
4317
4318     logf("Scanning files...");
4319     /* Scan for new files. */
4320     memset(&header, 0, sizeof(struct tagcache_header));
4321     write(cachefd, &header, sizeof(struct tagcache_header));
4322
4323     if (strcmp("/", path) != 0)
4324         strcpy(curpath, path);
4325     ret = check_dir(path, true);
4326
4327     /* Write the header. */
4328     header.magic = TAGCACHE_MAGIC;
4329     header.datasize = data_size;
4330     header.entry_count = total_entry_count;
4331     lseek(cachefd, 0, SEEK_SET);
4332     write(cachefd, &header, sizeof(struct tagcache_header));
4333     close(cachefd);
4334
4335     if (filenametag_fd >= 0)
4336     {
4337         close(filenametag_fd);
4338         filenametag_fd = -1;
4339     }
4340
4341     if (!ret)
4342     {
4343         logf("Aborted.");
4344         cpu_boost(false);
4345         return ;
4346     }
4347
4348     /* Commit changes to the database. */
4349 #ifdef __PCTOOL__
4350     allocate_tempbuf();
4351 #endif
4352     if (commit())
4353     {
4354         remove(TAGCACHE_FILE_TEMP);
4355         logf("tagcache built!");
4356     }
4357 #ifdef __PCTOOL__
4358     free_tempbuf();
4359 #endif
4360
4361 #ifdef HAVE_TC_RAMCACHE
4362     if (hdr)
4363     {
4364         /* Import runtime statistics if we just initialized the db. */
4365         if (hdr->h.serial == 0)
4366             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4367     }
4368 #endif
4369
4370     cpu_boost(false);
4371 }
4372
4373 #ifdef HAVE_TC_RAMCACHE
4374 static void load_ramcache(void)
4375 {
4376     if (!hdr)
4377         return ;
4378
4379     cpu_boost(true);
4380
4381     /* At first we should load the cache (if exists). */
4382     tc_stat.ramcache = load_tagcache();
4383
4384     if (!tc_stat.ramcache)
4385     {
4386         /* If loading failed, it must indicate some problem with the db
4387          * so disable it entirely to prevent further issues. */
4388         tc_stat.ready = false;
4389         hdr = NULL;
4390     }
4391
4392     cpu_boost(false);
4393 }
4394
4395 void tagcache_unload_ramcache(void)
4396 {
4397     tc_stat.ramcache = false;
4398     /* Just to make sure there is no statefile present. */
4399     // remove(TAGCACHE_STATEFILE);
4400 }
4401 #endif
4402
4403 #ifndef __PCTOOL__
4404 static void tagcache_thread(void)
4405 {
4406     struct queue_event ev;
4407     bool check_done = false;
4408
4409     /* If the previous cache build/update was interrupted, commit
4410      * the changes first in foreground. */
4411     cpu_boost(true);
4412     allocate_tempbuf();
4413     commit();
4414     free_tempbuf();
4415
4416 #ifdef HAVE_TC_RAMCACHE
4417 # ifdef HAVE_EEPROM_SETTINGS
4418     if (firmware_settings.initialized && firmware_settings.disk_clean
4419         && global_settings.tagcache_ram)
4420     {
4421         check_done = tagcache_dumpload();
4422     }
4423
4424     remove(TAGCACHE_STATEFILE);
4425 # endif
4426
4427     /* Allocate space for the tagcache if found on disk. */
4428     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4429         allocate_tagcache();
4430 #endif
4431
4432     cpu_boost(false);
4433     tc_stat.initialized = true;
4434
4435     /* Don't delay bootup with the header check but do it on background. */
4436     if (!tc_stat.ready)
4437     {
4438         sleep(HZ);
4439         tc_stat.ready = check_all_headers();
4440         tc_stat.readyvalid = true;
4441     }
4442
4443     while (1)
4444     {
4445         run_command_queue(false);
4446
4447         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4448
4449         switch (ev.id)
4450         {
4451             case Q_IMPORT_CHANGELOG:
4452                 tagcache_import_changelog();
4453                 break;
4454
4455             case Q_REBUILD:
4456                 remove_files();
4457                 remove(TAGCACHE_FILE_TEMP);
4458                 tagcache_build("/");
4459                 break;
4460
4461             case Q_UPDATE:
4462                 tagcache_build("/");
4463 #ifdef HAVE_TC_RAMCACHE
4464                 load_ramcache();
4465 #endif
4466                 check_deleted_files();
4467                 break ;
4468
4469             case Q_START_SCAN:
4470                 check_done = false;
4471             case SYS_TIMEOUT:
4472                 if (check_done || !tc_stat.ready)
4473                     break ;
4474
4475 #ifdef HAVE_TC_RAMCACHE
4476                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4477                 {
4478                     load_ramcache();
4479                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4480                         tagcache_build("/");
4481                 }
4482                 else
4483 #endif
4484                 if (global_settings.tagcache_autoupdate)
4485                 {
4486                     tagcache_build("/");
4487
4488                     /* This will be very slow unless dircache is enabled
4489                        or target is flash based, but do it anyway for
4490                        consistency. */
4491                     check_deleted_files();
4492                 }
4493
4494                 logf("tagcache check done");
4495
4496                 check_done = true;
4497                 break ;
4498
4499             case Q_STOP_SCAN:
4500                 break ;
4501
4502             case SYS_POWEROFF:
4503                 break ;
4504
4505 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4506             case SYS_USB_CONNECTED:
4507                 logf("USB: TagCache");
4508                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4509                 usb_wait_for_disconnect(&tagcache_queue);
4510                 break ;
4511 #endif
4512         }
4513     }
4514 }
4515
4516 bool tagcache_prepare_shutdown(void)
4517 {
4518     if (tagcache_get_commit_step() > 0)
4519         return false;
4520
4521     tagcache_stop_scan();
4522     while (read_lock || write_lock)
4523         sleep(1);
4524
4525     return true;
4526 }
4527
4528 void tagcache_shutdown(void)
4529 {
4530     /* Flush the command queue. */
4531     run_command_queue(true);
4532
4533 #ifdef HAVE_EEPROM_SETTINGS
4534     if (tc_stat.ramcache)
4535         tagcache_dumpsave();
4536 #endif
4537 }
4538
4539 static int get_progress(void)
4540 {
4541     int total_count = -1;
4542
4543 #ifdef HAVE_DIRCACHE
4544     if (dircache_is_enabled())
4545     {
4546         total_count = dircache_get_entry_count();
4547     }
4548     else
4549 #endif
4550 #ifdef HAVE_TC_RAMCACHE
4551     {
4552         if (hdr && tc_stat.ramcache)
4553             total_count = hdr->h.tch.entry_count;
4554     }
4555 #endif
4556
4557     if (total_count < 0)
4558         return -1;
4559
4560     return processed_dir_count * 100 / total_count;
4561 }
4562
4563 struct tagcache_stat* tagcache_get_stat(void)
4564 {
4565     tc_stat.progress = get_progress();
4566     tc_stat.processed_entries = processed_dir_count;
4567
4568     return &tc_stat;
4569 }
4570
4571 void tagcache_start_scan(void)
4572 {
4573     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4574 }
4575
4576 bool tagcache_update(void)
4577 {
4578     if (!tc_stat.ready)
4579         return false;
4580
4581     queue_post(&tagcache_queue, Q_UPDATE, 0);
4582     return false;
4583 }
4584
4585 bool tagcache_rebuild()
4586 {
4587     queue_post(&tagcache_queue, Q_REBUILD, 0);
4588     return false;
4589 }
4590
4591 void tagcache_stop_scan(void)
4592 {
4593     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4594 }
4595
4596 #ifdef HAVE_TC_RAMCACHE
4597 bool tagcache_is_ramcache(void)
4598 {
4599     return tc_stat.ramcache;
4600 }
4601 #endif
4602
4603 #endif /* !__PCTOOL__ */
4604
4605
4606 void tagcache_init(void)
4607 {
4608     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4609     memset(&current_tcmh, 0, sizeof(struct master_header));
4610     filenametag_fd = -1;
4611     write_lock = read_lock = 0;
4612
4613 #ifndef __PCTOOL__
4614     mutex_init(&command_queue_mutex);
4615     queue_init(&tagcache_queue, true);
4616     create_thread(tagcache_thread, tagcache_stack,
4617                   sizeof(tagcache_stack), 0, tagcache_thread_name
4618                   IF_PRIO(, PRIORITY_BACKGROUND)
4619                   IF_COP(, CPU));
4620 #else
4621     tc_stat.initialized = true;
4622     allocate_tempbuf();
4623     commit();
4624     free_tempbuf();
4625     tc_stat.ready = check_all_headers();
4626 #endif
4627 }
4628
4629 #ifdef __PCTOOL__
4630 void tagcache_reverse_scan(void)
4631 {
4632     logf("Checking for deleted files");
4633     check_deleted_files();
4634 }
4635 #endif
4636
4637 bool tagcache_is_initialized(void)
4638 {
4639     return tc_stat.initialized;
4640 }
4641 bool tagcache_is_usable(void)
4642 {
4643     return tc_stat.initialized && tc_stat.ready;
4644 }
4645 int tagcache_get_commit_step(void)
4646 {
4647     return tc_stat.commit_step;
4648 }
4649 int tagcache_get_max_commit_step(void)
4650 {
4651     return (int)(SORTED_TAGS_COUNT)+1;
4652 }
4653