apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 /* Previous path when scanning directory tree recursively. */
  99 static char curpath[TAG_MAXLEN+32];
 100
 101 /* Used when removing duplicates. */
 102 static char *tempbuf;     /* Allocated when needed. */
 103 static long tempbufidx;   /* Current location in buffer. */
 104 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 105 static long tempbuf_left; /* Buffer space left. */
 106 static long tempbuf_pos;
 107
 108 #define SORTED_TAGS_COUNT 8
 109 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 110 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 111 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 112     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 113 /* Tags we want to get sorted (loaded to the tempbuf). */
 114 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 115     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 116     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 117
 118 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 119 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 120     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 121     (1LU << tag_albumartist) | (1LU << tag_grouping))
 122
 123 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 124 static const char *tags_str[] = { "artist", "album", "genre", "title",
 125     "filename", "composer", "comment", "albumartist", "grouping", "year",
 126     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 127     "playtime", "lastplayed", "commitid", "mtime" };
 128
 129 /* Status information of the tagcache. */
 130 static struct tagcache_stat tc_stat;
 131
 132 /* Queue commands. */
 133 enum tagcache_queue {
 134     Q_STOP_SCAN = 0,
 135     Q_START_SCAN,
 136     Q_IMPORT_CHANGELOG,
 137     Q_UPDATE,
 138     Q_REBUILD,
 139
 140     /* Internal tagcache command queue. */
 141     CMD_UPDATE_MASTER_HEADER,
 142     CMD_UPDATE_NUMERIC,
 143 };
 144
 145 struct tagcache_command_entry {
 146     int32_t command;
 147     int32_t idx_id;
 148     int32_t tag;
 149     int32_t data;
 150 };
 151
 152 #ifndef __PCTOOL__
 153 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 154 static volatile int command_queue_widx = 0;
 155 static volatile int command_queue_ridx = 0;
 156 static struct mutex command_queue_mutex;
 157 #endif
 158
 159 /* Tag database structures. */
 160
 161 /* Variable-length tag entry in tag files. */
 162 struct tagfile_entry {
 163     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 164     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 165     char tag_data[0];  /* Begin of the tag data */
 166 };
 167
 168 /* Fixed-size tag entry in master db index. */
 169 struct index_entry {
 170     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 171     int32_t flag;                /* Status flags */
 172 };
 173
 174 /* Header is the same in every file. */
 175 struct tagcache_header {
 176     int32_t magic;       /* Header version number */
 177     int32_t datasize;    /* Data size in bytes */
 178     int32_t entry_count; /* Number of entries in this file */
 179 };
 180
 181 struct master_header {
 182     struct tagcache_header tch;
 183     int32_t serial; /* Increasing counting number */
 184     int32_t commitid; /* Number of commits so far */
 185     int32_t dirty;
 186 };
 187
 188 /* For the endianess correction */
 189 static const char *tagfile_entry_ec   = "ll";
 190 /**
 191  Note: This should be (1 + TAG_COUNT) amount of l's.
 192  */
 193 static const char *index_entry_ec     = "lllllllllllllllllllll";
 194
 195 static const char *tagcache_header_ec = "lll";
 196 static const char *master_header_ec   = "llllll";
 197
 198 static struct master_header current_tcmh;
 199
 200 #ifdef HAVE_TC_RAMCACHE
 201 /* Header is created when loading database to ram. */
 202 struct ramcache_header {
 203     struct master_header h;      /* Header from the master index */
 204     struct index_entry *indices; /* Master index file content */
 205     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 206     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 207 };
 208
 209 # ifdef HAVE_EEPROM_SETTINGS
 210 struct statefile_header {
 211     struct ramcache_header *hdr;
 212     struct tagcache_stat tc_stat;
 213 };
 214 # endif
 215
 216 /* Pointer to allocated ramcache_header */
 217 static struct ramcache_header *hdr;
 218 #endif
 219
 220 /**
 221  * Full tag entries stored in a temporary file waiting
 222  * for commit to the cache. */
 223 struct temp_file_entry {
 224     long tag_offset[TAG_COUNT];
 225     short tag_length[TAG_COUNT];
 226     long flag;
 227
 228     long data_length;
 229 };
 230
 231 struct tempbuf_id_list {
 232     long id;
 233     struct tempbuf_id_list *next;
 234 };
 235
 236 struct tempbuf_searchidx {
 237     long idx_id;
 238     char *str;
 239     int seek;
 240     struct tempbuf_id_list idlist;
 241 };
 242
 243 /* Lookup buffer for fixing messed up index while after sorting. */
 244 static long commit_entry_count;
 245 static long lookup_buffer_depth;
 246 static struct tempbuf_searchidx **lookup;
 247
 248 /* Used when building the temporary file. */
 249 static int cachefd = -1, filenametag_fd;
 250 static int total_entry_count = 0;
 251 static int data_size = 0;
 252 static int processed_dir_count;
 253
 254 /* Thread safe locking */
 255 static volatile int write_lock;
 256 static volatile int read_lock;
 257
 258 static bool delete_entry(long idx_id);
 259
 260 const char* tagcache_tag_to_str(int tag)
 261 {
 262     return tags_str[tag];
 263 }
 264
 265 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 266 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 267 {
 268     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 269 }
 270
 271 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 272 {
 273     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 274 }
 275
 276 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 277 {
 278     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 279 }
 280
 281 #ifdef HAVE_DIRCACHE
 282 /**
 283  * Returns true if specified flag is still present, i.e., dircache
 284  * has not been reloaded.
 285  */
 286 static bool is_dircache_intact(void)
 287 {
 288     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 289 }
 290 #endif
 291
 292 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 293 {
 294     int fd;
 295     char buf[MAX_PATH], path[MAX_PATH];
 296     const char * file;
 297     int rc;
 298
 299     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 300         return -1;
 301
 302     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 303     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
 304
 305     fd = open(file, write ? O_RDWR : O_RDONLY);
 306     if (fd < 0)
 307     {
 308         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 309         tc_stat.ready = false;
 310         return fd;
 311     }
 312
 313     /* Check the header. */
 314     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 315     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 316     {
 317         logf("header error");
 318         tc_stat.ready = false;
 319         close(fd);
 320         return -2;
 321     }
 322
 323     return fd;
 324 }
 325
 326 static int open_master_fd(struct master_header *hdr, bool write)
 327 {
 328     int fd;
 329     int rc;
 330     char path[MAX_PATH];
 331
 332     fd = open(get_user_file_path(TAGCACHE_FILE_MASTER,
 333                                  IS_FILE|NEED_WRITE,
 334                                  path, sizeof(path)),
 335                     write ? O_RDWR : O_RDONLY);
 336     if (fd < 0)
 337     {
 338         logf("master file open failed for R/W");
 339         tc_stat.ready = false;
 340         return fd;
 341     }
 342
 343     tc_stat.econ = false;
 344
 345     /* Check the header. */
 346     rc = read(fd, hdr, sizeof(struct master_header));
 347     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 348     {
 349         /* Success. */
 350         return fd;
 351     }
 352
 353     /* Trying to read again, this time with endianess correction enabled. */
 354     lseek(fd, 0, SEEK_SET);
 355
 356     rc = ecread(fd, hdr, 1, master_header_ec, true);
 357     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 358     {
 359         logf("header error");
 360         tc_stat.ready = false;
 361         close(fd);
 362         return -2;
 363     }
 364
 365     tc_stat.econ = true;
 366
 367     return fd;
 368 }
 369
 370 #ifndef __PCTOOL__
 371 static bool do_timed_yield(void)
 372 {
 373     /* Sorting can lock up for quite a while, so yield occasionally */
 374     static long wakeup_tick = 0;
 375     if (TIME_AFTER(current_tick, wakeup_tick))
 376     {
 377         wakeup_tick = current_tick + (HZ/4);
 378         yield();
 379         return true;
 380     }
 381     return false;
 382 }
 383 #endif
 384
 385 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 386 static long find_entry_ram(const char *filename,
 387                            const struct dircache_entry *dc)
 388 {
 389     static long last_pos = 0;
 390     int i;
 391
 392     /* Check if we tagcache is loaded into ram. */
 393     if (!tc_stat.ramcache)
 394         return -1;
 395
 396     if (dc == NULL)
 397         dc = dircache_get_entry_ptr(filename);
 398
 399     if (dc == NULL)
 400     {
 401         logf("tagcache: file not found.");
 402         return -1;
 403     }
 404
 405     try_again:
 406
 407     if (last_pos > 0)
 408         i = last_pos;
 409     else
 410         i = 0;
 411
 412     for (; i < hdr->h.tch.entry_count; i++)
 413     {
 414         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 415         {
 416             last_pos = MAX(0, i - 3);
 417             return i;
 418         }
 419
 420         do_timed_yield();
 421     }
 422
 423     if (last_pos > 0)
 424     {
 425         last_pos = 0;
 426         goto try_again;
 427     }
 428
 429     return -1;
 430 }
 431 #endif
 432
 433 static long find_entry_disk(const char *filename, bool localfd)
 434 {
 435     struct tagcache_header tch;
 436     static long last_pos = -1;
 437     long pos_history[POS_HISTORY_COUNT];
 438     long pos_history_idx = 0;
 439     bool found = false;
 440     struct tagfile_entry tfe;
 441     int fd;
 442     char buf[TAG_MAXLEN+32];
 443     int i;
 444     int pos = -1;
 445
 446     if (!tc_stat.ready)
 447         return -2;
 448
 449     fd = filenametag_fd;
 450     if (fd < 0 || localfd)
 451     {
 452         last_pos = -1;
 453         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 454             return -1;
 455     }
 456
 457     check_again:
 458
 459     if (last_pos > 0)
 460         lseek(fd, last_pos, SEEK_SET);
 461     else
 462         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 463
 464     while (true)
 465     {
 466         pos = lseek(fd, 0, SEEK_CUR);
 467         for (i = pos_history_idx-1; i >= 0; i--)
 468             pos_history[i+1] = pos_history[i];
 469         pos_history[0] = pos;
 470
 471         if (ecread_tagfile_entry(fd, &tfe)
 472             != sizeof(struct tagfile_entry))
 473         {
 474             break ;
 475         }
 476
 477         if (tfe.tag_length >= (long)sizeof(buf))
 478         {
 479             logf("too long tag #1");
 480             close(fd);
 481             if (!localfd)
 482                 filenametag_fd = -1;
 483             last_pos = -1;
 484             return -2;
 485         }
 486
 487         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 488         {
 489             logf("read error #2");
 490             close(fd);
 491             if (!localfd)
 492                 filenametag_fd = -1;
 493             last_pos = -1;
 494             return -3;
 495         }
 496
 497         if (!strcasecmp(filename, buf))
 498         {
 499             last_pos = pos_history[pos_history_idx];
 500             found = true;
 501             break ;
 502         }
 503
 504         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 505             pos_history_idx++;
 506     }
 507
 508     /* Not found? */
 509     if (!found)
 510     {
 511         if (last_pos > 0)
 512         {
 513             last_pos = -1;
 514             logf("seek again");
 515             goto check_again;
 516         }
 517
 518         if (fd != filenametag_fd || localfd)
 519             close(fd);
 520         return -4;
 521     }
 522
 523     if (fd != filenametag_fd || localfd)
 524         close(fd);
 525
 526     return tfe.idx_id;
 527 }
 528
 529 static int find_index(const char *filename)
 530 {
 531     long idx_id = -1;
 532
 533 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 534     if (tc_stat.ramcache && is_dircache_intact())
 535         idx_id = find_entry_ram(filename, NULL);
 536 #endif
 537
 538     if (idx_id < 0)
 539         idx_id = find_entry_disk(filename, true);
 540
 541     return idx_id;
 542 }
 543
 544 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 545 {
 546     int idx_id;
 547
 548     if (!tc_stat.ready)
 549         return false;
 550
 551     idx_id = find_index(filename);
 552     if (idx_id < 0)
 553         return false;
 554
 555     if (!tagcache_search(tcs, tag_filename))
 556         return false;
 557
 558     tcs->entry_count = 0;
 559     tcs->idx_id = idx_id;
 560
 561     return true;
 562 }
 563
 564 static bool get_index(int masterfd, int idxid,
 565                       struct index_entry *idx, bool use_ram)
 566 {
 567     bool localfd = false;
 568
 569     if (idxid < 0)
 570     {
 571         logf("Incorrect idxid: %d", idxid);
 572         return false;
 573     }
 574
 575 #ifdef HAVE_TC_RAMCACHE
 576     if (tc_stat.ramcache && use_ram)
 577     {
 578         if (hdr->indices[idxid].flag & FLAG_DELETED)
 579             return false;
 580
 581 # ifdef HAVE_DIRCACHE
 582         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 583             || is_dircache_intact())
 584 #endif
 585         {
 586             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 587             return true;
 588         }
 589     }
 590 #else
 591     (void)use_ram;
 592 #endif
 593
 594     if (masterfd < 0)
 595     {
 596         struct master_header tcmh;
 597
 598         localfd = true;
 599         masterfd = open_master_fd(&tcmh, false);
 600         if (masterfd < 0)
 601             return false;
 602     }
 603
 604     lseek(masterfd, idxid * sizeof(struct index_entry)
 605           + sizeof(struct master_header), SEEK_SET);
 606     if (ecread_index_entry(masterfd, idx)
 607         != sizeof(struct index_entry))
 608     {
 609         logf("read error #3");
 610         if (localfd)
 611             close(masterfd);
 612
 613         return false;
 614     }
 615
 616     if (localfd)
 617         close(masterfd);
 618
 619     if (idx->flag & FLAG_DELETED)
 620         return false;
 621
 622     return true;
 623 }
 624
 625 #ifndef __PCTOOL__
 626
 627 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 628 {
 629     /* We need to exclude all memory only flags & tags when writing to disk. */
 630     if (idx->flag & FLAG_DIRCACHE)
 631     {
 632         logf("memory only flags!");
 633         return false;
 634     }
 635
 636 #ifdef HAVE_TC_RAMCACHE
 637     /* Only update numeric data. Writing the whole index to RAM by memcpy
 638      * destroys dircache pointers!
 639      */
 640     if (tc_stat.ramcache)
 641     {
 642         int tag;
 643         struct index_entry *idx_ram = &hdr->indices[idxid];
 644
 645         for (tag = 0; tag < TAG_COUNT; tag++)
 646         {
 647             if (TAGCACHE_IS_NUMERIC(tag))
 648             {
 649                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 650             }
 651         }
 652
 653         /* Don't touch the dircache flag or attributes. */
 654         idx_ram->flag = (idx->flag & 0x0000ffff)
 655             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 656     }
 657 #endif
 658
 659     lseek(masterfd, idxid * sizeof(struct index_entry)
 660           + sizeof(struct master_header), SEEK_SET);
 661     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 662     {
 663         logf("write error #3");
 664         logf("idxid: %d", idxid);
 665         return false;
 666     }
 667
 668     return true;
 669 }
 670
 671 #endif /* !__PCTOOL__ */
 672
 673 static bool open_files(struct tagcache_search *tcs, int tag)
 674 {
 675     if (tcs->idxfd[tag] < 0)
 676     {
 677         char fn[MAX_PATH], path[MAX_PATH];
 678         const char *file;
 679
 680         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 681         file = get_user_file_path(fn, IS_FILE | NEED_WRITE, path, sizeof(path));
 682         tcs->idxfd[tag] = open(fn, O_RDONLY);
 683     }
 684
 685     if (tcs->idxfd[tag] < 0)
 686     {
 687         logf("File not open!");
 688         return false;
 689     }
 690
 691     return true;
 692 }
 693
 694 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 695                      int tag, char *buf, long size)
 696 {
 697     struct tagfile_entry tfe;
 698     long seek;
 699
 700     *buf = '\0';
 701
 702     if (TAGCACHE_IS_NUMERIC(tag))
 703         return false;
 704
 705     seek = idx->tag_seek[tag];
 706     if (seek < 0)
 707     {
 708         logf("Retrieve failed");
 709         return false;
 710     }
 711
 712 #ifdef HAVE_TC_RAMCACHE
 713     if (tcs->ramsearch)
 714     {
 715         struct tagfile_entry *ep;
 716
 717 # ifdef HAVE_DIRCACHE
 718         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 719             && is_dircache_intact())
 720         {
 721             dircache_copy_path((struct dircache_entry *)seek,
 722                                buf, size);
 723             return true;
 724         }
 725         else
 726 # endif
 727         if (tag != tag_filename)
 728         {
 729             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 730             strlcpy(buf, ep->tag_data, size);
 731
 732             return true;
 733         }
 734     }
 735 #endif
 736
 737     if (!open_files(tcs, tag))
 738         return false;
 739
 740     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 741     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 742         != sizeof(struct tagfile_entry))
 743     {
 744         logf("read error #5");
 745         return false;
 746     }
 747
 748     if (tfe.tag_length >= size)
 749     {
 750         logf("too small buffer");
 751         return false;
 752     }
 753
 754     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 755         tfe.tag_length)
 756     {
 757         logf("read error #6");
 758         return false;
 759     }
 760
 761     buf[tfe.tag_length] = '\0';
 762
 763     return true;
 764 }
 765
 766 static long check_virtual_tags(int tag, const struct index_entry *idx)
 767 {
 768     long data = 0;
 769
 770     switch (tag)
 771     {
 772         case tag_virt_length_sec:
 773             data = (idx->tag_seek[tag_length]/1000) % 60;
 774             break;
 775
 776         case tag_virt_length_min:
 777             data = (idx->tag_seek[tag_length]/1000) / 60;
 778             break;
 779
 780         case tag_virt_playtime_sec:
 781             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 782             break;
 783
 784         case tag_virt_playtime_min:
 785             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 786             break;
 787
 788         case tag_virt_autoscore:
 789             if (idx->tag_seek[tag_length] == 0
 790                 || idx->tag_seek[tag_playcount] == 0)
 791             {
 792                 data = 0;
 793             }
 794             else
 795             {
 796                 /* A straight calculus gives:
 797                      autoscore = 100 * playtime / length / playcout (1)
 798                    Now, consider the euclidian division of playtime by length:
 799                      playtime = alpha * length + beta
 800                    With:
 801                      0 <= beta < length
 802                    Now, (1) becomes:
 803                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 804                    Both terms should be small enough to avoid any overflow
 805                 */
 806                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 807                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 808                 data /=  idx->tag_seek[tag_playcount];
 809             }
 810             break;
 811
 812         /* How many commits before the file has been added to the DB. */
 813         case tag_virt_entryage:
 814             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 815             break;
 816
 817         default:
 818             data = idx->tag_seek[tag];
 819     }
 820
 821     return data;
 822 }
 823
 824 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 825 {
 826     struct index_entry idx;
 827
 828     if (!tc_stat.ready)
 829         return false;
 830
 831     if (!TAGCACHE_IS_NUMERIC(tag))
 832         return -1;
 833
 834     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 835         return -2;
 836
 837     return check_virtual_tags(tag, &idx);
 838 }
 839
 840 inline static bool str_ends_with(const char *str1, const char *str2)
 841 {
 842     int str_len = strlen(str1);
 843     int clause_len = strlen(str2);
 844
 845     if (clause_len > str_len)
 846         return false;
 847
 848     return !strcasecmp(&str1[str_len - clause_len], str2);
 849 }
 850
 851 inline static bool str_oneof(const char *str, const char *list)
 852 {
 853     const char *sep;
 854     int l, len = strlen(str);
 855
 856     while (*list)
 857     {
 858         sep = strchr(list, '|');
 859         l = sep ? (long)sep - (long)list : (int)strlen(list);
 860         if ((l==len) && !strncasecmp(str, list, len))
 861             return true;
 862         list += sep ? l + 1 : l;
 863     }
 864
 865     return false;
 866 }
 867
 868 static bool check_against_clause(long numeric, const char *str,
 869                                  const struct tagcache_search_clause *clause)
 870 {
 871     if (clause->numeric)
 872     {
 873         switch (clause->type)
 874         {
 875             case clause_is:
 876                 return numeric == clause->numeric_data;
 877             case clause_is_not:
 878                 return numeric != clause->numeric_data;
 879             case clause_gt:
 880                 return numeric > clause->numeric_data;
 881             case clause_gteq:
 882                 return numeric >= clause->numeric_data;
 883             case clause_lt:
 884                 return numeric < clause->numeric_data;
 885             case clause_lteq:
 886                 return numeric <= clause->numeric_data;
 887             default:
 888                 logf("Incorrect numeric tag: %d", clause->type);
 889         }
 890     }
 891     else
 892     {
 893         switch (clause->type)
 894         {
 895             case clause_is:
 896                 return !strcasecmp(clause->str, str);
 897             case clause_is_not:
 898                 return strcasecmp(clause->str, str);
 899             case clause_gt:
 900                 return 0>strcasecmp(clause->str, str);
 901             case clause_gteq:
 902                 return 0>=strcasecmp(clause->str, str);
 903             case clause_lt:
 904                 return 0<strcasecmp(clause->str, str);
 905             case clause_lteq:
 906                 return 0<=strcasecmp(clause->str, str);
 907             case clause_contains:
 908                 return (strcasestr(str, clause->str) != NULL);
 909             case clause_not_contains:
 910                 return (strcasestr(str, clause->str) == NULL);
 911             case clause_begins_with:
 912                 return (strcasestr(str, clause->str) == str);
 913             case clause_not_begins_with:
 914                 return (strcasestr(str, clause->str) != str);
 915             case clause_ends_with:
 916                 return str_ends_with(str, clause->str);
 917             case clause_not_ends_with:
 918                 return !str_ends_with(str, clause->str);
 919             case clause_oneof:
 920                 return str_oneof(str, clause->str);
 921
 922             default:
 923                 logf("Incorrect tag: %d", clause->type);
 924         }
 925     }
 926
 927     return false;
 928 }
 929
 930 static bool check_clauses(struct tagcache_search *tcs,
 931                           struct index_entry *idx,
 932                           struct tagcache_search_clause **clause, int count)
 933 {
 934     int i;
 935
 936 #ifdef HAVE_TC_RAMCACHE
 937     if (tcs->ramsearch)
 938     {
 939         /* Go through all conditional clauses. */
 940         for (i = 0; i < count; i++)
 941         {
 942             struct tagfile_entry *tfe;
 943             int seek;
 944             char buf[256];
 945             char *str = NULL;
 946
 947             seek = check_virtual_tags(clause[i]->tag, idx);
 948
 949             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 950             {
 951                 if (clause[i]->tag == tag_filename)
 952                 {
 953                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 954                     str = buf;
 955                 }
 956                 else
 957                 {
 958                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 959                     str = tfe->tag_data;
 960                 }
 961             }
 962
 963             if (!check_against_clause(seek, str, clause[i]))
 964                 return false;
 965         }
 966     }
 967     else
 968 #endif
 969     {
 970         /* Check for conditions. */
 971         for (i = 0; i < count; i++)
 972         {
 973             struct tagfile_entry tfe;
 974             int seek;
 975             char str[256];
 976
 977             seek = check_virtual_tags(clause[i]->tag, idx);
 978
 979             memset(str, 0, sizeof str);
 980             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 981             {
 982                 int fd = tcs->idxfd[clause[i]->tag];
 983                 lseek(fd, seek, SEEK_SET);
 984                 ecread_tagfile_entry(fd, &tfe);
 985                 if (tfe.tag_length >= (int)sizeof(str))
 986                 {
 987                     logf("Too long tag read!");
 988                     break ;
 989                 }
 990
 991                 read(fd, str, tfe.tag_length);
 992
 993                 /* Check if entry has been deleted. */
 994                 if (str[0] == '\0')
 995                     break;
 996             }
 997
 998             if (!check_against_clause(seek, str, clause[i]))
 999                 return false;
1000         }
1001     }
1002
1003     return true;
1004 }
1005
1006 bool tagcache_check_clauses(struct tagcache_search *tcs,
1007                             struct tagcache_search_clause **clause, int count)
1008 {
1009     struct index_entry idx;
1010
1011     if (count == 0)
1012         return true;
1013
1014     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1015         return false;
1016
1017     return check_clauses(tcs, &idx, clause, count);
1018 }
1019
1020 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1021 {
1022     int i;
1023
1024     /* If uniq buffer is not defined we must return true for search to work. */
1025     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1026         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1027     {
1028         return true;
1029     }
1030
1031     for (i = 0; i < tcs->unique_list_count; i++)
1032     {
1033         /* Return false if entry is found. */
1034         if (tcs->unique_list[i] == id)
1035             return false;
1036     }
1037
1038     if (tcs->unique_list_count < tcs->unique_list_capacity)
1039     {
1040         tcs->unique_list[i] = id;
1041         tcs->unique_list_count++;
1042     }
1043
1044     return true;
1045 }
1046
1047 static bool build_lookup_list(struct tagcache_search *tcs)
1048 {
1049     struct index_entry entry;
1050     int i, j;
1051
1052     tcs->seek_list_count = 0;
1053
1054 #ifdef HAVE_TC_RAMCACHE
1055     if (tcs->ramsearch
1056 # ifdef HAVE_DIRCACHE
1057         && (tcs->type != tag_filename || is_dircache_intact())
1058 # endif
1059         )
1060     {
1061         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1062         {
1063             struct tagcache_seeklist_entry *seeklist;
1064             struct index_entry *idx = &hdr->indices[i];
1065             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1066                 break ;
1067
1068             /* Skip deleted files. */
1069             if (idx->flag & FLAG_DELETED)
1070                 continue;
1071
1072             /* Go through all filters.. */
1073             for (j = 0; j < tcs->filter_count; j++)
1074             {
1075                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1076                 {
1077                     break ;
1078                 }
1079             }
1080
1081             if (j < tcs->filter_count)
1082                 continue ;
1083
1084             /* Check for conditions. */
1085             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1086                 continue;
1087
1088             /* Add to the seek list if not already in uniq buffer. */
1089             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1090                 continue;
1091
1092             /* Lets add it. */
1093             seeklist = &tcs->seeklist[tcs->seek_list_count];
1094             seeklist->seek = idx->tag_seek[tcs->type];
1095             seeklist->flag = idx->flag;
1096             seeklist->idx_id = i;
1097             tcs->seek_list_count++;
1098         }
1099
1100         tcs->seek_pos = i;
1101
1102         return tcs->seek_list_count > 0;
1103     }
1104 #endif
1105
1106     if (tcs->masterfd < 0)
1107     {
1108         struct master_header tcmh;
1109         tcs->masterfd = open_master_fd(&tcmh, false);
1110     }
1111
1112     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1113             sizeof(struct master_header), SEEK_SET);
1114
1115     while (ecread_index_entry(tcs->masterfd, &entry)
1116            == sizeof(struct index_entry))
1117     {
1118         struct tagcache_seeklist_entry *seeklist;
1119
1120         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1121             break ;
1122
1123         i = tcs->seek_pos;
1124         tcs->seek_pos++;
1125
1126         /* Check if entry has been deleted. */
1127         if (entry.flag & FLAG_DELETED)
1128             continue;
1129
1130         /* Go through all filters.. */
1131         for (j = 0; j < tcs->filter_count; j++)
1132         {
1133             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1134                 break ;
1135         }
1136
1137         if (j < tcs->filter_count)
1138             continue ;
1139
1140         /* Check for conditions. */
1141         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1142             continue;
1143
1144         /* Add to the seek list if not already in uniq buffer. */
1145         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1146             continue;
1147
1148         /* Lets add it. */
1149         seeklist = &tcs->seeklist[tcs->seek_list_count];
1150         seeklist->seek = entry.tag_seek[tcs->type];
1151         seeklist->flag = entry.flag;
1152         seeklist->idx_id = i;
1153         tcs->seek_list_count++;
1154
1155         yield();
1156     }
1157
1158     return tcs->seek_list_count > 0;
1159 }
1160
1161
1162 static void remove_files(void)
1163 {
1164     int i;
1165     char buf[MAX_PATH];
1166
1167     tc_stat.ready = false;
1168     tc_stat.ramcache = false;
1169     tc_stat.econ = false;
1170     remove(get_user_file_path(TAGCACHE_FILE_MASTER, NEED_WRITE|IS_FILE,
1171                               buf, sizeof(buf)));
1172     for (i = 0; i < TAG_COUNT; i++)
1173     {
1174         char buf2[MAX_PATH];
1175         if (TAGCACHE_IS_NUMERIC(i))
1176             continue;
1177
1178         /* database_%d.tcd -> database_0.tcd */
1179         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1180         remove(get_user_file_path(buf, NEED_WRITE | IS_FILE, buf2, sizeof(buf2)));
1181     }
1182 }
1183
1184
1185 static bool check_all_headers(void)
1186 {
1187     struct master_header myhdr;
1188     struct tagcache_header tch;
1189     int tag;
1190     int fd;
1191
1192     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1193         return false;
1194
1195     close(fd);
1196     if (myhdr.dirty)
1197     {
1198         logf("tagcache is dirty!");
1199         return false;
1200     }
1201
1202     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1203
1204     for (tag = 0; tag < TAG_COUNT; tag++)
1205     {
1206         if (TAGCACHE_IS_NUMERIC(tag))
1207             continue;
1208
1209         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1210             return false;
1211
1212         close(fd);
1213     }
1214
1215     return true;
1216 }
1217
1218 bool tagcache_is_busy(void)
1219 {
1220     return read_lock || write_lock;
1221 }
1222
1223 bool tagcache_search(struct tagcache_search *tcs, int tag)
1224 {
1225     struct tagcache_header tag_hdr;
1226     struct master_header   master_hdr;
1227     int i;
1228
1229     while (read_lock)
1230         sleep(1);
1231
1232     memset(tcs, 0, sizeof(struct tagcache_search));
1233     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1234         return false;
1235
1236     tcs->position = sizeof(struct tagcache_header);
1237     tcs->type = tag;
1238     tcs->seek_pos = 0;
1239     tcs->list_position = 0;
1240     tcs->seek_list_count = 0;
1241     tcs->filter_count = 0;
1242     tcs->masterfd = -1;
1243
1244     for (i = 0; i < TAG_COUNT; i++)
1245         tcs->idxfd[i] = -1;
1246
1247 #ifndef HAVE_TC_RAMCACHE
1248     tcs->ramsearch = false;
1249 #else
1250     tcs->ramsearch = tc_stat.ramcache;
1251     if (tcs->ramsearch)
1252     {
1253         tcs->entry_count = hdr->entry_count[tcs->type];
1254     }
1255     else
1256 #endif
1257     {
1258         /* Always open as R/W so we can pass tcs to functions that modify data also
1259          * without failing. */
1260         tcs->masterfd = open_master_fd(&master_hdr, true);
1261         if (tcs->masterfd < 0)
1262             return false;
1263
1264         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1265         {
1266             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1267             if (tcs->idxfd[tcs->type] < 0)
1268                 return false;
1269
1270             tcs->entry_count = tag_hdr.entry_count;
1271         }
1272         else
1273         {
1274             tcs->entry_count = master_hdr.tch.entry_count;
1275         }
1276     }
1277
1278     tcs->valid = true;
1279     tcs->initialized = true;
1280     write_lock++;
1281
1282     return true;
1283 }
1284
1285 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1286                                  void *buffer, long length)
1287 {
1288     tcs->unique_list = (unsigned long *)buffer;
1289     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1290     tcs->unique_list_count = 0;
1291 }
1292
1293 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1294                                 int tag, int seek)
1295 {
1296     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1297         return false;
1298
1299     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1300         return false;
1301
1302     tcs->filter_tag[tcs->filter_count] = tag;
1303     tcs->filter_seek[tcs->filter_count] = seek;
1304     tcs->filter_count++;
1305
1306     return true;
1307 }
1308
1309 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1310                                 struct tagcache_search_clause *clause)
1311 {
1312     int i;
1313
1314     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1315     {
1316         logf("Too many clauses");
1317         return false;
1318     }
1319
1320     /* Check if there is already a similar filter in present (filters are
1321      * much faster than clauses).
1322      */
1323     for (i = 0; i < tcs->filter_count; i++)
1324     {
1325         if (tcs->filter_tag[i] == clause->tag)
1326             return true;
1327     }
1328
1329     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1330     {
1331         char buf[MAX_PATH], path[MAX_PATH];
1332         const char *file;
1333         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1334         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
1335         tcs->idxfd[clause->tag] = open(file, O_RDONLY);
1336     }
1337
1338     tcs->clause[tcs->clause_count] = clause;
1339     tcs->clause_count++;
1340
1341     return true;
1342 }
1343
1344 static bool get_next(struct tagcache_search *tcs)
1345 {
1346     static char buf[TAG_MAXLEN+32];
1347     struct tagfile_entry entry;
1348     long flag = 0;
1349
1350     if (!tcs->valid || !tc_stat.ready)
1351         return false;
1352
1353     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1354 #ifdef HAVE_TC_RAMCACHE
1355         && !tcs->ramsearch
1356 #endif
1357         )
1358         return false;
1359
1360     /* Relative fetch. */
1361     if (tcs->filter_count > 0 || tcs->clause_count > 0
1362         || TAGCACHE_IS_NUMERIC(tcs->type)
1363 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1364         /* We need to retrieve flag status for dircache. */
1365         || (tcs->ramsearch && tcs->type == tag_filename)
1366 #endif
1367         )
1368     {
1369         struct tagcache_seeklist_entry *seeklist;
1370
1371         /* Check for end of list. */
1372         if (tcs->list_position == tcs->seek_list_count)
1373         {
1374             tcs->list_position = 0;
1375
1376             /* Try to fetch more. */
1377             if (!build_lookup_list(tcs))
1378             {
1379                 tcs->valid = false;
1380                 return false;
1381             }
1382         }
1383
1384         seeklist = &tcs->seeklist[tcs->list_position];
1385         flag = seeklist->flag;
1386         tcs->position = seeklist->seek;
1387         tcs->idx_id = seeklist->idx_id;
1388         tcs->list_position++;
1389     }
1390     else
1391     {
1392         if (tcs->entry_count == 0)
1393         {
1394             tcs->valid = false;
1395             return false;
1396         }
1397
1398         tcs->entry_count--;
1399     }
1400
1401     tcs->result_seek = tcs->position;
1402
1403     if (TAGCACHE_IS_NUMERIC(tcs->type))
1404     {
1405         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1406         tcs->result = buf;
1407         tcs->result_len = strlen(buf) + 1;
1408         return true;
1409     }
1410
1411     /* Direct fetch. */
1412 #ifdef HAVE_TC_RAMCACHE
1413     if (tcs->ramsearch)
1414     {
1415
1416 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1417         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1418             && is_dircache_intact())
1419         {
1420             dircache_copy_path((struct dircache_entry *)tcs->position,
1421                                buf, sizeof buf);
1422             tcs->result = buf;
1423             tcs->result_len = strlen(buf) + 1;
1424             tcs->ramresult = false;
1425
1426             return true;
1427         }
1428         else
1429 #endif
1430         if (tcs->type != tag_filename)
1431         {
1432             struct tagfile_entry *ep;
1433
1434             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1435             tcs->result = ep->tag_data;
1436             tcs->result_len = strlen(tcs->result) + 1;
1437             tcs->idx_id = ep->idx_id;
1438             tcs->ramresult = true;
1439
1440             /* Increase position for the next run. This may get overwritten. */
1441             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1442
1443             return true;
1444         }
1445     }
1446 #endif
1447
1448     if (!open_files(tcs, tcs->type))
1449     {
1450         tcs->valid = false;
1451         return false;
1452     }
1453
1454     /* Seek stream to the correct position and continue to direct fetch. */
1455     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1456
1457     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1458     {
1459         logf("read error #5");
1460         tcs->valid = false;
1461         return false;
1462     }
1463
1464     if (entry.tag_length > (long)sizeof(buf))
1465     {
1466         tcs->valid = false;
1467         logf("too long tag #2");
1468         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1469         return false;
1470     }
1471
1472     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1473     {
1474         tcs->valid = false;
1475         logf("read error #4");
1476         return false;
1477     }
1478
1479     /**
1480      Update the position for the next read (this may be overridden
1481      if filters or clauses are being used).
1482      */
1483     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1484     tcs->result = buf;
1485     tcs->result_len = strlen(tcs->result) + 1;
1486     tcs->idx_id = entry.idx_id;
1487     tcs->ramresult = false;
1488
1489     return true;
1490 }
1491
1492 bool tagcache_get_next(struct tagcache_search *tcs)
1493 {
1494     while (get_next(tcs))
1495     {
1496         if (tcs->result_len > 1)
1497             return true;
1498     }
1499
1500     return false;
1501 }
1502
1503 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1504                        int tag, char *buf, long size)
1505 {
1506     struct index_entry idx;
1507
1508     *buf = '\0';
1509     if (!get_index(tcs->masterfd, idxid, &idx, true))
1510         return false;
1511
1512     return retrieve(tcs, &idx, tag, buf, size);
1513 }
1514
1515 static bool update_master_header(void)
1516 {
1517     struct master_header myhdr;
1518     int fd;
1519
1520     if (!tc_stat.ready)
1521         return false;
1522
1523     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1524         return false;
1525
1526     myhdr.serial = current_tcmh.serial;
1527     myhdr.commitid = current_tcmh.commitid;
1528     myhdr.dirty = current_tcmh.dirty;
1529
1530     /* Write it back */
1531     lseek(fd, 0, SEEK_SET);
1532     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1533     close(fd);
1534
1535 #ifdef HAVE_TC_RAMCACHE
1536     if (hdr)
1537     {
1538         hdr->h.serial = current_tcmh.serial;
1539         hdr->h.commitid = current_tcmh.commitid;
1540         hdr->h.dirty = current_tcmh.dirty;
1541     }
1542 #endif
1543
1544     return true;
1545 }
1546
1547 #if 0
1548
1549 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1550 {
1551     struct tagentry *entry;
1552
1553     if (tcs->type != tag_title)
1554         return ;
1555
1556     /* We will need reserve buffer for this. */
1557     if (tcs->ramcache)
1558     {
1559         struct tagfile_entry *ep;
1560
1561         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1562         tcs->seek_list[tcs->seek_list_count];
1563     }
1564
1565     entry = find_entry_ram();
1566
1567 }
1568 #endif
1569
1570 void tagcache_search_finish(struct tagcache_search *tcs)
1571 {
1572     int i;
1573
1574     if (!tcs->initialized)
1575         return;
1576
1577     if (tcs->masterfd >= 0)
1578     {
1579         close(tcs->masterfd);
1580         tcs->masterfd = -1;
1581     }
1582
1583     for (i = 0; i < TAG_COUNT; i++)
1584     {
1585         if (tcs->idxfd[i] >= 0)
1586         {
1587             close(tcs->idxfd[i]);
1588             tcs->idxfd[i] = -1;
1589         }
1590     }
1591
1592     tcs->ramsearch = false;
1593     tcs->valid = false;
1594     tcs->initialized = 0;
1595     if (write_lock > 0)
1596         write_lock--;
1597 }
1598
1599 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1600 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1601 {
1602     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1603 }
1604
1605 static long get_tag_numeric(const struct index_entry *entry, int tag)
1606 {
1607     return check_virtual_tags(tag, entry);
1608 }
1609
1610 static char* get_tag_string(const struct index_entry *entry, int tag)
1611 {
1612     char* s = get_tag(entry, tag)->tag_data;
1613     return strcmp(s, UNTAGGED) ? s : NULL;
1614 }
1615
1616 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1617 {
1618     struct index_entry *entry;
1619     int idx_id;
1620
1621     if (!tc_stat.ready || !tc_stat.ramcache)
1622         return false;
1623
1624     /* Find the corresponding entry in tagcache. */
1625     idx_id = find_entry_ram(filename, NULL);
1626     if (idx_id < 0)
1627         return false;
1628
1629     entry = &hdr->indices[idx_id];
1630
1631     memset(id3, 0, sizeof(struct mp3entry));
1632
1633     id3->title        = get_tag_string(entry, tag_title);
1634     id3->artist       = get_tag_string(entry, tag_artist);
1635     id3->album        = get_tag_string(entry, tag_album);
1636     id3->genre_string = get_tag_string(entry, tag_genre);
1637     id3->composer     = get_tag_string(entry, tag_composer);
1638     id3->comment      = get_tag_string(entry, tag_comment);
1639     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1640     id3->grouping     = get_tag_string(entry, tag_grouping);
1641
1642     id3->length     = get_tag_numeric(entry, tag_length);
1643     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1644     id3->rating     = get_tag_numeric(entry, tag_rating);
1645     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1646     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1647     id3->year       = get_tag_numeric(entry, tag_year);
1648
1649     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1650     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1651     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1652     if (id3->bitrate == 0)
1653         id3->bitrate = 1;
1654
1655     return true;
1656 }
1657 #endif
1658
1659 static inline void write_item(const char *item)
1660 {
1661     int len = strlen(item) + 1;
1662
1663     data_size += len;
1664     write(cachefd, item, len);
1665 }
1666
1667 static int check_if_empty(char **tag)
1668 {
1669     int length;
1670
1671     if (*tag == NULL || **tag == '\0')
1672     {
1673         *tag = UNTAGGED;
1674         return sizeof(UNTAGGED); /* Tag length */
1675     }
1676
1677     length = strlen(*tag);
1678     if (length > TAG_MAXLEN)
1679     {
1680         logf("over length tag: %s", *tag);
1681         length = TAG_MAXLEN;
1682         (*tag)[length] = '\0';
1683     }
1684
1685     return length + 1;
1686 }
1687
1688 #define ADD_TAG(entry,tag,data) \
1689     /* Adding tag */ \
1690     entry.tag_offset[tag] = offset; \
1691     entry.tag_length[tag] = check_if_empty(data); \
1692     offset += entry.tag_length[tag]
1693 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1694  * idea, as it uses lots of stack and is called from a recursive function
1695  * (check_dir).
1696  */
1697 static void __attribute__ ((noinline)) add_tagcache(char *path,
1698                                                     unsigned long mtime
1699 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1700                                                     ,const struct dircache_entry *dc
1701 #endif
1702                                                    )
1703 {
1704     struct mp3entry id3;
1705     struct temp_file_entry entry;
1706     bool ret;
1707     int fd;
1708     int idx_id = -1;
1709     char tracknumfix[3];
1710     int offset = 0;
1711     int path_length = strlen(path);
1712     bool has_albumartist;
1713     bool has_grouping;
1714
1715 #ifdef SIMULATOR
1716     /* Crude logging for the sim - to aid in debugging */
1717     int logfd = open(ROCKBOX_DIR "/database.log",
1718                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1719     if (logfd >= 0) {
1720         write(logfd, path, strlen(path));
1721         write(logfd, "\n", 1);
1722         close(logfd);
1723     }
1724 #endif
1725
1726     if (cachefd < 0)
1727         return ;
1728
1729     /* Check for overlength file path. */
1730     if (path_length > TAG_MAXLEN)
1731     {
1732         /* Path can't be shortened. */
1733         logf("Too long path: %s", path);
1734         return ;
1735     }
1736
1737     /* Check if the file is supported. */
1738     if (probe_file_format(path) == AFMT_UNKNOWN)
1739         return ;
1740
1741     /* Check if the file is already cached. */
1742 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1743     if (tc_stat.ramcache && is_dircache_intact())
1744     {
1745         idx_id = find_entry_ram(path, dc);
1746     }
1747 #endif
1748
1749     /* Be sure the entry doesn't exist. */
1750     if (filenametag_fd >= 0 && idx_id < 0)
1751         idx_id = find_entry_disk(path, false);
1752
1753     /* Check if file has been modified. */
1754     if (idx_id >= 0)
1755     {
1756         struct index_entry idx;
1757
1758         /* TODO: Mark that the index exists (for fast reverse scan) */
1759         //found_idx[idx_id/8] |= idx_id%8;
1760
1761         if (!get_index(-1, idx_id, &idx, true))
1762         {
1763             logf("failed to retrieve index entry");
1764             return ;
1765         }
1766
1767         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1768         {
1769             /* No changes to file. */
1770             return ;
1771         }
1772
1773         /* Metadata might have been changed. Delete the entry. */
1774         logf("Re-adding: %s", path);
1775         if (!delete_entry(idx_id))
1776         {
1777             logf("delete_entry failed: %d", idx_id);
1778             return ;
1779         }
1780     }
1781
1782     fd = open(path, O_RDONLY);
1783     if (fd < 0)
1784     {
1785         logf("open fail: %s", path);
1786         return ;
1787     }
1788
1789     memset(&id3, 0, sizeof(struct mp3entry));
1790     memset(&entry, 0, sizeof(struct temp_file_entry));
1791     memset(&tracknumfix, 0, sizeof(tracknumfix));
1792     ret = get_metadata(&id3, fd, path);
1793     close(fd);
1794
1795     if (!ret)
1796         return ;
1797
1798     logf("-> %s", path);
1799
1800     /* Generate track number if missing. */
1801     if (id3.tracknum <= 0)
1802     {
1803         const char *p = strrchr(path, '.');
1804
1805         if (p == NULL)
1806             p = &path[strlen(path)-1];
1807
1808         while (*p != '/')
1809         {
1810             if (isdigit(*p) && isdigit(*(p-1)))
1811             {
1812                 tracknumfix[1] = *p--;
1813                 tracknumfix[0] = *p;
1814                 break;
1815             }
1816             p--;
1817         }
1818
1819         if (tracknumfix[0] != '\0')
1820         {
1821             id3.tracknum = atoi(tracknumfix);
1822             /* Set a flag to indicate track number has been generated. */
1823             entry.flag |= FLAG_TRKNUMGEN;
1824         }
1825         else
1826         {
1827             /* Unable to generate track number. */
1828             id3.tracknum = -1;
1829         }
1830     }
1831
1832     /* Numeric tags */
1833     entry.tag_offset[tag_year] = id3.year;
1834     entry.tag_offset[tag_discnumber] = id3.discnum;
1835     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1836     entry.tag_offset[tag_length] = id3.length;
1837     entry.tag_offset[tag_bitrate] = id3.bitrate;
1838     entry.tag_offset[tag_mtime] = mtime;
1839
1840     /* String tags. */
1841     has_albumartist = id3.albumartist != NULL
1842         && strlen(id3.albumartist) > 0;
1843     has_grouping = id3.grouping != NULL
1844         && strlen(id3.grouping) > 0;
1845
1846     ADD_TAG(entry, tag_filename, &path);
1847     ADD_TAG(entry, tag_title, &id3.title);
1848     ADD_TAG(entry, tag_artist, &id3.artist);
1849     ADD_TAG(entry, tag_album, &id3.album);
1850     ADD_TAG(entry, tag_genre, &id3.genre_string);
1851     ADD_TAG(entry, tag_composer, &id3.composer);
1852     ADD_TAG(entry, tag_comment, &id3.comment);
1853     if (has_albumartist)
1854     {
1855         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1856     }
1857     else
1858     {
1859         ADD_TAG(entry, tag_albumartist, &id3.artist);
1860     }
1861     if (has_grouping)
1862     {
1863         ADD_TAG(entry, tag_grouping, &id3.grouping);
1864     }
1865     else
1866     {
1867         ADD_TAG(entry, tag_grouping, &id3.title);
1868     }
1869     entry.data_length = offset;
1870
1871     /* Write the header */
1872     write(cachefd, &entry, sizeof(struct temp_file_entry));
1873
1874     /* And tags also... Correct order is critical */
1875     write_item(path);
1876     write_item(id3.title);
1877     write_item(id3.artist);
1878     write_item(id3.album);
1879     write_item(id3.genre_string);
1880     write_item(id3.composer);
1881     write_item(id3.comment);
1882     if (has_albumartist)
1883     {
1884         write_item(id3.albumartist);
1885     }
1886     else
1887     {
1888         write_item(id3.artist);
1889     }
1890     if (has_grouping)
1891     {
1892         write_item(id3.grouping);
1893     }
1894     else
1895     {
1896         write_item(id3.title);
1897     }
1898     total_entry_count++;
1899 }
1900
1901 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1902 {
1903     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1904     int len = strlen(str)+1;
1905     int i;
1906     unsigned crc32;
1907     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1908     char buf[TAG_MAXLEN+32];
1909
1910     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1911         buf[i] = tolower(str[i]);
1912     buf[i] = '\0';
1913
1914     crc32 = crc_32(buf, i, 0xffffffff);
1915
1916     if (unique)
1917     {
1918         /* Check if the crc does not exist -> entry does not exist for sure. */
1919         for (i = 0; i < tempbufidx; i++)
1920         {
1921             if (crcbuf[-i] != crc32)
1922                 continue;
1923
1924             if (!strcasecmp(str, index[i].str))
1925             {
1926                 if (id < 0 || id >= lookup_buffer_depth)
1927                 {
1928                     logf("lookup buf overf.: %d", id);
1929                     return false;
1930                 }
1931
1932                 lookup[id] = &index[i];
1933                 return true;
1934             }
1935         }
1936     }
1937
1938     /* Insert to CRC buffer. */
1939     crcbuf[-tempbufidx] = crc32;
1940     tempbuf_left -= 4;
1941
1942     /* Insert it to the buffer. */
1943     tempbuf_left -= len;
1944     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1945         return false;
1946
1947     if (id >= lookup_buffer_depth)
1948     {
1949         logf("lookup buf overf. #2: %d", id);
1950         return false;
1951     }
1952
1953     if (id >= 0)
1954     {
1955         lookup[id] = &index[tempbufidx];
1956         index[tempbufidx].idlist.id = id;
1957     }
1958     else
1959         index[tempbufidx].idlist.id = -1;
1960
1961     index[tempbufidx].idlist.next = NULL;
1962     index[tempbufidx].idx_id = idx_id;
1963     index[tempbufidx].seek = -1;
1964     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1965     memcpy(index[tempbufidx].str, str, len);
1966     tempbuf_pos += len;
1967     tempbufidx++;
1968
1969     return true;
1970 }
1971
1972 static int compare(const void *p1, const void *p2)
1973 {
1974     do_timed_yield();
1975
1976     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1977     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1978
1979     if (strcmp(e1->str, UNTAGGED) == 0)
1980     {
1981         if (strcmp(e2->str, UNTAGGED) == 0)
1982             return 0;
1983         return -1;
1984     }
1985     else if (strcmp(e2->str, UNTAGGED) == 0)
1986         return 1;
1987
1988     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1989 }
1990
1991 static int tempbuf_sort(int fd)
1992 {
1993     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1994     struct tagfile_entry fe;
1995     int i;
1996     int length;
1997
1998     /* Generate reverse lookup entries. */
1999     for (i = 0; i < lookup_buffer_depth; i++)
2000     {
2001         struct tempbuf_id_list *idlist;
2002
2003         if (!lookup[i])
2004             continue;
2005
2006         if (lookup[i]->idlist.id == i)
2007             continue;
2008
2009         idlist = &lookup[i]->idlist;
2010         while (idlist->next != NULL)
2011             idlist = idlist->next;
2012
2013         tempbuf_left -= sizeof(struct tempbuf_id_list);
2014         if (tempbuf_left - 4 < 0)
2015             return -1;
2016
2017         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2018         if (tempbuf_pos & 0x03)
2019         {
2020             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2021             tempbuf_left -= 3;
2022             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2023         }
2024         tempbuf_pos += sizeof(struct tempbuf_id_list);
2025
2026         idlist = idlist->next;
2027         idlist->id = i;
2028         idlist->next = NULL;
2029
2030         do_timed_yield();
2031     }
2032
2033     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2034     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2035
2036     for (i = 0; i < tempbufidx; i++)
2037     {
2038         struct tempbuf_id_list *idlist = &index[i].idlist;
2039
2040         /* Fix the lookup list. */
2041         while (idlist != NULL)
2042         {
2043             if (idlist->id >= 0)
2044                 lookup[idlist->id] = &index[i];
2045             idlist = idlist->next;
2046         }
2047
2048         index[i].seek = lseek(fd, 0, SEEK_CUR);
2049         length = strlen(index[i].str) + 1;
2050         fe.tag_length = length;
2051         fe.idx_id = index[i].idx_id;
2052
2053         /* Check the chunk alignment. */
2054         if ((fe.tag_length + sizeof(struct tagfile_entry))
2055             % TAGFILE_ENTRY_CHUNK_LENGTH)
2056         {
2057             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2058                 ((fe.tag_length + sizeof(struct tagfile_entry))
2059                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2060         }
2061
2062 #ifdef TAGCACHE_STRICT_ALIGN
2063         /* Make sure the entry is long aligned. */
2064         if (index[i].seek & 0x03)
2065         {
2066             logf("tempbuf_sort: alignment error!");
2067             return -3;
2068         }
2069 #endif
2070
2071         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2072             sizeof(struct tagfile_entry))
2073         {
2074             logf("tempbuf_sort: write error #1");
2075             return -1;
2076         }
2077
2078         if (write(fd, index[i].str, length) != length)
2079         {
2080             logf("tempbuf_sort: write error #2");
2081             return -2;
2082         }
2083
2084         /* Write some padding. */
2085         if (fe.tag_length - length > 0)
2086             write(fd, "XXXXXXXX", fe.tag_length - length);
2087     }
2088
2089     return i;
2090 }
2091
2092 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2093 {
2094     if (id < 0 || id >= lookup_buffer_depth)
2095         return NULL;
2096
2097     return lookup[id];
2098 }
2099
2100
2101 inline static int tempbuf_find_location(int id)
2102 {
2103     struct tempbuf_searchidx *entry;
2104
2105     entry = tempbuf_locate(id);
2106     if (entry == NULL)
2107         return -1;
2108
2109     return entry->seek;
2110 }
2111
2112 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2113 {
2114     struct master_header tcmh;
2115     struct index_entry idx;
2116     int masterfd;
2117     int masterfd_pos;
2118     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2119     int max_entries;
2120     int entries_processed = 0;
2121     int i, j;
2122     char buf[TAG_MAXLEN];
2123
2124     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2125
2126     logf("Building numeric indices...");
2127     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2128
2129     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2130         return false;
2131
2132     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2133                          SEEK_CUR);
2134     if (masterfd_pos == filesize(masterfd))
2135     {
2136         logf("we can't append!");
2137         close(masterfd);
2138         return false;
2139     }
2140
2141     while (entries_processed < h->entry_count)
2142     {
2143         int count = MIN(h->entry_count - entries_processed, max_entries);
2144
2145         /* Read in as many entries as possible. */
2146         for (i = 0; i < count; i++)
2147         {
2148             struct temp_file_entry *tfe = &entrybuf[i];
2149             int datastart;
2150
2151             /* Read in numeric data. */
2152             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2153                 sizeof(struct temp_file_entry))
2154             {
2155                 logf("read fail #1");
2156                 close(masterfd);
2157                 return false;
2158             }
2159
2160             datastart = lseek(tmpfd, 0, SEEK_CUR);
2161
2162             /**
2163              * Read string data from the following tags:
2164              * - tag_filename
2165              * - tag_artist
2166              * - tag_album
2167              * - tag_title
2168              *
2169              * A crc32 hash is calculated from the read data
2170              * and stored back to the data offset field kept in memory.
2171              */
2172 #define tmpdb_read_string_tag(tag) \
2173     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2174     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2175     { \
2176         logf("read fail: buffer overflow"); \
2177         close(masterfd); \
2178         return false; \
2179     } \
2180     \
2181     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2182         tfe->tag_length[tag]) \
2183     { \
2184         logf("read fail #2"); \
2185         close(masterfd); \
2186         return false; \
2187     } \
2188     \
2189     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2190     lseek(tmpfd, datastart, SEEK_SET)
2191
2192             tmpdb_read_string_tag(tag_filename);
2193             tmpdb_read_string_tag(tag_artist);
2194             tmpdb_read_string_tag(tag_album);
2195             tmpdb_read_string_tag(tag_title);
2196
2197             /* Seek to the end of the string data. */
2198             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2199         }
2200
2201         /* Backup the master index position. */
2202         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2203         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2204
2205         /* Check if we can resurrect some deleted runtime statistics data. */
2206         for (i = 0; i < tcmh.tch.entry_count; i++)
2207         {
2208             /* Read the index entry. */
2209             if (ecread_index_entry(masterfd, &idx)
2210                 != sizeof(struct index_entry))
2211             {
2212                 logf("read fail #3");
2213                 close(masterfd);
2214                 return false;
2215             }
2216
2217             /**
2218              * Skip unless the entry is marked as being deleted
2219              * or the data has already been resurrected.
2220              */
2221             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2222                 continue;
2223
2224             /* Now try to match the entry. */
2225             /**
2226              * To succesfully match a song, the following conditions
2227              * must apply:
2228              *
2229              * For numeric fields: tag_length
2230              * - Full identical match is required
2231              *
2232              * If tag_filename matches, no further checking necessary.
2233              *
2234              * For string hashes: tag_artist, tag_album, tag_title
2235              * - Two of these must match
2236              */
2237             for (j = 0; j < count; j++)
2238             {
2239                 struct temp_file_entry *tfe = &entrybuf[j];
2240
2241                 /* Try to match numeric fields first. */
2242                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2243                     continue;
2244
2245                 /* Now it's time to do the hash matching. */
2246                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2247                 {
2248                     int match_count = 0;
2249
2250                     /* No filename match, check if we can match two other tags. */
2251 #define tmpdb_match(tag) \
2252     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2253         match_count++
2254
2255                     tmpdb_match(tag_artist);
2256                     tmpdb_match(tag_album);
2257                     tmpdb_match(tag_title);
2258
2259                     if (match_count < 2)
2260                     {
2261                         /* Still no match found, give up. */
2262                         continue;
2263                     }
2264                 }
2265
2266                 /* A match found, now copy & resurrect the statistical data. */
2267 #define tmpdb_copy_tag(tag) \
2268     tfe->tag_offset[tag] = idx.tag_seek[tag]
2269
2270                 tmpdb_copy_tag(tag_playcount);
2271                 tmpdb_copy_tag(tag_rating);
2272                 tmpdb_copy_tag(tag_playtime);
2273                 tmpdb_copy_tag(tag_lastplayed);
2274                 tmpdb_copy_tag(tag_commitid);
2275
2276                 /* Avoid processing this entry again. */
2277                 idx.flag |= FLAG_RESURRECTED;
2278
2279                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2280                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2281                 {
2282                     logf("masterfd writeback fail #1");
2283                     close(masterfd);
2284                     return false;
2285                 }
2286
2287                 logf("Entry resurrected");
2288             }
2289         }
2290
2291
2292         /* Restore the master index position. */
2293         lseek(masterfd, masterfd_pos, SEEK_SET);
2294
2295         /* Commit the data to the index. */
2296         for (i = 0; i < count; i++)
2297         {
2298             int loc = lseek(masterfd, 0, SEEK_CUR);
2299
2300             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2301             {
2302                 logf("read fail #3");
2303                 close(masterfd);
2304                 return false;
2305             }
2306
2307             for (j = 0; j < TAG_COUNT; j++)
2308             {
2309                 if (!TAGCACHE_IS_NUMERIC(j))
2310                     continue;
2311
2312                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2313             }
2314             idx.flag = entrybuf[i].flag;
2315
2316             if (idx.tag_seek[tag_commitid])
2317             {
2318                 /* Data has been resurrected. */
2319                 idx.flag |= FLAG_DIRTYNUM;
2320             }
2321             else if (tc_stat.ready && current_tcmh.commitid > 0)
2322             {
2323                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2324                 idx.flag |= FLAG_DIRTYNUM;
2325             }
2326
2327             /* Write back the updated index. */
2328             lseek(masterfd, loc, SEEK_SET);
2329             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2330             {
2331                 logf("write fail");
2332                 close(masterfd);
2333                 return false;
2334             }
2335         }
2336
2337         entries_processed += count;
2338         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2339     }
2340
2341     close(masterfd);
2342
2343     return true;
2344 }
2345
2346 /**
2347  * Return values:
2348  *     > 0   success
2349  *    == 0   temporary failure
2350  *     < 0   fatal error
2351  */
2352 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2353 {
2354     int i;
2355     struct tagcache_header tch;
2356     struct master_header   tcmh;
2357     struct index_entry idxbuf[IDX_BUF_DEPTH];
2358     int idxbuf_pos;
2359     char buf[TAG_MAXLEN+32], path[MAX_PATH];
2360     const char *file;
2361     int fd = -1, masterfd;
2362     bool error = false;
2363     int init;
2364     int masterfd_pos;
2365
2366     logf("Building index: %d", index_type);
2367
2368     /* Check the number of entries we need to allocate ram for. */
2369     commit_entry_count = h->entry_count + 1;
2370
2371     masterfd = open_master_fd(&tcmh, false);
2372     if (masterfd >= 0)
2373     {
2374         commit_entry_count += tcmh.tch.entry_count;
2375         close(masterfd);
2376     }
2377     else
2378         remove_files(); /* Just to be sure we are clean. */
2379
2380     /* Open the index file, which contains the tag names. */
2381     fd = open_tag_fd(&tch, index_type, true);
2382     if (fd >= 0)
2383     {
2384         logf("tch.datasize=%ld", tch.datasize);
2385         lookup_buffer_depth = 1 +
2386         /* First part */ commit_entry_count +
2387         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2388     }
2389     else
2390     {
2391         lookup_buffer_depth = 1 +
2392         /* First part */ commit_entry_count +
2393         /* Second part */ 0;
2394     }
2395
2396     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2397     logf("commit_entry_count=%ld", commit_entry_count);
2398
2399     /* Allocate buffer for all index entries from both old and new
2400      * tag files. */
2401     tempbufidx = 0;
2402     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2403
2404     /* Allocate lookup buffer. The first portion of commit_entry_count
2405      * contains the new tags in the temporary file and the second
2406      * part for locating entries already in the db.
2407      *
2408      *  New tags  Old tags
2409      * +---------+---------------------------+
2410      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2411      * +---------+---------------------------+
2412      *
2413      * Old tags are inserted to a temporary buffer with position:
2414      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2415      * And new tags with index:
2416      *     tempbuf_insert(idx, ...);
2417      *
2418      * The buffer is sorted and written into tag file:
2419      *     tempbuf_sort(...);
2420      * leaving master index locations messed up.
2421      *
2422      * That is fixed using the lookup buffer for old tags:
2423      *     new_seek = tempbuf_find_location(old_seek, ...);
2424      * and for new tags:
2425      *     new_seek = tempbuf_find_location(idx);
2426      */
2427     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2428     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2429     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2430
2431     /* And calculate the remaining data space used mainly for storing
2432      * tag data (strings). */
2433     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2434     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2435     {
2436         logf("Buffer way too small!");
2437         return 0;
2438     }
2439
2440     if (fd >= 0)
2441     {
2442         /**
2443          * If tag file contains unique tags (sorted index), we will load
2444          * it entirely into memory so we can resort it later for use with
2445          * chunked browsing.
2446          */
2447         if (TAGCACHE_IS_SORTED(index_type))
2448         {
2449             logf("loading tags...");
2450             for (i = 0; i < tch.entry_count; i++)
2451             {
2452                 struct tagfile_entry entry;
2453                 int loc = lseek(fd, 0, SEEK_CUR);
2454                 bool ret;
2455
2456                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2457                 {
2458                     logf("read error #7");
2459                     close(fd);
2460                     return -2;
2461                 }
2462
2463                 if (entry.tag_length >= (int)sizeof(buf))
2464                 {
2465                     logf("too long tag #3");
2466                     close(fd);
2467                     return -2;
2468                 }
2469
2470                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2471                 {
2472                     logf("read error #8");
2473                     close(fd);
2474                     return -2;
2475                 }
2476
2477                 /* Skip deleted entries. */
2478                 if (buf[0] == '\0')
2479                     continue;
2480
2481                 /**
2482                  * Save the tag and tag id in the memory buffer. Tag id
2483                  * is saved so we can later reindex the master lookup
2484                  * table when the index gets resorted.
2485                  */
2486                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2487                                      + commit_entry_count, entry.idx_id,
2488                                      TAGCACHE_IS_UNIQUE(index_type));
2489                 if (!ret)
2490                 {
2491                     close(fd);
2492                     return -3;
2493                 }
2494                 do_timed_yield();
2495             }
2496             logf("done");
2497         }
2498         else
2499             tempbufidx = tch.entry_count;
2500     }
2501     else
2502     {
2503         /**
2504          * Creating new index file to store the tags. No need to preload
2505          * anything whether the index type is sorted or not.
2506          */
2507         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2508         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
2509         fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2510         if (fd < 0)
2511         {
2512             logf("%s open fail", buf);
2513             return -2;
2514         }
2515
2516         tch.magic = TAGCACHE_MAGIC;
2517         tch.entry_count = 0;
2518         tch.datasize = 0;
2519
2520         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2521             != sizeof(struct tagcache_header))
2522         {
2523             logf("header write failed");
2524             close(fd);
2525             return -2;
2526         }
2527     }
2528
2529     file = get_user_file_path(TAGCACHE_FILE_MASTER,
2530                                             IS_FILE|NEED_WRITE,
2531                                             buf, sizeof(buf));
2532     /* Loading the tag lookup file as "master file". */
2533     logf("Loading index file");
2534     masterfd = open(file, O_RDWR);
2535
2536     if (masterfd < 0)
2537     {
2538         logf("Creating new DB");
2539         masterfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2540
2541         if (masterfd < 0)
2542         {
2543             logf("Failure to create index file (%s)", file);
2544             close(fd);
2545             return -2;
2546         }
2547
2548         /* Write the header (write real values later). */
2549         memset(&tcmh, 0, sizeof(struct master_header));
2550         tcmh.tch = *h;
2551         tcmh.tch.entry_count = 0;
2552         tcmh.tch.datasize = 0;
2553         tcmh.dirty = true;
2554         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2555         init = true;
2556         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2557     }
2558     else
2559     {
2560         /**
2561          * Master file already exists so we need to process the current
2562          * file first.
2563          */
2564         init = false;
2565
2566         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2567             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2568         {
2569             logf("header error");
2570             close(fd);
2571             close(masterfd);
2572             return -2;
2573         }
2574
2575         /**
2576          * If we reach end of the master file, we need to expand it to
2577          * hold new tags. If the current index is not sorted, we can
2578          * simply append new data to end of the file.
2579          * However, if the index is sorted, we need to update all tag
2580          * pointers in the master file for the current index.
2581          */
2582         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2583             SEEK_CUR);
2584         if (masterfd_pos == filesize(masterfd))
2585         {
2586             logf("appending...");
2587             init = true;
2588         }
2589     }
2590
2591     /**
2592      * Load new unique tags in memory to be sorted later and added
2593      * to the master lookup file.
2594      */
2595     if (TAGCACHE_IS_SORTED(index_type))
2596     {
2597         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2598         /* h is the header of the temporary file containing new tags. */
2599         logf("inserting new tags...");
2600         for (i = 0; i < h->entry_count; i++)
2601         {
2602             struct temp_file_entry entry;
2603
2604             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2605                 sizeof(struct temp_file_entry))
2606             {
2607                 logf("read fail #3");
2608                 error = true;
2609                 goto error_exit;
2610             }
2611
2612             /* Read data. */
2613             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2614             {
2615                 logf("too long entry!");
2616                 error = true;
2617                 goto error_exit;
2618             }
2619
2620             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2621             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2622                 entry.tag_length[index_type])
2623             {
2624                 logf("read fail #4");
2625                 error = true;
2626                 goto error_exit;
2627             }
2628
2629             if (TAGCACHE_IS_UNIQUE(index_type))
2630                 error = !tempbuf_insert(buf, i, -1, true);
2631             else
2632                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2633
2634             if (error)
2635             {
2636                 logf("insert error");
2637                 goto error_exit;
2638             }
2639
2640             /* Skip to next. */
2641             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2642                     entry.tag_length[index_type], SEEK_CUR);
2643             do_timed_yield();
2644         }
2645         logf("done");
2646
2647         /* Sort the buffer data and write it to the index file. */
2648         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2649         /**
2650          * We need to truncate the index file now. There can be junk left
2651          * at the end of file (however, we _should_ always follow the
2652          * entry_count and don't crash with that).
2653          */
2654         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2655
2656         i = tempbuf_sort(fd);
2657         if (i < 0)
2658             goto error_exit;
2659         logf("sorted %d tags", i);
2660
2661         /**
2662          * Now update all indexes in the master lookup file.
2663          */
2664         logf("updating indices...");
2665         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2666         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2667         {
2668             int j;
2669             int loc = lseek(masterfd, 0, SEEK_CUR);
2670
2671             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2672
2673             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2674                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2675             {
2676                 logf("read fail #5");
2677                 error = true;
2678                 goto error_exit ;
2679             }
2680             lseek(masterfd, loc, SEEK_SET);
2681
2682             for (j = 0; j < idxbuf_pos; j++)
2683             {
2684                 if (idxbuf[j].flag & FLAG_DELETED)
2685                 {
2686                     /* We can just ignore deleted entries. */
2687                     // idxbuf[j].tag_seek[index_type] = 0;
2688                     continue;
2689                 }
2690
2691                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2692                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2693                     + commit_entry_count);
2694
2695                 if (idxbuf[j].tag_seek[index_type] < 0)
2696                 {
2697                     logf("update error: %ld/%d/%ld",
2698                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2699                     error = true;
2700                     goto error_exit;
2701                 }
2702
2703                 do_timed_yield();
2704             }
2705
2706             /* Write back the updated index. */
2707             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2708                         index_entry_ec, tc_stat.econ) !=
2709                 (int)sizeof(struct index_entry)*idxbuf_pos)
2710             {
2711                 logf("write fail");
2712                 error = true;
2713                 goto error_exit;
2714             }
2715         }
2716         logf("done");
2717     }
2718
2719     /**
2720      * Walk through the temporary file containing the new tags.
2721      */
2722     // build_normal_index(h, tmpfd, masterfd, idx);
2723     logf("updating new indices...");
2724     lseek(masterfd, masterfd_pos, SEEK_SET);
2725     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2726     lseek(fd, 0, SEEK_END);
2727     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2728     {
2729         int j;
2730
2731         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2732         if (init)
2733         {
2734             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2735         }
2736         else
2737         {
2738             int loc = lseek(masterfd, 0, SEEK_CUR);
2739
2740             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2741                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2742             {
2743                 logf("read fail #6");
2744                 error = true;
2745                 break ;
2746             }
2747             lseek(masterfd, loc, SEEK_SET);
2748         }
2749
2750         /* Read entry headers. */
2751         for (j = 0; j < idxbuf_pos; j++)
2752         {
2753             if (!TAGCACHE_IS_SORTED(index_type))
2754             {
2755                 struct temp_file_entry entry;
2756                 struct tagfile_entry fe;
2757
2758                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2759                     sizeof(struct temp_file_entry))
2760                 {
2761                     logf("read fail #7");
2762                     error = true;
2763                     break ;
2764                 }
2765
2766                 /* Read data. */
2767                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2768                 {
2769                     logf("too long entry!");
2770                     logf("length=%d", entry.tag_length[index_type]);
2771                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2772                     error = true;
2773                     break ;
2774                 }
2775
2776                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2777                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2778                     entry.tag_length[index_type])
2779                 {
2780                     logf("read fail #8");
2781                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2782                     logf("length=0x%02x", entry.tag_length[index_type]);
2783                     error = true;
2784                     break ;
2785                 }
2786
2787                 /* Write to index file. */
2788                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2789                 fe.tag_length = entry.tag_length[index_type];
2790                 fe.idx_id = tcmh.tch.entry_count + i + j;
2791                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2792                 write(fd, buf, fe.tag_length);
2793                 tempbufidx++;
2794
2795                 /* Skip to next. */
2796                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2797                       entry.tag_length[index_type], SEEK_CUR);
2798             }
2799             else
2800             {
2801                 /* Locate the correct entry from the sorted array. */
2802                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2803                 if (idxbuf[j].tag_seek[index_type] < 0)
2804                 {
2805                     logf("entry not found (%d)", j);
2806                     error = true;
2807                     break ;
2808                 }
2809             }
2810         }
2811
2812         /* Write index. */
2813         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2814                   index_entry_ec, tc_stat.econ) !=
2815             (int)sizeof(struct index_entry)*idxbuf_pos)
2816         {
2817             logf("tagcache: write fail #4");
2818             error = true;
2819             break ;
2820         }
2821
2822         do_timed_yield();
2823     }
2824     logf("done");
2825
2826     /* Finally write the header. */
2827     tch.magic = TAGCACHE_MAGIC;
2828     tch.entry_count = tempbufidx;
2829     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2830     lseek(fd, 0, SEEK_SET);
2831     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2832
2833     if (index_type != tag_filename)
2834         h->datasize += tch.datasize;
2835     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2836     error_exit:
2837
2838     close(fd);
2839     close(masterfd);
2840
2841     if (error)
2842         return -2;
2843
2844     return 1;
2845 }
2846
2847 static bool commit(void)
2848 {
2849     struct tagcache_header tch;
2850     struct master_header   tcmh;
2851     char path[MAX_PATH];
2852     const char *file;
2853     int i, len, rc;
2854     int tmpfd;
2855     int masterfd;
2856 #ifdef HAVE_DIRCACHE
2857     bool dircache_buffer_stolen = false;
2858 #endif
2859     bool local_allocation = false;
2860
2861     logf("committing tagcache");
2862
2863     while (write_lock)
2864         sleep(1);
2865
2866     file = get_user_file_path(TAGCACHE_FILE_TEMP,
2867                             IS_FILE|NEED_WRITE, path, sizeof(path));
2868
2869     tmpfd = open(file, O_RDONLY);
2870     if (tmpfd < 0)
2871     {
2872         logf("nothing to commit");
2873         return true;
2874     }
2875
2876
2877     /* Load the header. */
2878     len = sizeof(struct tagcache_header);
2879     rc = read(tmpfd, &tch, len);
2880
2881     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2882     {
2883         logf("incorrect tmpheader");
2884         close(tmpfd);
2885         remove(file);
2886         return false;
2887     }
2888
2889     if (tch.entry_count == 0)
2890     {
2891         logf("nothing to commit");
2892         close(tmpfd);
2893         remove(file);
2894         return true;
2895     }
2896
2897     /* Fully initialize existing headers (if any) before going further. */
2898     tc_stat.ready = check_all_headers();
2899
2900 #ifdef HAVE_EEPROM_SETTINGS
2901     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
2902                 path, sizeof(path)));
2903 #endif
2904
2905     /* At first be sure to unload the ramcache! */
2906 #ifdef HAVE_TC_RAMCACHE
2907     tc_stat.ramcache = false;
2908 #endif
2909
2910     read_lock++;
2911
2912     /* Try to steal every buffer we can :) */
2913     if (tempbuf_size == 0)
2914         local_allocation = true;
2915
2916 #ifdef HAVE_DIRCACHE
2917     if (tempbuf_size == 0)
2918     {
2919         /* Try to steal the dircache buffer. */
2920         tempbuf = dircache_steal_buffer(&tempbuf_size);
2921         tempbuf_size &= ~0x03;
2922
2923         if (tempbuf_size > 0)
2924         {
2925             dircache_buffer_stolen = true;
2926         }
2927     }
2928 #endif
2929
2930 #ifdef HAVE_TC_RAMCACHE
2931     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2932     {
2933         tempbuf = (char *)(hdr + 1);
2934         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2935         tempbuf_size &= ~0x03;
2936     }
2937 #endif
2938
2939     /* And finally fail if there are no buffers available. */
2940     if (tempbuf_size == 0)
2941     {
2942         logf("delaying commit until next boot");
2943         tc_stat.commit_delayed = true;
2944         close(tmpfd);
2945         read_lock--;
2946         return false;
2947     }
2948
2949     logf("commit %ld entries...", tch.entry_count);
2950
2951     /* Mark DB dirty so it will stay disabled if commit fails. */
2952     current_tcmh.dirty = true;
2953     update_master_header();
2954
2955     /* Now create the index files. */
2956     tc_stat.commit_step = 0;
2957     tch.datasize = 0;
2958     tc_stat.commit_delayed = false;
2959
2960     for (i = 0; i < TAG_COUNT; i++)
2961     {
2962         int ret;
2963
2964         if (TAGCACHE_IS_NUMERIC(i))
2965             continue;
2966
2967         tc_stat.commit_step++;
2968         ret = build_index(i, &tch, tmpfd);
2969         if (ret <= 0)
2970         {
2971             close(tmpfd);
2972             logf("tagcache failed init");
2973             if (ret == 0)
2974                 tc_stat.commit_delayed = true;
2975
2976             tc_stat.commit_step = 0;
2977             read_lock--;
2978             return false;
2979         }
2980     }
2981
2982     if (!build_numeric_indices(&tch, tmpfd))
2983     {
2984         logf("Failure to commit numeric indices");
2985         close(tmpfd);
2986         tc_stat.commit_step = 0;
2987         read_lock--;
2988         return false;
2989     }
2990
2991     close(tmpfd);
2992     remove(file);
2993
2994     tc_stat.commit_step = 0;
2995
2996     /* Update the master index headers. */
2997     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2998     {
2999         read_lock--;
3000         return false;
3001     }
3002
3003     tcmh.tch.entry_count += tch.entry_count;
3004     tcmh.tch.datasize = sizeof(struct master_header)
3005         + sizeof(struct index_entry) * tcmh.tch.entry_count
3006         + tch.datasize;
3007     tcmh.dirty = false;
3008     tcmh.commitid++;
3009
3010     lseek(masterfd, 0, SEEK_SET);
3011     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3012     close(masterfd);
3013
3014     logf("tagcache committed");
3015     tc_stat.ready = check_all_headers();
3016     tc_stat.readyvalid = true;
3017
3018     if (local_allocation)
3019     {
3020         tempbuf = NULL;
3021         tempbuf_size = 0;
3022     }
3023
3024 #ifdef HAVE_DIRCACHE
3025     /* Rebuild the dircache, if we stole the buffer. */
3026     if (dircache_buffer_stolen)
3027         dircache_build(0);
3028 #endif
3029
3030 #ifdef HAVE_TC_RAMCACHE
3031     /* Reload tagcache. */
3032     if (tc_stat.ramcache_allocated > 0)
3033         tagcache_start_scan();
3034 #endif
3035
3036     read_lock--;
3037
3038     return true;
3039 }
3040
3041 static void allocate_tempbuf(void)
3042 {
3043     /* Yeah, malloc would be really nice now :) */
3044 #ifdef __PCTOOL__
3045     tempbuf_size = 32*1024*1024;
3046     tempbuf = malloc(tempbuf_size);
3047 #else
3048     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3049     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3050     audiobuf += tempbuf_size;
3051 #endif
3052 }
3053
3054 static void free_tempbuf(void)
3055 {
3056     if (tempbuf_size == 0)
3057         return ;
3058
3059 #ifdef __PCTOOL__
3060     free(tempbuf);
3061 #else
3062     audiobuf -= tempbuf_size;
3063 #endif
3064     tempbuf = NULL;
3065     tempbuf_size = 0;
3066 }
3067
3068 #ifndef __PCTOOL__
3069
3070 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3071 {
3072     struct index_entry idx;
3073
3074     if (!tc_stat.ready)
3075         return false;
3076
3077     if (!TAGCACHE_IS_NUMERIC(tag))
3078         return false;
3079
3080     if (!get_index(masterfd, idx_id, &idx, false))
3081         return false;
3082
3083     idx.tag_seek[tag] = data;
3084     idx.flag |= FLAG_DIRTYNUM;
3085
3086     return write_index(masterfd, idx_id, &idx);
3087 }
3088
3089 #if 0
3090 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3091                                    int tag, long data)
3092 {
3093     struct master_header myhdr;
3094
3095     if (tcs->masterfd < 0)
3096     {
3097         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3098             return false;
3099     }
3100
3101     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3102 }
3103 #endif
3104
3105 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3106
3107 static bool command_queue_is_full(void)
3108 {
3109     int next;
3110
3111     next = command_queue_widx + 1;
3112     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3113         next = 0;
3114
3115     return (next == command_queue_ridx);
3116 }
3117
3118 static void command_queue_sync_callback(void *data)
3119 {
3120     (void)data;
3121     struct master_header myhdr;
3122     int masterfd;
3123
3124     mutex_lock(&command_queue_mutex);
3125
3126     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3127         return;
3128
3129     while (command_queue_ridx != command_queue_widx)
3130     {
3131         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3132
3133         switch (ce->command)
3134         {
3135             case CMD_UPDATE_MASTER_HEADER:
3136             {
3137                 close(masterfd);
3138                 update_master_header();
3139
3140                 /* Re-open the masterfd. */
3141                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3142                     return;
3143
3144                 break;
3145             }
3146             case CMD_UPDATE_NUMERIC:
3147             {
3148                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3149                 break;
3150             }
3151         }
3152
3153         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3154             command_queue_ridx = 0;
3155     }
3156
3157     close(masterfd);
3158
3159     tc_stat.queue_length = 0;
3160     mutex_unlock(&command_queue_mutex);
3161 }
3162
3163 static void run_command_queue(bool force)
3164 {
3165     if (COMMAND_QUEUE_IS_EMPTY)
3166         return;
3167
3168     if (force || command_queue_is_full())
3169         command_queue_sync_callback(NULL);
3170     else
3171         register_storage_idle_func(command_queue_sync_callback);
3172 }
3173
3174 static void queue_command(int cmd, long idx_id, int tag, long data)
3175 {
3176     while (1)
3177     {
3178         int next;
3179
3180         mutex_lock(&command_queue_mutex);
3181         next = command_queue_widx + 1;
3182         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3183             next = 0;
3184
3185         /* Make sure queue is not full. */
3186         if (next != command_queue_ridx)
3187         {
3188             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3189
3190             ce->command = cmd;
3191             ce->idx_id = idx_id;
3192             ce->tag = tag;
3193             ce->data = data;
3194
3195             command_queue_widx = next;
3196
3197             tc_stat.queue_length++;
3198
3199             mutex_unlock(&command_queue_mutex);
3200             break;
3201         }
3202
3203         /* Queue is full, try again later... */
3204         mutex_unlock(&command_queue_mutex);
3205         sleep(1);
3206     }
3207 }
3208
3209 long tagcache_increase_serial(void)
3210 {
3211     long old;
3212
3213     if (!tc_stat.ready)
3214         return -2;
3215
3216     while (read_lock)
3217         sleep(1);
3218
3219     old = current_tcmh.serial++;
3220     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3221
3222     return old;
3223 }
3224
3225 void tagcache_update_numeric(int idx_id, int tag, long data)
3226 {
3227     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3228 }
3229 #endif /* !__PCTOOL__ */
3230
3231 long tagcache_get_serial(void)
3232 {
3233     return current_tcmh.serial;
3234 }
3235
3236 long tagcache_get_commitid(void)
3237 {
3238     return current_tcmh.commitid;
3239 }
3240
3241 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3242 {
3243     char buf[512];
3244     int i;
3245
3246     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3247     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3248     {
3249         if (*datastr == '\0')
3250             break;
3251
3252         if (*datastr == '"' || *datastr == '\\')
3253             buf[i++] = '\\';
3254
3255         buf[i] = *(datastr++);
3256     }
3257
3258     strcpy(&buf[i], "\" ");
3259
3260     write(fd, buf, i + 2);
3261
3262     return true;
3263 }
3264
3265 #ifndef __PCTOOL__
3266
3267 static bool read_tag(char *dest, long size,
3268                      const char *src, const char *tagstr)
3269 {
3270     int pos;
3271     char current_tag[32];
3272
3273     while (*src != '\0')
3274     {
3275         /* Skip all whitespace */
3276         while (*src == ' ')
3277             src++;
3278
3279         if (*src == '\0')
3280             break;
3281
3282         pos = 0;
3283         /* Read in tag name */
3284         while (*src != '=' && *src != ' ')
3285         {
3286             current_tag[pos] = *src;
3287             src++;
3288             pos++;
3289
3290             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3291                 return false;
3292         }
3293         current_tag[pos] = '\0';
3294
3295         /* Read in tag data */
3296
3297         /* Find the start. */
3298         while (*src != '"' && *src != '\0')
3299             src++;
3300
3301         if (*src == '\0' || *(++src) == '\0')
3302             return false;
3303
3304         /* Read the data. */
3305         for (pos = 0; pos < size; pos++)
3306         {
3307             if (*src == '\0')
3308                 break;
3309
3310             if (*src == '\\')
3311             {
3312                 dest[pos] = *(src+1);
3313                 src += 2;
3314                 continue;
3315             }
3316
3317             dest[pos] = *src;
3318
3319             if (*src == '"')
3320             {
3321                 src++;
3322                 break;
3323             }
3324
3325             if (*src == '\0')
3326                 break;
3327
3328             src++;
3329         }
3330         dest[pos] = '\0';
3331
3332         if (!strcasecmp(tagstr, current_tag))
3333             return true;
3334     }
3335
3336     return false;
3337 }
3338
3339 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3340 {
3341     struct index_entry idx;
3342     char tag_data[TAG_MAXLEN+32];
3343     int idx_id;
3344     long masterfd = (long)parameters;
3345     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3346         tag_commitid };
3347     int i;
3348     (void)line_n;
3349
3350     if (*buf == '#')
3351         return 0;
3352
3353     logf("%d/%s", line_n, buf);
3354     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3355     {
3356         logf("filename missing");
3357         logf("-> %s", buf);
3358         return 0;
3359     }
3360
3361     idx_id = find_index(tag_data);
3362     if (idx_id < 0)
3363     {
3364         logf("entry not found");
3365         return 0;
3366     }
3367
3368     if (!get_index(masterfd, idx_id, &idx, false))
3369     {
3370         logf("failed to retrieve index entry");
3371         return 0;
3372     }
3373
3374     /* Stop if tag has already been modified. */
3375     if (idx.flag & FLAG_DIRTYNUM)
3376         return 0;
3377
3378     logf("import: %s", tag_data);
3379
3380     idx.flag |= FLAG_DIRTYNUM;
3381     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3382     {
3383         int data;
3384
3385         if (!read_tag(tag_data, sizeof tag_data, buf,
3386                       tagcache_tag_to_str(import_tags[i])))
3387         {
3388             continue;
3389         }
3390
3391         data = atoi(tag_data);
3392         if (data < 0)
3393             continue;
3394
3395         idx.tag_seek[import_tags[i]] = data;
3396
3397         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3398             current_tcmh.serial = data + 1;
3399         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3400             current_tcmh.commitid = data + 1;
3401     }
3402
3403     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3404 }
3405
3406 bool tagcache_import_changelog(void)
3407 {
3408     struct master_header myhdr;
3409     struct tagcache_header tch;
3410     int clfd;
3411     long masterfd;
3412     char buf[MAX(MAX_PATH, 2048)];
3413     const char *file;
3414
3415     if (!tc_stat.ready)
3416         return false;
3417
3418     while (read_lock)
3419         sleep(1);
3420
3421     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG,
3422                                    IS_FILE|NEED_WRITE, buf, sizeof(buf));
3423     clfd = open(file, O_RDONLY);
3424     if (clfd < 0)
3425     {
3426         logf("failure to open changelog");
3427         return false;
3428     }
3429
3430     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3431     {
3432         close(clfd);
3433         return false;
3434     }
3435
3436     write_lock++;
3437
3438     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3439
3440     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3441                   parse_changelog_line);
3442
3443     close(clfd);
3444     close(masterfd);
3445
3446     if (filenametag_fd >= 0)
3447     {
3448         close(filenametag_fd);
3449         filenametag_fd = -1;
3450     }
3451
3452     write_lock--;
3453
3454     update_master_header();
3455
3456     return true;
3457 }
3458
3459 #endif /* !__PCTOOL__ */
3460
3461 bool tagcache_create_changelog(struct tagcache_search *tcs)
3462 {
3463     struct master_header myhdr;
3464     struct index_entry idx;
3465     const char *file;
3466     char buf[MAX(TAG_MAXLEN+32, MAX_PATH)];
3467     char temp[32];
3468     int clfd;
3469     int i, j;
3470
3471     if (!tc_stat.ready)
3472         return false;
3473
3474     if (!tagcache_search(tcs, tag_filename))
3475         return false;
3476
3477     /* Initialize the changelog */
3478     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG, IS_FILE | NEED_WRITE,
3479                               buf, sizeof(buf));
3480     clfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3481     if (clfd < 0)
3482     {
3483         logf("failure to open changelog");
3484         return false;
3485     }
3486
3487     if (tcs->masterfd < 0)
3488     {
3489         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3490             return false;
3491     }
3492     else
3493     {
3494         lseek(tcs->masterfd, 0, SEEK_SET);
3495         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3496     }
3497
3498     write(clfd, "## Changelog version 1\n", 23);
3499
3500     for (i = 0; i < myhdr.tch.entry_count; i++)
3501     {
3502         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3503         {
3504             logf("read error #9");
3505             tagcache_search_finish(tcs);
3506             close(clfd);
3507             return false;
3508         }
3509
3510         /* Skip until the entry found has been modified. */
3511         if (! (idx.flag & FLAG_DIRTYNUM) )
3512             continue;
3513
3514         /* Skip deleted entries too. */
3515         if (idx.flag & FLAG_DELETED)
3516             continue;
3517
3518         /* Now retrieve all tags. */
3519         for (j = 0; j < TAG_COUNT; j++)
3520         {
3521             if (TAGCACHE_IS_NUMERIC(j))
3522             {
3523                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3524                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3525                 continue;
3526             }
3527
3528             tcs->type = j;
3529             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3530             write_tag(clfd, tagcache_tag_to_str(j), buf);
3531         }
3532
3533         write(clfd, "\n", 1);
3534         do_timed_yield();
3535     }
3536
3537     close(clfd);
3538
3539     tagcache_search_finish(tcs);
3540
3541     return true;
3542 }
3543
3544 static bool delete_entry(long idx_id)
3545 {
3546     int fd = -1;
3547     int masterfd = -1;
3548     int tag, i;
3549     struct index_entry idx, myidx;
3550     struct master_header myhdr;
3551     char buf[TAG_MAXLEN+32];
3552     int in_use[TAG_COUNT];
3553
3554     logf("delete_entry(): %ld", idx_id);
3555
3556 #ifdef HAVE_TC_RAMCACHE
3557     /* At first mark the entry removed from ram cache. */
3558     if (tc_stat.ramcache)
3559         hdr->indices[idx_id].flag |= FLAG_DELETED;
3560 #endif
3561
3562     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3563         return false;
3564
3565     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3566     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3567     {
3568         logf("delete_entry(): read error");
3569         goto cleanup;
3570     }
3571
3572     if (myidx.flag & FLAG_DELETED)
3573     {
3574         logf("delete_entry(): already deleted!");
3575         goto cleanup;
3576     }
3577
3578     myidx.flag |= FLAG_DELETED;
3579     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3580     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3581     {
3582         logf("delete_entry(): write_error #1");
3583         goto cleanup;
3584     }
3585
3586     /* Now check which tags are no longer in use (if any) */
3587     for (tag = 0; tag < TAG_COUNT; tag++)
3588         in_use[tag] = 0;
3589
3590     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3591     for (i = 0; i < myhdr.tch.entry_count; i++)
3592     {
3593         struct index_entry *idxp;
3594
3595 #ifdef HAVE_TC_RAMCACHE
3596         /* Use RAM DB if available for greater speed */
3597         if (tc_stat.ramcache)
3598             idxp = &hdr->indices[i];
3599         else
3600 #endif
3601         {
3602             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3603             {
3604                 logf("delete_entry(): read error #2");
3605                 goto cleanup;
3606             }
3607             idxp = &idx;
3608         }
3609
3610         if (idxp->flag & FLAG_DELETED)
3611             continue;
3612
3613         for (tag = 0; tag < TAG_COUNT; tag++)
3614         {
3615             if (TAGCACHE_IS_NUMERIC(tag))
3616                 continue;
3617
3618             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3619                 in_use[tag]++;
3620         }
3621     }
3622
3623     /* Now delete all tags no longer in use. */
3624     for (tag = 0; tag < TAG_COUNT; tag++)
3625     {
3626         struct tagcache_header tch;
3627         int oldseek = myidx.tag_seek[tag];
3628
3629         if (TAGCACHE_IS_NUMERIC(tag))
3630             continue;
3631
3632         /**
3633          * Replace tag seek with a hash value of the field string data.
3634          * That way runtime statistics of moved or altered files can be
3635          * resurrected.
3636          */
3637 #ifdef HAVE_TC_RAMCACHE
3638         if (tc_stat.ramcache && tag != tag_filename)
3639         {
3640             struct tagfile_entry *tfe;
3641             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3642
3643             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3644             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3645             myidx.tag_seek[tag] = *seek;
3646         }
3647         else
3648 #endif
3649         {
3650             struct tagfile_entry tfe;
3651
3652             /* Open the index file, which contains the tag names. */
3653             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3654                 goto cleanup;
3655
3656             /* Skip the header block */
3657             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3658             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3659             {
3660                 logf("delete_entry(): read error #3");
3661                 goto cleanup;
3662             }
3663
3664             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3665             {
3666                 logf("delete_entry(): read error #4");
3667                 goto cleanup;
3668             }
3669
3670             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3671         }
3672
3673         if (in_use[tag])
3674         {
3675             logf("in use: %d/%d", tag, in_use[tag]);
3676             if (fd >= 0)
3677             {
3678                 close(fd);
3679                 fd = -1;
3680             }
3681             continue;
3682         }
3683
3684 #ifdef HAVE_TC_RAMCACHE
3685         /* Delete from ram. */
3686         if (tc_stat.ramcache && tag != tag_filename)
3687         {
3688             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3689             tagentry->tag_data[0] = '\0';
3690         }
3691 #endif
3692
3693         /* Open the index file, which contains the tag names. */
3694         if (fd < 0)
3695         {
3696             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3697                 goto cleanup;
3698         }
3699
3700         /* Skip the header block */
3701         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3702
3703         /* Debug, print 10 first characters of the tag
3704         read(fd, buf, 10);
3705         buf[10]='\0';
3706         logf("TAG:%s", buf);
3707         lseek(fd, -10, SEEK_CUR);
3708         */
3709
3710         /* Write first data byte in tag as \0 */
3711         write(fd, "", 1);
3712
3713         /* Now tag data has been removed */
3714         close(fd);
3715         fd = -1;
3716     }
3717
3718     /* Write index entry back into master index. */
3719     lseek(masterfd, sizeof(struct master_header) +
3720           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3721     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3722     {
3723         logf("delete_entry(): write_error #2");
3724         goto cleanup;
3725     }
3726
3727     close(masterfd);
3728
3729     return true;
3730
3731     cleanup:
3732     if (fd >= 0)
3733         close(fd);
3734     if (masterfd >= 0)
3735         close(masterfd);
3736
3737     return false;
3738 }
3739
3740 #ifndef __PCTOOL__
3741 /**
3742  * Returns true if there is an event waiting in the queue
3743  * that requires the current operation to be aborted.
3744  */
3745 static bool check_event_queue(void)
3746 {
3747     struct queue_event ev;
3748
3749     if(!queue_peek(&tagcache_queue, &ev))
3750         return false;
3751
3752     switch (ev.id)
3753     {
3754         case Q_STOP_SCAN:
3755         case SYS_POWEROFF:
3756         case SYS_USB_CONNECTED:
3757             return true;
3758     }
3759
3760     return false;
3761 }
3762 #endif
3763
3764 #ifdef HAVE_TC_RAMCACHE
3765 static bool allocate_tagcache(void)
3766 {
3767     struct master_header tcmh;
3768     int fd;
3769
3770     /* Load the header. */
3771     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3772     {
3773         hdr = NULL;
3774         return false;
3775     }
3776
3777     close(fd);
3778
3779     /**
3780      * Now calculate the required cache size plus
3781      * some extra space for alignment fixes.
3782      */
3783     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3784         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3785     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3786     memset(hdr, 0, sizeof(struct ramcache_header));
3787     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3788     hdr->indices = (struct index_entry *)(hdr + 1);
3789     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3790
3791     return true;
3792 }
3793
3794 # ifdef HAVE_EEPROM_SETTINGS
3795 static bool tagcache_dumpload(void)
3796 {
3797     struct statefile_header shdr;
3798     char path[MAX_PATH];
3799     const char *file;
3800     int fd, rc;
3801     long offpos;
3802     int i;
3803
3804     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3805                                     path, sizeof(path));
3806     fd = open(file, O_RDONLY);
3807     if (fd < 0)
3808     {
3809         logf("no tagcache statedump");
3810         return false;
3811     }
3812
3813     /* Check the statefile memory placement */
3814     hdr = buffer_alloc(0);
3815     rc = read(fd, &shdr, sizeof(struct statefile_header));
3816     if (rc != sizeof(struct statefile_header)
3817         /* || (long)hdr != (long)shdr.hdr */)
3818     {
3819         logf("incorrect statefile");
3820         hdr = NULL;
3821         close(fd);
3822         return false;
3823     }
3824
3825     offpos = (long)hdr - (long)shdr.hdr;
3826
3827     /* Lets allocate real memory and load it */
3828     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3829     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3830     close(fd);
3831
3832     if (rc != shdr.tc_stat.ramcache_allocated)
3833     {
3834         logf("read failure!");
3835         hdr = NULL;
3836         return false;
3837     }
3838
3839     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3840
3841     /* Now fix the pointers */
3842     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3843     for (i = 0; i < TAG_COUNT; i++)
3844         hdr->tags[i] += offpos;
3845
3846     return true;
3847 }
3848
3849 static bool tagcache_dumpsave(void)
3850 {
3851     struct statefile_header shdr;
3852     char path[MAX_PATH];
3853     const char *file;
3854     int fd;
3855
3856     if (!tc_stat.ramcache)
3857         return false;
3858
3859     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3860                                     path, sizeof(path));
3861     fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3862     if (fd < 0)
3863     {
3864         logf("failed to create a statedump");
3865         return false;
3866     }
3867
3868     /* Create the header */
3869     shdr.hdr = hdr;
3870     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3871     write(fd, &shdr, sizeof(struct statefile_header));
3872
3873     /* And dump the data too */
3874     write(fd, hdr, tc_stat.ramcache_allocated);
3875     close(fd);
3876
3877     return true;
3878 }
3879 # endif
3880
3881 static bool load_tagcache(void)
3882 {
3883     struct tagcache_header *tch;
3884     long bytesleft = tc_stat.ramcache_allocated;
3885     struct index_entry *idx;
3886     int rc, fd;
3887     char *p, path[MAX_PATH];
3888     const char *file;
3889     int i, tag;
3890
3891 # ifdef HAVE_DIRCACHE
3892     while (dircache_is_initializing())
3893         sleep(1);
3894
3895     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3896 # endif
3897
3898     logf("loading tagcache to ram...");
3899
3900     file = get_user_file_path(TAGCACHE_FILE_MASTER,
3901                               IS_FILE|NEED_WRITE,
3902                               path, sizeof(path));
3903     fd = open(file, O_RDONLY);
3904     if (fd < 0)
3905     {
3906         logf("tagcache open failed");
3907         return false;
3908     }
3909
3910     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3911         != sizeof(struct master_header)
3912         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3913     {
3914         logf("incorrect header");
3915         return false;
3916     }
3917
3918     idx = hdr->indices;
3919
3920     /* Load the master index table. */
3921     for (i = 0; i < hdr->h.tch.entry_count; i++)
3922     {
3923         rc = ecread_index_entry(fd, idx);
3924         if (rc != sizeof(struct index_entry))
3925         {
3926             logf("read error #10");
3927             close(fd);
3928             return false;
3929         }
3930
3931         bytesleft -= sizeof(struct index_entry);
3932         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3933         {
3934             logf("too big tagcache.");
3935             close(fd);
3936             return false;
3937         }
3938
3939         idx++;
3940     }
3941
3942     close(fd);
3943
3944     /* Load the tags. */
3945     p = (char *)idx;
3946     for (tag = 0; tag < TAG_COUNT; tag++)
3947     {
3948         struct tagfile_entry *fe;
3949         char buf[TAG_MAXLEN+32];
3950
3951         if (TAGCACHE_IS_NUMERIC(tag))
3952             continue ;
3953
3954         //p = ((void *)p+1);
3955         p = (char *)((long)p & ~0x03) + 0x04;
3956         hdr->tags[tag] = p;
3957
3958         /* Check the header. */
3959         tch = (struct tagcache_header *)p;
3960         p += sizeof(struct tagcache_header);
3961
3962         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3963             return false;
3964
3965         for (hdr->entry_count[tag] = 0;
3966              hdr->entry_count[tag] < tch->entry_count;
3967              hdr->entry_count[tag]++)
3968         {
3969             long pos;
3970
3971             if (do_timed_yield())
3972             {
3973                 /* Abort if we got a critical event in queue */
3974                 if (check_event_queue())
3975                     return false;
3976             }
3977
3978             fe = (struct tagfile_entry *)p;
3979             pos = lseek(fd, 0, SEEK_CUR);
3980             rc = ecread_tagfile_entry(fd, fe);
3981             if (rc != sizeof(struct tagfile_entry))
3982             {
3983                 /* End of lookup table. */
3984                 logf("read error #11");
3985                 close(fd);
3986                 return false;
3987             }
3988
3989             /* We have a special handling for the filename tags. */
3990             if (tag == tag_filename)
3991             {
3992 # ifdef HAVE_DIRCACHE
3993                 const struct dircache_entry *dc;
3994 # endif
3995
3996                 // FIXME: This is wrong!
3997                 // idx = &hdr->indices[hdr->entry_count[i]];
3998                 idx = &hdr->indices[fe->idx_id];
3999
4000                 if (fe->tag_length >= (long)sizeof(buf)-1)
4001                 {
4002                     read(fd, buf, 10);
4003                     buf[10] = '\0';
4004                     logf("TAG:%s", buf);
4005                     logf("too long filename");
4006                     close(fd);
4007                     return false;
4008                 }
4009
4010                 rc = read(fd, buf, fe->tag_length);
4011                 if (rc != fe->tag_length)
4012                 {
4013                     logf("read error #12");
4014                     close(fd);
4015                     return false;
4016                 }
4017
4018                 /* Check if the entry has already been removed */
4019                 if (idx->flag & FLAG_DELETED)
4020                     continue;
4021
4022                 /* This flag must not be used yet. */
4023                 if (idx->flag & FLAG_DIRCACHE)
4024                 {
4025                     logf("internal error!");
4026                     close(fd);
4027                     return false;
4028                 }
4029
4030                 if (idx->tag_seek[tag] != pos)
4031                 {
4032                     logf("corrupt data structures!");
4033                     close(fd);
4034                     return false;
4035                 }
4036
4037 # ifdef HAVE_DIRCACHE
4038                 if (dircache_is_enabled())
4039                 {
4040                     dc = dircache_get_entry_ptr(buf);
4041                     if (dc == NULL)
4042                     {
4043                         logf("Entry no longer valid.");
4044                         logf("-> %s", buf);
4045                         if (global_settings.tagcache_autoupdate)
4046                             delete_entry(fe->idx_id);
4047                         continue ;
4048                     }
4049
4050                     idx->flag |= FLAG_DIRCACHE;
4051                     idx->tag_seek[tag_filename] = (long)dc;
4052                 }
4053                 else
4054 # endif
4055                 {
4056                     /* This will be very slow unless dircache is enabled
4057                        or target is flash based, but do it anyway for
4058                        consistency. */
4059                     /* Check if entry has been removed. */
4060                     if (global_settings.tagcache_autoupdate)
4061                     {
4062                         if (!file_exists(buf))
4063                         {
4064                             logf("Entry no longer valid.");
4065                             logf("-> %s", buf);
4066                             delete_entry(fe->idx_id);
4067                             continue;
4068                         }
4069                     }
4070                 }
4071
4072                 continue ;
4073             }
4074
4075             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4076             if (bytesleft < 0)
4077             {
4078                 logf("too big tagcache #2");
4079                 logf("tl: %ld", fe->tag_length);
4080                 logf("bl: %ld", bytesleft);
4081                 close(fd);
4082                 return false;
4083             }
4084
4085             p = fe->tag_data;
4086             rc = read(fd, fe->tag_data, fe->tag_length);
4087             p += rc;
4088
4089             if (rc != fe->tag_length)
4090             {
4091                 logf("read error #13");
4092                 logf("rc=0x%04x", rc); // 0x431
4093                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4094                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4095                 logf("tag=0x%02x", tag); // 0x00
4096                 close(fd);
4097                 return false;
4098             }
4099         }
4100         close(fd);
4101     }
4102
4103     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4104     logf("tagcache loaded into ram!");
4105
4106     return true;
4107 }
4108 #endif /* HAVE_TC_RAMCACHE */
4109
4110 static bool check_deleted_files(void)
4111 {
4112     int fd;
4113     char buf[TAG_MAXLEN+32], path[MAX_PATH];
4114     const char *file;
4115     struct tagfile_entry tfe;
4116
4117     logf("reverse scan...");
4118     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4119     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
4120     fd = open(file, O_RDONLY);
4121
4122     if (fd < 0)
4123     {
4124         logf("%s open fail", buf);
4125         return false;
4126     }
4127
4128     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4129     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4130 #ifndef __PCTOOL__
4131            && !check_event_queue()
4132 #endif
4133            )
4134     {
4135         if (tfe.tag_length >= (long)sizeof(buf)-1)
4136         {
4137             logf("too long tag");
4138             close(fd);
4139             return false;
4140         }
4141
4142         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4143         {
4144             logf("read error #14");
4145             close(fd);
4146             return false;
4147         }
4148
4149         /* Check if the file has already deleted from the db. */
4150         if (*buf == '\0')
4151             continue;
4152
4153         /* Now check if the file exists. */
4154         if (!file_exists(buf))
4155         {
4156             logf("Entry no longer valid.");
4157             logf("-> %s / %ld", buf, tfe.tag_length);
4158             delete_entry(tfe.idx_id);
4159         }
4160     }
4161
4162     close(fd);
4163
4164     logf("done");
4165
4166     return true;
4167 }
4168
4169
4170 /* Note that this function must not be inlined, otherwise the whole point
4171  * of having the code in a separate function is lost.
4172  */
4173 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4174     int *ignore, int *unignore)
4175 {
4176     char newpath[MAX_PATH];
4177
4178     /* check for a database.ignore file */
4179     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4180     *ignore = file_exists(newpath);
4181     /* check for a database.unignore file */
4182     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4183     *unignore = file_exists(newpath);
4184 }
4185
4186
4187 static bool check_dir(const char *dirname, int add_files)
4188 {
4189     DIR *dir;
4190     int len;
4191     int success = false;
4192     int ignore, unignore;
4193
4194     dir = opendir(dirname);
4195     if (!dir)
4196     {
4197         logf("tagcache: opendir(%s) failed", dirname);
4198         return false;
4199     }
4200
4201     /* check for a database.ignore and database.unignore */
4202     check_ignore(dirname, &ignore, &unignore);
4203
4204     /* don't do anything if both ignore and unignore are there */
4205     if (ignore != unignore)
4206         add_files = unignore;
4207
4208     /* Recursively scan the dir. */
4209 #ifdef __PCTOOL__
4210     while (1)
4211 #else
4212     while (!check_event_queue())
4213 #endif
4214     {
4215         struct dirent *entry;
4216
4217         entry = readdir(dir);
4218
4219         if (entry == NULL)
4220         {
4221             success = true;
4222             break ;
4223         }
4224
4225         if (!strcmp((char *)entry->d_name, ".") ||
4226             !strcmp((char *)entry->d_name, ".."))
4227             continue;
4228
4229         yield();
4230
4231         len = strlen(curpath);
4232         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4233                  entry->d_name);
4234
4235         processed_dir_count++;
4236         if (entry->attribute & ATTR_DIRECTORY)
4237             check_dir(curpath, add_files);
4238         else if (add_files)
4239         {
4240             tc_stat.curentry = curpath;
4241
4242             /* Add a new entry to the temporary db file. */
4243             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4244 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4245                          , dir->internal_entry
4246 #endif
4247                          );
4248
4249             /* Wait until current path for debug screen is read and unset. */
4250             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4251                 yield();
4252
4253             tc_stat.curentry = NULL;
4254         }
4255
4256         curpath[len] = '\0';
4257     }
4258
4259     closedir(dir);
4260
4261     return success;
4262 }
4263
4264 void tagcache_screensync_event(void)
4265 {
4266     tc_stat.curentry = NULL;
4267 }
4268
4269 void tagcache_screensync_enable(bool state)
4270 {
4271     tc_stat.syncscreen = state;
4272 }
4273
4274 void tagcache_build(const char *path)
4275 {
4276     struct tagcache_header header;
4277     bool ret;
4278     char buf[MAX_PATH];
4279     const char *file;
4280
4281     curpath[0] = '\0';
4282     data_size = 0;
4283     total_entry_count = 0;
4284     processed_dir_count = 0;
4285
4286 #ifdef HAVE_DIRCACHE
4287     while (dircache_is_initializing())
4288         sleep(1);
4289 #endif
4290
4291     logf("updating tagcache");
4292
4293     file = get_user_file_path(TAGCACHE_FILE_TEMP,
4294                                     IS_FILE|NEED_WRITE, buf, sizeof(buf));
4295
4296
4297     if (file_exists(file))
4298     {
4299         logf("skipping, cache already waiting for commit");
4300         return ;
4301     }
4302
4303     cachefd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0666);
4304     if (cachefd < 0)
4305     {
4306         logf("master file open failed: %s", file);
4307         return ;
4308     }
4309
4310     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4311
4312     cpu_boost(true);
4313
4314     logf("Scanning files...");
4315     /* Scan for new files. */
4316     memset(&header, 0, sizeof(struct tagcache_header));
4317     write(cachefd, &header, sizeof(struct tagcache_header));
4318
4319     if (strcmp("/", path) != 0)
4320         strcpy(curpath, path);
4321     ret = check_dir(path, true);
4322
4323     /* Write the header. */
4324     header.magic = TAGCACHE_MAGIC;
4325     header.datasize = data_size;
4326     header.entry_count = total_entry_count;
4327     lseek(cachefd, 0, SEEK_SET);
4328     write(cachefd, &header, sizeof(struct tagcache_header));
4329     close(cachefd);
4330
4331     if (filenametag_fd >= 0)
4332     {
4333         close(filenametag_fd);
4334         filenametag_fd = -1;
4335     }
4336
4337     if (!ret)
4338     {
4339         logf("Aborted.");
4340         cpu_boost(false);
4341         return ;
4342     }
4343
4344     /* Commit changes to the database. */
4345 #ifdef __PCTOOL__
4346     allocate_tempbuf();
4347 #endif
4348     if (commit())
4349     {
4350         remove(file);
4351         logf("tagcache built!");
4352     }
4353 #ifdef __PCTOOL__
4354     free_tempbuf();
4355 #endif
4356
4357 #ifdef HAVE_TC_RAMCACHE
4358     if (hdr)
4359     {
4360         /* Import runtime statistics if we just initialized the db. */
4361         if (hdr->h.serial == 0)
4362             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4363     }
4364 #endif
4365
4366     cpu_boost(false);
4367 }
4368
4369 #ifdef HAVE_TC_RAMCACHE
4370 static void load_ramcache(void)
4371 {
4372     if (!hdr)
4373         return ;
4374
4375     cpu_boost(true);
4376
4377     /* At first we should load the cache (if exists). */
4378     tc_stat.ramcache = load_tagcache();
4379
4380     if (!tc_stat.ramcache)
4381     {
4382         /* If loading failed, it must indicate some problem with the db
4383          * so disable it entirely to prevent further issues. */
4384         tc_stat.ready = false;
4385         hdr = NULL;
4386     }
4387
4388     cpu_boost(false);
4389 }
4390
4391 void tagcache_unload_ramcache(void)
4392 {
4393     tc_stat.ramcache = false;
4394     /* Just to make sure there is no statefile present. */
4395
4396 #if 0
4397     char path[MAX_PATH];
4398     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4399                 path, sizeof(path)));
4400 #endif
4401 }
4402 #endif
4403
4404 #ifndef __PCTOOL__
4405 static void tagcache_thread(void)
4406 {
4407     struct queue_event ev;
4408     bool check_done = false;
4409     char path[MAX_PATH];
4410
4411     /* If the previous cache build/update was interrupted, commit
4412      * the changes first in foreground. */
4413     cpu_boost(true);
4414     allocate_tempbuf();
4415     commit();
4416     free_tempbuf();
4417
4418 #ifdef HAVE_TC_RAMCACHE
4419 # ifdef HAVE_EEPROM_SETTINGS
4420     if (firmware_settings.initialized && firmware_settings.disk_clean
4421         && global_settings.tagcache_ram)
4422     {
4423         check_done = tagcache_dumpload();
4424     }
4425
4426     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4427                 buf, sizeof(buf)));
4428 # endif
4429
4430     /* Allocate space for the tagcache if found on disk. */
4431     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4432         allocate_tagcache();
4433 #endif
4434
4435     cpu_boost(false);
4436     tc_stat.initialized = true;
4437
4438     /* Don't delay bootup with the header check but do it on background. */
4439     if (!tc_stat.ready)
4440     {
4441         sleep(HZ);
4442         tc_stat.ready = check_all_headers();
4443         tc_stat.readyvalid = true;
4444     }
4445
4446     while (1)
4447     {
4448         run_command_queue(false);
4449
4450         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4451
4452         switch (ev.id)
4453         {
4454             case Q_IMPORT_CHANGELOG:
4455                 tagcache_import_changelog();
4456                 break;
4457
4458             case Q_REBUILD:
4459                 remove_files();
4460                 remove(get_user_file_path(TAGCACHE_FILE_TEMP,
4461                                 IS_FILE|NEED_WRITE, path, sizeof(path)));
4462                 tagcache_build("/");
4463                 break;
4464
4465             case Q_UPDATE:
4466                 tagcache_build("/");
4467 #ifdef HAVE_TC_RAMCACHE
4468                 load_ramcache();
4469 #endif
4470                 check_deleted_files();
4471                 break ;
4472
4473             case Q_START_SCAN:
4474                 check_done = false;
4475             case SYS_TIMEOUT:
4476                 if (check_done || !tc_stat.ready)
4477                     break ;
4478
4479 #ifdef HAVE_TC_RAMCACHE
4480                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4481                 {
4482                     load_ramcache();
4483                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4484                         tagcache_build("/");
4485                 }
4486                 else
4487 #endif
4488                 if (global_settings.tagcache_autoupdate)
4489                 {
4490                     tagcache_build("/");
4491
4492                     /* This will be very slow unless dircache is enabled
4493                        or target is flash based, but do it anyway for
4494                        consistency. */
4495                     check_deleted_files();
4496                 }
4497
4498                 logf("tagcache check done");
4499
4500                 check_done = true;
4501                 break ;
4502
4503             case Q_STOP_SCAN:
4504                 break ;
4505
4506             case SYS_POWEROFF:
4507                 break ;
4508
4509 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4510             case SYS_USB_CONNECTED:
4511                 logf("USB: TagCache");
4512                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4513                 usb_wait_for_disconnect(&tagcache_queue);
4514                 break ;
4515 #endif
4516         }
4517     }
4518 }
4519
4520 bool tagcache_prepare_shutdown(void)
4521 {
4522     if (tagcache_get_commit_step() > 0)
4523         return false;
4524
4525     tagcache_stop_scan();
4526     while (read_lock || write_lock)
4527         sleep(1);
4528
4529     return true;
4530 }
4531
4532 void tagcache_shutdown(void)
4533 {
4534     /* Flush the command queue. */
4535     run_command_queue(true);
4536
4537 #ifdef HAVE_EEPROM_SETTINGS
4538     if (tc_stat.ramcache)
4539         tagcache_dumpsave();
4540 #endif
4541 }
4542
4543 static int get_progress(void)
4544 {
4545     int total_count = -1;
4546
4547 #ifdef HAVE_DIRCACHE
4548     if (dircache_is_enabled())
4549     {
4550         total_count = dircache_get_entry_count();
4551     }
4552     else
4553 #endif
4554 #ifdef HAVE_TC_RAMCACHE
4555     {
4556         if (hdr && tc_stat.ramcache)
4557             total_count = hdr->h.tch.entry_count;
4558     }
4559 #endif
4560
4561     if (total_count < 0)
4562         return -1;
4563
4564     return processed_dir_count * 100 / total_count;
4565 }
4566
4567 struct tagcache_stat* tagcache_get_stat(void)
4568 {
4569     tc_stat.progress = get_progress();
4570     tc_stat.processed_entries = processed_dir_count;
4571
4572     return &tc_stat;
4573 }
4574
4575 void tagcache_start_scan(void)
4576 {
4577     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4578 }
4579
4580 bool tagcache_update(void)
4581 {
4582     if (!tc_stat.ready)
4583         return false;
4584
4585     queue_post(&tagcache_queue, Q_UPDATE, 0);
4586     return false;
4587 }
4588
4589 bool tagcache_rebuild()
4590 {
4591     queue_post(&tagcache_queue, Q_REBUILD, 0);
4592     return false;
4593 }
4594
4595 void tagcache_stop_scan(void)
4596 {
4597     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4598 }
4599
4600 #ifdef HAVE_TC_RAMCACHE
4601 bool tagcache_is_ramcache(void)
4602 {
4603     return tc_stat.ramcache;
4604 }
4605 #endif
4606
4607 #endif /* !__PCTOOL__ */
4608
4609
4610 void tagcache_init(void)
4611 {
4612     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4613     memset(&current_tcmh, 0, sizeof(struct master_header));
4614     filenametag_fd = -1;
4615     write_lock = read_lock = 0;
4616
4617 #ifndef __PCTOOL__
4618     mutex_init(&command_queue_mutex);
4619     queue_init(&tagcache_queue, true);
4620     create_thread(tagcache_thread, tagcache_stack,
4621                   sizeof(tagcache_stack), 0, tagcache_thread_name
4622                   IF_PRIO(, PRIORITY_BACKGROUND)
4623                   IF_COP(, CPU));
4624 #else
4625     tc_stat.initialized = true;
4626     allocate_tempbuf();
4627     commit();
4628     free_tempbuf();
4629     tc_stat.ready = check_all_headers();
4630 #endif
4631 }
4632
4633 #ifdef __PCTOOL__
4634 void tagcache_reverse_scan(void)
4635 {
4636     logf("Checking for deleted files");
4637     check_deleted_files();
4638 }
4639 #endif
4640
4641 bool tagcache_is_initialized(void)
4642 {
4643     return tc_stat.initialized;
4644 }
4645 bool tagcache_is_usable(void)
4646 {
4647     return tc_stat.initialized && tc_stat.ready;
4648 }
4649 int tagcache_get_commit_step(void)
4650 {
4651     return tc_stat.commit_step;
4652 }
4653 int tagcache_get_max_commit_step(void)
4654 {
4655     return (int)(SORTED_TAGS_COUNT)+1;
4656 }
4657