apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "filefuncs.h"
  77 #include "settings.h"
  78 #include "dir.h"
  79 #include "structec.h"
  80
  81 #ifndef __PCTOOL__
  82 #include "lang.h"
  83 #include "eeprom_settings.h"
  84 #endif
  85
  86 #ifdef __PCTOOL__
  87 #define yield() do { } while(0)
  88 #define sim_sleep(timeout) do { } while(0)
  89 #define do_timed_yield() do { } while(0)
  90 #endif
  91
  92 #ifndef __PCTOOL__
  93 /* Tag Cache thread. */
  94 static struct event_queue tagcache_queue;
  95 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  96 static const char tagcache_thread_name[] = "tagcache";
  97 #endif
  98
  99 /* Previous path when scanning directory tree recursively. */
 100 static char curpath[TAG_MAXLEN+32];
 101
 102 /* Used when removing duplicates. */
 103 static char *tempbuf;     /* Allocated when needed. */
 104 static long tempbufidx;   /* Current location in buffer. */
 105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 106 static long tempbuf_left; /* Buffer space left. */
 107 static long tempbuf_pos;
 108
 109 #define SORTED_TAGS_COUNT 8
 110 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 111 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 112 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 113     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 114 /* Tags we want to get sorted (loaded to the tempbuf). */
 115 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 116     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 117     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 118
 119 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 120 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping))
 123
 124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 125 static const char *tags_str[] = { "artist", "album", "genre", "title",
 126     "filename", "composer", "comment", "albumartist", "grouping", "year",
 127     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 128     "playtime", "lastplayed", "commitid", "mtime" };
 129
 130 /* Status information of the tagcache. */
 131 static struct tagcache_stat tc_stat;
 132
 133 /* Queue commands. */
 134 enum tagcache_queue {
 135     Q_STOP_SCAN = 0,
 136     Q_START_SCAN,
 137     Q_IMPORT_CHANGELOG,
 138     Q_UPDATE,
 139     Q_REBUILD,
 140
 141     /* Internal tagcache command queue. */
 142     CMD_UPDATE_MASTER_HEADER,
 143     CMD_UPDATE_NUMERIC,
 144 };
 145
 146 struct tagcache_command_entry {
 147     int32_t command;
 148     int32_t idx_id;
 149     int32_t tag;
 150     int32_t data;
 151 };
 152
 153 #ifndef __PCTOOL__
 154 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 155 static volatile int command_queue_widx = 0;
 156 static volatile int command_queue_ridx = 0;
 157 static struct mutex command_queue_mutex;
 158 #endif
 159
 160 /* Tag database structures. */
 161
 162 /* Variable-length tag entry in tag files. */
 163 struct tagfile_entry {
 164     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 165     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 166     char tag_data[0];  /* Begin of the tag data */
 167 };
 168
 169 /* Fixed-size tag entry in master db index. */
 170 struct index_entry {
 171     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 172     int32_t flag;                /* Status flags */
 173 };
 174
 175 /* Header is the same in every file. */
 176 struct tagcache_header {
 177     int32_t magic;       /* Header version number */
 178     int32_t datasize;    /* Data size in bytes */
 179     int32_t entry_count; /* Number of entries in this file */
 180 };
 181
 182 struct master_header {
 183     struct tagcache_header tch;
 184     int32_t serial; /* Increasing counting number */
 185     int32_t commitid; /* Number of commits so far */
 186     int32_t dirty;
 187 };
 188
 189 /* For the endianess correction */
 190 static const char *tagfile_entry_ec   = "ll";
 191 /**
 192  Note: This should be (1 + TAG_COUNT) amount of l's.
 193  */
 194 static const char *index_entry_ec     = "lllllllllllllllllllll";
 195
 196 static const char *tagcache_header_ec = "lll";
 197 static const char *master_header_ec   = "llllll";
 198
 199 static struct master_header current_tcmh;
 200
 201 #ifdef HAVE_TC_RAMCACHE
 202 /* Header is created when loading database to ram. */
 203 struct ramcache_header {
 204     struct master_header h;      /* Header from the master index */
 205     struct index_entry *indices; /* Master index file content */
 206     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 207     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 208 };
 209
 210 # ifdef HAVE_EEPROM_SETTINGS
 211 struct statefile_header {
 212     struct ramcache_header *hdr;
 213     struct tagcache_stat tc_stat;
 214 };
 215 # endif
 216
 217 /* Pointer to allocated ramcache_header */
 218 static struct ramcache_header *hdr;
 219 #endif
 220
 221 /**
 222  * Full tag entries stored in a temporary file waiting
 223  * for commit to the cache. */
 224 struct temp_file_entry {
 225     long tag_offset[TAG_COUNT];
 226     short tag_length[TAG_COUNT];
 227     long flag;
 228
 229     long data_length;
 230 };
 231
 232 struct tempbuf_id_list {
 233     long id;
 234     struct tempbuf_id_list *next;
 235 };
 236
 237 struct tempbuf_searchidx {
 238     long idx_id;
 239     char *str;
 240     int seek;
 241     struct tempbuf_id_list idlist;
 242 };
 243
 244 /* Lookup buffer for fixing messed up index while after sorting. */
 245 static long commit_entry_count;
 246 static long lookup_buffer_depth;
 247 static struct tempbuf_searchidx **lookup;
 248
 249 /* Used when building the temporary file. */
 250 static int cachefd = -1, filenametag_fd;
 251 static int total_entry_count = 0;
 252 static int data_size = 0;
 253 static int processed_dir_count;
 254
 255 /* Thread safe locking */
 256 static volatile int write_lock;
 257 static volatile int read_lock;
 258
 259 static bool delete_entry(long idx_id);
 260
 261 const char* tagcache_tag_to_str(int tag)
 262 {
 263     return tags_str[tag];
 264 }
 265
 266 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 267 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 268 {
 269     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 270 }
 271
 272 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 #ifdef HAVE_DIRCACHE
 283 /**
 284  * Returns true if specified flag is still present, i.e., dircache
 285  * has not been reloaded.
 286  */
 287 static bool is_dircache_intact(void)
 288 {
 289     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 290 }
 291 #endif
 292
 293 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 294 {
 295     int fd;
 296     char buf[MAX_PATH], path[MAX_PATH];
 297     const char * file;
 298     int rc;
 299
 300     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 301         return -1;
 302
 303     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 304     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
 305
 306     fd = open(file, write ? O_RDWR : O_RDONLY);
 307     if (fd < 0)
 308     {
 309         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 310         tc_stat.ready = false;
 311         return fd;
 312     }
 313
 314     /* Check the header. */
 315     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 316     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 317     {
 318         logf("header error");
 319         tc_stat.ready = false;
 320         close(fd);
 321         return -2;
 322     }
 323
 324     return fd;
 325 }
 326
 327 static int open_master_fd(struct master_header *hdr, bool write)
 328 {
 329     int fd;
 330     int rc;
 331     char path[MAX_PATH];
 332
 333     fd = open(get_user_file_path(TAGCACHE_FILE_MASTER,
 334                                  IS_FILE|NEED_WRITE,
 335                                  path, sizeof(path)),
 336                     write ? O_RDWR : O_RDONLY);
 337     if (fd < 0)
 338     {
 339         logf("master file open failed for R/W");
 340         tc_stat.ready = false;
 341         return fd;
 342     }
 343
 344     tc_stat.econ = false;
 345
 346     /* Check the header. */
 347     rc = read(fd, hdr, sizeof(struct master_header));
 348     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 349     {
 350         /* Success. */
 351         return fd;
 352     }
 353
 354     /* Trying to read again, this time with endianess correction enabled. */
 355     lseek(fd, 0, SEEK_SET);
 356
 357     rc = ecread(fd, hdr, 1, master_header_ec, true);
 358     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 359     {
 360         logf("header error");
 361         tc_stat.ready = false;
 362         close(fd);
 363         return -2;
 364     }
 365
 366     tc_stat.econ = true;
 367
 368     return fd;
 369 }
 370
 371 #ifndef __PCTOOL__
 372 static bool do_timed_yield(void)
 373 {
 374     /* Sorting can lock up for quite a while, so yield occasionally */
 375     static long wakeup_tick = 0;
 376     if (TIME_AFTER(current_tick, wakeup_tick))
 377     {
 378         wakeup_tick = current_tick + (HZ/4);
 379         yield();
 380         return true;
 381     }
 382     return false;
 383 }
 384 #endif
 385
 386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 387 static long find_entry_ram(const char *filename,
 388                            const struct dircache_entry *dc)
 389 {
 390     static long last_pos = 0;
 391     int i;
 392
 393     /* Check if we tagcache is loaded into ram. */
 394     if (!tc_stat.ramcache)
 395         return -1;
 396
 397     if (dc == NULL)
 398         dc = dircache_get_entry_ptr(filename);
 399
 400     if (dc == NULL)
 401     {
 402         logf("tagcache: file not found.");
 403         return -1;
 404     }
 405
 406     try_again:
 407
 408     if (last_pos > 0)
 409         i = last_pos;
 410     else
 411         i = 0;
 412
 413     for (; i < hdr->h.tch.entry_count; i++)
 414     {
 415         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 416         {
 417             last_pos = MAX(0, i - 3);
 418             return i;
 419         }
 420
 421         do_timed_yield();
 422     }
 423
 424     if (last_pos > 0)
 425     {
 426         last_pos = 0;
 427         goto try_again;
 428     }
 429
 430     return -1;
 431 }
 432 #endif
 433
 434 static long find_entry_disk(const char *filename, bool localfd)
 435 {
 436     struct tagcache_header tch;
 437     static long last_pos = -1;
 438     long pos_history[POS_HISTORY_COUNT];
 439     long pos_history_idx = 0;
 440     bool found = false;
 441     struct tagfile_entry tfe;
 442     int fd;
 443     char buf[TAG_MAXLEN+32];
 444     int i;
 445     int pos = -1;
 446
 447     if (!tc_stat.ready)
 448         return -2;
 449
 450     fd = filenametag_fd;
 451     if (fd < 0 || localfd)
 452     {
 453         last_pos = -1;
 454         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 455             return -1;
 456     }
 457
 458     check_again:
 459
 460     if (last_pos > 0)
 461         lseek(fd, last_pos, SEEK_SET);
 462     else
 463         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 464
 465     while (true)
 466     {
 467         pos = lseek(fd, 0, SEEK_CUR);
 468         for (i = pos_history_idx-1; i >= 0; i--)
 469             pos_history[i+1] = pos_history[i];
 470         pos_history[0] = pos;
 471
 472         if (ecread_tagfile_entry(fd, &tfe)
 473             != sizeof(struct tagfile_entry))
 474         {
 475             break ;
 476         }
 477
 478         if (tfe.tag_length >= (long)sizeof(buf))
 479         {
 480             logf("too long tag #1");
 481             close(fd);
 482             if (!localfd)
 483                 filenametag_fd = -1;
 484             last_pos = -1;
 485             return -2;
 486         }
 487
 488         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 489         {
 490             logf("read error #2");
 491             close(fd);
 492             if (!localfd)
 493                 filenametag_fd = -1;
 494             last_pos = -1;
 495             return -3;
 496         }
 497
 498         if (!strcasecmp(filename, buf))
 499         {
 500             last_pos = pos_history[pos_history_idx];
 501             found = true;
 502             break ;
 503         }
 504
 505         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 506             pos_history_idx++;
 507     }
 508
 509     /* Not found? */
 510     if (!found)
 511     {
 512         if (last_pos > 0)
 513         {
 514             last_pos = -1;
 515             logf("seek again");
 516             goto check_again;
 517         }
 518
 519         if (fd != filenametag_fd || localfd)
 520             close(fd);
 521         return -4;
 522     }
 523
 524     if (fd != filenametag_fd || localfd)
 525         close(fd);
 526
 527     return tfe.idx_id;
 528 }
 529
 530 static int find_index(const char *filename)
 531 {
 532     long idx_id = -1;
 533
 534 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 535     if (tc_stat.ramcache && is_dircache_intact())
 536         idx_id = find_entry_ram(filename, NULL);
 537 #endif
 538
 539     if (idx_id < 0)
 540         idx_id = find_entry_disk(filename, true);
 541
 542     return idx_id;
 543 }
 544
 545 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 546 {
 547     int idx_id;
 548
 549     if (!tc_stat.ready)
 550         return false;
 551
 552     idx_id = find_index(filename);
 553     if (idx_id < 0)
 554         return false;
 555
 556     if (!tagcache_search(tcs, tag_filename))
 557         return false;
 558
 559     tcs->entry_count = 0;
 560     tcs->idx_id = idx_id;
 561
 562     return true;
 563 }
 564
 565 static bool get_index(int masterfd, int idxid,
 566                       struct index_entry *idx, bool use_ram)
 567 {
 568     bool localfd = false;
 569
 570     if (idxid < 0)
 571     {
 572         logf("Incorrect idxid: %d", idxid);
 573         return false;
 574     }
 575
 576 #ifdef HAVE_TC_RAMCACHE
 577     if (tc_stat.ramcache && use_ram)
 578     {
 579         if (hdr->indices[idxid].flag & FLAG_DELETED)
 580             return false;
 581
 582 # ifdef HAVE_DIRCACHE
 583         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 584             || is_dircache_intact())
 585 #endif
 586         {
 587             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 588             return true;
 589         }
 590     }
 591 #else
 592     (void)use_ram;
 593 #endif
 594
 595     if (masterfd < 0)
 596     {
 597         struct master_header tcmh;
 598
 599         localfd = true;
 600         masterfd = open_master_fd(&tcmh, false);
 601         if (masterfd < 0)
 602             return false;
 603     }
 604
 605     lseek(masterfd, idxid * sizeof(struct index_entry)
 606           + sizeof(struct master_header), SEEK_SET);
 607     if (ecread_index_entry(masterfd, idx)
 608         != sizeof(struct index_entry))
 609     {
 610         logf("read error #3");
 611         if (localfd)
 612             close(masterfd);
 613
 614         return false;
 615     }
 616
 617     if (localfd)
 618         close(masterfd);
 619
 620     if (idx->flag & FLAG_DELETED)
 621         return false;
 622
 623     return true;
 624 }
 625
 626 #ifndef __PCTOOL__
 627
 628 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 629 {
 630     /* We need to exclude all memory only flags & tags when writing to disk. */
 631     if (idx->flag & FLAG_DIRCACHE)
 632     {
 633         logf("memory only flags!");
 634         return false;
 635     }
 636
 637 #ifdef HAVE_TC_RAMCACHE
 638     /* Only update numeric data. Writing the whole index to RAM by memcpy
 639      * destroys dircache pointers!
 640      */
 641     if (tc_stat.ramcache)
 642     {
 643         int tag;
 644         struct index_entry *idx_ram = &hdr->indices[idxid];
 645
 646         for (tag = 0; tag < TAG_COUNT; tag++)
 647         {
 648             if (TAGCACHE_IS_NUMERIC(tag))
 649             {
 650                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 651             }
 652         }
 653
 654         /* Don't touch the dircache flag or attributes. */
 655         idx_ram->flag = (idx->flag & 0x0000ffff)
 656             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 657     }
 658 #endif
 659
 660     lseek(masterfd, idxid * sizeof(struct index_entry)
 661           + sizeof(struct master_header), SEEK_SET);
 662     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 663     {
 664         logf("write error #3");
 665         logf("idxid: %d", idxid);
 666         return false;
 667     }
 668
 669     return true;
 670 }
 671
 672 #endif /* !__PCTOOL__ */
 673
 674 static bool open_files(struct tagcache_search *tcs, int tag)
 675 {
 676     if (tcs->idxfd[tag] < 0)
 677     {
 678         char fn[MAX_PATH], path[MAX_PATH];
 679         const char *file;
 680
 681         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 682         file = get_user_file_path(fn, IS_FILE | NEED_WRITE, path, sizeof(path));
 683         tcs->idxfd[tag] = open(fn, O_RDONLY);
 684     }
 685
 686     if (tcs->idxfd[tag] < 0)
 687     {
 688         logf("File not open!");
 689         return false;
 690     }
 691
 692     return true;
 693 }
 694
 695 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 696                      int tag, char *buf, long size)
 697 {
 698     struct tagfile_entry tfe;
 699     long seek;
 700
 701     *buf = '\0';
 702
 703     if (TAGCACHE_IS_NUMERIC(tag))
 704         return false;
 705
 706     seek = idx->tag_seek[tag];
 707     if (seek < 0)
 708     {
 709         logf("Retrieve failed");
 710         return false;
 711     }
 712
 713 #ifdef HAVE_TC_RAMCACHE
 714     if (tcs->ramsearch)
 715     {
 716         struct tagfile_entry *ep;
 717
 718 # ifdef HAVE_DIRCACHE
 719         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 720             && is_dircache_intact())
 721         {
 722             dircache_copy_path((struct dircache_entry *)seek,
 723                                buf, size);
 724             return true;
 725         }
 726         else
 727 # endif
 728         if (tag != tag_filename)
 729         {
 730             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 731             strlcpy(buf, ep->tag_data, size);
 732
 733             return true;
 734         }
 735     }
 736 #endif
 737
 738     if (!open_files(tcs, tag))
 739         return false;
 740
 741     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 742     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 743         != sizeof(struct tagfile_entry))
 744     {
 745         logf("read error #5");
 746         return false;
 747     }
 748
 749     if (tfe.tag_length >= size)
 750     {
 751         logf("too small buffer");
 752         return false;
 753     }
 754
 755     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 756         tfe.tag_length)
 757     {
 758         logf("read error #6");
 759         return false;
 760     }
 761
 762     buf[tfe.tag_length] = '\0';
 763
 764     return true;
 765 }
 766
 767 static long check_virtual_tags(int tag, const struct index_entry *idx)
 768 {
 769     long data = 0;
 770
 771     switch (tag)
 772     {
 773         case tag_virt_length_sec:
 774             data = (idx->tag_seek[tag_length]/1000) % 60;
 775             break;
 776
 777         case tag_virt_length_min:
 778             data = (idx->tag_seek[tag_length]/1000) / 60;
 779             break;
 780
 781         case tag_virt_playtime_sec:
 782             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 783             break;
 784
 785         case tag_virt_playtime_min:
 786             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 787             break;
 788
 789         case tag_virt_autoscore:
 790             if (idx->tag_seek[tag_length] == 0
 791                 || idx->tag_seek[tag_playcount] == 0)
 792             {
 793                 data = 0;
 794             }
 795             else
 796             {
 797                 /* A straight calculus gives:
 798                      autoscore = 100 * playtime / length / playcout (1)
 799                    Now, consider the euclidian division of playtime by length:
 800                      playtime = alpha * length + beta
 801                    With:
 802                      0 <= beta < length
 803                    Now, (1) becomes:
 804                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 805                    Both terms should be small enough to avoid any overflow
 806                 */
 807                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 808                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 809                 data /=  idx->tag_seek[tag_playcount];
 810             }
 811             break;
 812
 813         /* How many commits before the file has been added to the DB. */
 814         case tag_virt_entryage:
 815             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 816             break;
 817
 818         default:
 819             data = idx->tag_seek[tag];
 820     }
 821
 822     return data;
 823 }
 824
 825 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 826 {
 827     struct index_entry idx;
 828
 829     if (!tc_stat.ready)
 830         return false;
 831
 832     if (!TAGCACHE_IS_NUMERIC(tag))
 833         return -1;
 834
 835     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 836         return -2;
 837
 838     return check_virtual_tags(tag, &idx);
 839 }
 840
 841 inline static bool str_ends_with(const char *str1, const char *str2)
 842 {
 843     int str_len = strlen(str1);
 844     int clause_len = strlen(str2);
 845
 846     if (clause_len > str_len)
 847         return false;
 848
 849     return !strcasecmp(&str1[str_len - clause_len], str2);
 850 }
 851
 852 inline static bool str_oneof(const char *str, const char *list)
 853 {
 854     const char *sep;
 855     int l, len = strlen(str);
 856
 857     while (*list)
 858     {
 859         sep = strchr(list, '|');
 860         l = sep ? (long)sep - (long)list : (int)strlen(list);
 861         if ((l==len) && !strncasecmp(str, list, len))
 862             return true;
 863         list += sep ? l + 1 : l;
 864     }
 865
 866     return false;
 867 }
 868
 869 static bool check_against_clause(long numeric, const char *str,
 870                                  const struct tagcache_search_clause *clause)
 871 {
 872     if (clause->numeric)
 873     {
 874         switch (clause->type)
 875         {
 876             case clause_is:
 877                 return numeric == clause->numeric_data;
 878             case clause_is_not:
 879                 return numeric != clause->numeric_data;
 880             case clause_gt:
 881                 return numeric > clause->numeric_data;
 882             case clause_gteq:
 883                 return numeric >= clause->numeric_data;
 884             case clause_lt:
 885                 return numeric < clause->numeric_data;
 886             case clause_lteq:
 887                 return numeric <= clause->numeric_data;
 888             default:
 889                 logf("Incorrect numeric tag: %d", clause->type);
 890         }
 891     }
 892     else
 893     {
 894         switch (clause->type)
 895         {
 896             case clause_is:
 897                 return !strcasecmp(clause->str, str);
 898             case clause_is_not:
 899                 return strcasecmp(clause->str, str);
 900             case clause_gt:
 901                 return 0>strcasecmp(clause->str, str);
 902             case clause_gteq:
 903                 return 0>=strcasecmp(clause->str, str);
 904             case clause_lt:
 905                 return 0<strcasecmp(clause->str, str);
 906             case clause_lteq:
 907                 return 0<=strcasecmp(clause->str, str);
 908             case clause_contains:
 909                 return (strcasestr(str, clause->str) != NULL);
 910             case clause_not_contains:
 911                 return (strcasestr(str, clause->str) == NULL);
 912             case clause_begins_with:
 913                 return (strcasestr(str, clause->str) == str);
 914             case clause_not_begins_with:
 915                 return (strcasestr(str, clause->str) != str);
 916             case clause_ends_with:
 917                 return str_ends_with(str, clause->str);
 918             case clause_not_ends_with:
 919                 return !str_ends_with(str, clause->str);
 920             case clause_oneof:
 921                 return str_oneof(str, clause->str);
 922
 923             default:
 924                 logf("Incorrect tag: %d", clause->type);
 925         }
 926     }
 927
 928     return false;
 929 }
 930
 931 static bool check_clauses(struct tagcache_search *tcs,
 932                           struct index_entry *idx,
 933                           struct tagcache_search_clause **clause, int count)
 934 {
 935     int i;
 936
 937 #ifdef HAVE_TC_RAMCACHE
 938     if (tcs->ramsearch)
 939     {
 940         /* Go through all conditional clauses. */
 941         for (i = 0; i < count; i++)
 942         {
 943             struct tagfile_entry *tfe;
 944             int seek;
 945             char buf[256];
 946             char *str = NULL;
 947
 948             seek = check_virtual_tags(clause[i]->tag, idx);
 949
 950             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 951             {
 952                 if (clause[i]->tag == tag_filename)
 953                 {
 954                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 955                     str = buf;
 956                 }
 957                 else
 958                 {
 959                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 960                     str = tfe->tag_data;
 961                 }
 962             }
 963
 964             if (!check_against_clause(seek, str, clause[i]))
 965                 return false;
 966         }
 967     }
 968     else
 969 #endif
 970     {
 971         /* Check for conditions. */
 972         for (i = 0; i < count; i++)
 973         {
 974             struct tagfile_entry tfe;
 975             int seek;
 976             char str[256];
 977
 978             seek = check_virtual_tags(clause[i]->tag, idx);
 979
 980             memset(str, 0, sizeof str);
 981             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 982             {
 983                 int fd = tcs->idxfd[clause[i]->tag];
 984                 lseek(fd, seek, SEEK_SET);
 985                 ecread_tagfile_entry(fd, &tfe);
 986                 if (tfe.tag_length >= (int)sizeof(str))
 987                 {
 988                     logf("Too long tag read!");
 989                     break ;
 990                 }
 991
 992                 read(fd, str, tfe.tag_length);
 993
 994                 /* Check if entry has been deleted. */
 995                 if (str[0] == '\0')
 996                     break;
 997             }
 998
 999             if (!check_against_clause(seek, str, clause[i]))
1000                 return false;
1001         }
1002     }
1003
1004     return true;
1005 }
1006
1007 bool tagcache_check_clauses(struct tagcache_search *tcs,
1008                             struct tagcache_search_clause **clause, int count)
1009 {
1010     struct index_entry idx;
1011
1012     if (count == 0)
1013         return true;
1014
1015     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1016         return false;
1017
1018     return check_clauses(tcs, &idx, clause, count);
1019 }
1020
1021 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1022 {
1023     int i;
1024
1025     /* If uniq buffer is not defined we must return true for search to work. */
1026     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1027         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1028     {
1029         return true;
1030     }
1031
1032     for (i = 0; i < tcs->unique_list_count; i++)
1033     {
1034         /* Return false if entry is found. */
1035         if (tcs->unique_list[i] == id)
1036             return false;
1037     }
1038
1039     if (tcs->unique_list_count < tcs->unique_list_capacity)
1040     {
1041         tcs->unique_list[i] = id;
1042         tcs->unique_list_count++;
1043     }
1044
1045     return true;
1046 }
1047
1048 static bool build_lookup_list(struct tagcache_search *tcs)
1049 {
1050     struct index_entry entry;
1051     int i, j;
1052
1053     tcs->seek_list_count = 0;
1054
1055 #ifdef HAVE_TC_RAMCACHE
1056     if (tcs->ramsearch
1057 # ifdef HAVE_DIRCACHE
1058         && (tcs->type != tag_filename || is_dircache_intact())
1059 # endif
1060         )
1061     {
1062         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1063         {
1064             struct tagcache_seeklist_entry *seeklist;
1065             struct index_entry *idx = &hdr->indices[i];
1066             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1067                 break ;
1068
1069             /* Skip deleted files. */
1070             if (idx->flag & FLAG_DELETED)
1071                 continue;
1072
1073             /* Go through all filters.. */
1074             for (j = 0; j < tcs->filter_count; j++)
1075             {
1076                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1077                 {
1078                     break ;
1079                 }
1080             }
1081
1082             if (j < tcs->filter_count)
1083                 continue ;
1084
1085             /* Check for conditions. */
1086             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1087                 continue;
1088
1089             /* Add to the seek list if not already in uniq buffer. */
1090             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1091                 continue;
1092
1093             /* Lets add it. */
1094             seeklist = &tcs->seeklist[tcs->seek_list_count];
1095             seeklist->seek = idx->tag_seek[tcs->type];
1096             seeklist->flag = idx->flag;
1097             seeklist->idx_id = i;
1098             tcs->seek_list_count++;
1099         }
1100
1101         tcs->seek_pos = i;
1102
1103         return tcs->seek_list_count > 0;
1104     }
1105 #endif
1106
1107     if (tcs->masterfd < 0)
1108     {
1109         struct master_header tcmh;
1110         tcs->masterfd = open_master_fd(&tcmh, false);
1111     }
1112
1113     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1114             sizeof(struct master_header), SEEK_SET);
1115
1116     while (ecread_index_entry(tcs->masterfd, &entry)
1117            == sizeof(struct index_entry))
1118     {
1119         struct tagcache_seeklist_entry *seeklist;
1120
1121         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1122             break ;
1123
1124         i = tcs->seek_pos;
1125         tcs->seek_pos++;
1126
1127         /* Check if entry has been deleted. */
1128         if (entry.flag & FLAG_DELETED)
1129             continue;
1130
1131         /* Go through all filters.. */
1132         for (j = 0; j < tcs->filter_count; j++)
1133         {
1134             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1135                 break ;
1136         }
1137
1138         if (j < tcs->filter_count)
1139             continue ;
1140
1141         /* Check for conditions. */
1142         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1143             continue;
1144
1145         /* Add to the seek list if not already in uniq buffer. */
1146         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1147             continue;
1148
1149         /* Lets add it. */
1150         seeklist = &tcs->seeklist[tcs->seek_list_count];
1151         seeklist->seek = entry.tag_seek[tcs->type];
1152         seeklist->flag = entry.flag;
1153         seeklist->idx_id = i;
1154         tcs->seek_list_count++;
1155
1156         yield();
1157     }
1158
1159     return tcs->seek_list_count > 0;
1160 }
1161
1162
1163 static void remove_files(void)
1164 {
1165     int i;
1166     char buf[MAX_PATH];
1167
1168     tc_stat.ready = false;
1169     tc_stat.ramcache = false;
1170     tc_stat.econ = false;
1171     remove(get_user_file_path(TAGCACHE_FILE_MASTER, NEED_WRITE|IS_FILE,
1172                               buf, sizeof(buf)));
1173     for (i = 0; i < TAG_COUNT; i++)
1174     {
1175         char buf2[MAX_PATH];
1176         if (TAGCACHE_IS_NUMERIC(i))
1177             continue;
1178
1179         /* database_%d.tcd -> database_0.tcd */
1180         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1181         remove(get_user_file_path(buf, NEED_WRITE | IS_FILE, buf2, sizeof(buf2)));
1182     }
1183 }
1184
1185
1186 static bool check_all_headers(void)
1187 {
1188     struct master_header myhdr;
1189     struct tagcache_header tch;
1190     int tag;
1191     int fd;
1192
1193     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1194         return false;
1195
1196     close(fd);
1197     if (myhdr.dirty)
1198     {
1199         logf("tagcache is dirty!");
1200         return false;
1201     }
1202
1203     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1204
1205     for (tag = 0; tag < TAG_COUNT; tag++)
1206     {
1207         if (TAGCACHE_IS_NUMERIC(tag))
1208             continue;
1209
1210         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1211             return false;
1212
1213         close(fd);
1214     }
1215
1216     return true;
1217 }
1218
1219 bool tagcache_is_busy(void)
1220 {
1221     return read_lock || write_lock;
1222 }
1223
1224 bool tagcache_search(struct tagcache_search *tcs, int tag)
1225 {
1226     struct tagcache_header tag_hdr;
1227     struct master_header   master_hdr;
1228     int i;
1229
1230     while (read_lock)
1231         sleep(1);
1232
1233     memset(tcs, 0, sizeof(struct tagcache_search));
1234     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1235         return false;
1236
1237     tcs->position = sizeof(struct tagcache_header);
1238     tcs->type = tag;
1239     tcs->seek_pos = 0;
1240     tcs->list_position = 0;
1241     tcs->seek_list_count = 0;
1242     tcs->filter_count = 0;
1243     tcs->masterfd = -1;
1244
1245     for (i = 0; i < TAG_COUNT; i++)
1246         tcs->idxfd[i] = -1;
1247
1248 #ifndef HAVE_TC_RAMCACHE
1249     tcs->ramsearch = false;
1250 #else
1251     tcs->ramsearch = tc_stat.ramcache;
1252     if (tcs->ramsearch)
1253     {
1254         tcs->entry_count = hdr->entry_count[tcs->type];
1255     }
1256     else
1257 #endif
1258     {
1259         /* Always open as R/W so we can pass tcs to functions that modify data also
1260          * without failing. */
1261         tcs->masterfd = open_master_fd(&master_hdr, true);
1262         if (tcs->masterfd < 0)
1263             return false;
1264
1265         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1266         {
1267             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1268             if (tcs->idxfd[tcs->type] < 0)
1269                 return false;
1270
1271             tcs->entry_count = tag_hdr.entry_count;
1272         }
1273         else
1274         {
1275             tcs->entry_count = master_hdr.tch.entry_count;
1276         }
1277     }
1278
1279     tcs->valid = true;
1280     tcs->initialized = true;
1281     write_lock++;
1282
1283     return true;
1284 }
1285
1286 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1287                                  void *buffer, long length)
1288 {
1289     tcs->unique_list = (unsigned long *)buffer;
1290     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1291     tcs->unique_list_count = 0;
1292 }
1293
1294 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1295                                 int tag, int seek)
1296 {
1297     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1298         return false;
1299
1300     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1301         return false;
1302
1303     tcs->filter_tag[tcs->filter_count] = tag;
1304     tcs->filter_seek[tcs->filter_count] = seek;
1305     tcs->filter_count++;
1306
1307     return true;
1308 }
1309
1310 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1311                                 struct tagcache_search_clause *clause)
1312 {
1313     int i;
1314
1315     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1316     {
1317         logf("Too many clauses");
1318         return false;
1319     }
1320
1321     /* Check if there is already a similar filter in present (filters are
1322      * much faster than clauses).
1323      */
1324     for (i = 0; i < tcs->filter_count; i++)
1325     {
1326         if (tcs->filter_tag[i] == clause->tag)
1327             return true;
1328     }
1329
1330     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1331     {
1332         char buf[MAX_PATH], path[MAX_PATH];
1333         const char *file;
1334         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1335         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
1336         tcs->idxfd[clause->tag] = open(file, O_RDONLY);
1337     }
1338
1339     tcs->clause[tcs->clause_count] = clause;
1340     tcs->clause_count++;
1341
1342     return true;
1343 }
1344
1345 static bool get_next(struct tagcache_search *tcs)
1346 {
1347     static char buf[TAG_MAXLEN+32];
1348     struct tagfile_entry entry;
1349     long flag = 0;
1350
1351     if (!tcs->valid || !tc_stat.ready)
1352         return false;
1353
1354     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1355 #ifdef HAVE_TC_RAMCACHE
1356         && !tcs->ramsearch
1357 #endif
1358         )
1359         return false;
1360
1361     /* Relative fetch. */
1362     if (tcs->filter_count > 0 || tcs->clause_count > 0
1363         || TAGCACHE_IS_NUMERIC(tcs->type)
1364 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1365         /* We need to retrieve flag status for dircache. */
1366         || (tcs->ramsearch && tcs->type == tag_filename)
1367 #endif
1368         )
1369     {
1370         struct tagcache_seeklist_entry *seeklist;
1371
1372         /* Check for end of list. */
1373         if (tcs->list_position == tcs->seek_list_count)
1374         {
1375             tcs->list_position = 0;
1376
1377             /* Try to fetch more. */
1378             if (!build_lookup_list(tcs))
1379             {
1380                 tcs->valid = false;
1381                 return false;
1382             }
1383         }
1384
1385         seeklist = &tcs->seeklist[tcs->list_position];
1386         flag = seeklist->flag;
1387         tcs->position = seeklist->seek;
1388         tcs->idx_id = seeklist->idx_id;
1389         tcs->list_position++;
1390     }
1391     else
1392     {
1393         if (tcs->entry_count == 0)
1394         {
1395             tcs->valid = false;
1396             return false;
1397         }
1398
1399         tcs->entry_count--;
1400     }
1401
1402     tcs->result_seek = tcs->position;
1403
1404     if (TAGCACHE_IS_NUMERIC(tcs->type))
1405     {
1406         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1407         tcs->result = buf;
1408         tcs->result_len = strlen(buf) + 1;
1409         return true;
1410     }
1411
1412     /* Direct fetch. */
1413 #ifdef HAVE_TC_RAMCACHE
1414     if (tcs->ramsearch)
1415     {
1416
1417 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1418         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1419             && is_dircache_intact())
1420         {
1421             dircache_copy_path((struct dircache_entry *)tcs->position,
1422                                buf, sizeof buf);
1423             tcs->result = buf;
1424             tcs->result_len = strlen(buf) + 1;
1425             tcs->ramresult = false;
1426
1427             return true;
1428         }
1429         else
1430 #endif
1431         if (tcs->type != tag_filename)
1432         {
1433             struct tagfile_entry *ep;
1434
1435             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1436             tcs->result = ep->tag_data;
1437             tcs->result_len = strlen(tcs->result) + 1;
1438             tcs->idx_id = ep->idx_id;
1439             tcs->ramresult = true;
1440
1441             /* Increase position for the next run. This may get overwritten. */
1442             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1443
1444             return true;
1445         }
1446     }
1447 #endif
1448
1449     if (!open_files(tcs, tcs->type))
1450     {
1451         tcs->valid = false;
1452         return false;
1453     }
1454
1455     /* Seek stream to the correct position and continue to direct fetch. */
1456     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1457
1458     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1459     {
1460         logf("read error #5");
1461         tcs->valid = false;
1462         return false;
1463     }
1464
1465     if (entry.tag_length > (long)sizeof(buf))
1466     {
1467         tcs->valid = false;
1468         logf("too long tag #2");
1469         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1470         return false;
1471     }
1472
1473     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1474     {
1475         tcs->valid = false;
1476         logf("read error #4");
1477         return false;
1478     }
1479
1480     /**
1481      Update the position for the next read (this may be overridden
1482      if filters or clauses are being used).
1483      */
1484     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1485     tcs->result = buf;
1486     tcs->result_len = strlen(tcs->result) + 1;
1487     tcs->idx_id = entry.idx_id;
1488     tcs->ramresult = false;
1489
1490     return true;
1491 }
1492
1493 bool tagcache_get_next(struct tagcache_search *tcs)
1494 {
1495     while (get_next(tcs))
1496     {
1497         if (tcs->result_len > 1)
1498             return true;
1499     }
1500
1501     return false;
1502 }
1503
1504 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1505                        int tag, char *buf, long size)
1506 {
1507     struct index_entry idx;
1508
1509     *buf = '\0';
1510     if (!get_index(tcs->masterfd, idxid, &idx, true))
1511         return false;
1512
1513     return retrieve(tcs, &idx, tag, buf, size);
1514 }
1515
1516 static bool update_master_header(void)
1517 {
1518     struct master_header myhdr;
1519     int fd;
1520
1521     if (!tc_stat.ready)
1522         return false;
1523
1524     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1525         return false;
1526
1527     myhdr.serial = current_tcmh.serial;
1528     myhdr.commitid = current_tcmh.commitid;
1529     myhdr.dirty = current_tcmh.dirty;
1530
1531     /* Write it back */
1532     lseek(fd, 0, SEEK_SET);
1533     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1534     close(fd);
1535
1536 #ifdef HAVE_TC_RAMCACHE
1537     if (hdr)
1538     {
1539         hdr->h.serial = current_tcmh.serial;
1540         hdr->h.commitid = current_tcmh.commitid;
1541         hdr->h.dirty = current_tcmh.dirty;
1542     }
1543 #endif
1544
1545     return true;
1546 }
1547
1548 #if 0
1549
1550 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1551 {
1552     struct tagentry *entry;
1553
1554     if (tcs->type != tag_title)
1555         return ;
1556
1557     /* We will need reserve buffer for this. */
1558     if (tcs->ramcache)
1559     {
1560         struct tagfile_entry *ep;
1561
1562         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1563         tcs->seek_list[tcs->seek_list_count];
1564     }
1565
1566     entry = find_entry_ram();
1567
1568 }
1569 #endif
1570
1571 void tagcache_search_finish(struct tagcache_search *tcs)
1572 {
1573     int i;
1574
1575     if (!tcs->initialized)
1576         return;
1577
1578     if (tcs->masterfd >= 0)
1579     {
1580         close(tcs->masterfd);
1581         tcs->masterfd = -1;
1582     }
1583
1584     for (i = 0; i < TAG_COUNT; i++)
1585     {
1586         if (tcs->idxfd[i] >= 0)
1587         {
1588             close(tcs->idxfd[i]);
1589             tcs->idxfd[i] = -1;
1590         }
1591     }
1592
1593     tcs->ramsearch = false;
1594     tcs->valid = false;
1595     tcs->initialized = 0;
1596     if (write_lock > 0)
1597         write_lock--;
1598 }
1599
1600 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1601 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1602 {
1603     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1604 }
1605
1606 static long get_tag_numeric(const struct index_entry *entry, int tag)
1607 {
1608     return check_virtual_tags(tag, entry);
1609 }
1610
1611 static char* get_tag_string(const struct index_entry *entry, int tag)
1612 {
1613     char* s = get_tag(entry, tag)->tag_data;
1614     return strcmp(s, UNTAGGED) ? s : NULL;
1615 }
1616
1617 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1618 {
1619     struct index_entry *entry;
1620     int idx_id;
1621
1622     if (!tc_stat.ready || !tc_stat.ramcache)
1623         return false;
1624
1625     /* Find the corresponding entry in tagcache. */
1626     idx_id = find_entry_ram(filename, NULL);
1627     if (idx_id < 0)
1628         return false;
1629
1630     entry = &hdr->indices[idx_id];
1631
1632     memset(id3, 0, sizeof(struct mp3entry));
1633
1634     id3->title        = get_tag_string(entry, tag_title);
1635     id3->artist       = get_tag_string(entry, tag_artist);
1636     id3->album        = get_tag_string(entry, tag_album);
1637     id3->genre_string = get_tag_string(entry, tag_genre);
1638     id3->composer     = get_tag_string(entry, tag_composer);
1639     id3->comment      = get_tag_string(entry, tag_comment);
1640     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1641     id3->grouping     = get_tag_string(entry, tag_grouping);
1642
1643     id3->length     = get_tag_numeric(entry, tag_length);
1644     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1645     id3->rating     = get_tag_numeric(entry, tag_rating);
1646     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1647     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1648     id3->year       = get_tag_numeric(entry, tag_year);
1649
1650     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1651     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1652     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1653     if (id3->bitrate == 0)
1654         id3->bitrate = 1;
1655
1656     return true;
1657 }
1658 #endif
1659
1660 static inline void write_item(const char *item)
1661 {
1662     int len = strlen(item) + 1;
1663
1664     data_size += len;
1665     write(cachefd, item, len);
1666 }
1667
1668 static int check_if_empty(char **tag)
1669 {
1670     int length;
1671
1672     if (*tag == NULL || **tag == '\0')
1673     {
1674         *tag = UNTAGGED;
1675         return sizeof(UNTAGGED); /* Tag length */
1676     }
1677
1678     length = strlen(*tag);
1679     if (length > TAG_MAXLEN)
1680     {
1681         logf("over length tag: %s", *tag);
1682         length = TAG_MAXLEN;
1683         (*tag)[length] = '\0';
1684     }
1685
1686     return length + 1;
1687 }
1688
1689 #define ADD_TAG(entry,tag,data) \
1690     /* Adding tag */ \
1691     entry.tag_offset[tag] = offset; \
1692     entry.tag_length[tag] = check_if_empty(data); \
1693     offset += entry.tag_length[tag]
1694 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1695  * idea, as it uses lots of stack and is called from a recursive function
1696  * (check_dir).
1697  */
1698 static void __attribute__ ((noinline)) add_tagcache(char *path,
1699                                                     unsigned long mtime
1700 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1701                                                     ,const struct dircache_entry *dc
1702 #endif
1703                                                    )
1704 {
1705     struct mp3entry id3;
1706     struct temp_file_entry entry;
1707     bool ret;
1708     int fd;
1709     int idx_id = -1;
1710     char tracknumfix[3];
1711     int offset = 0;
1712     int path_length = strlen(path);
1713     bool has_albumartist;
1714     bool has_grouping;
1715
1716 #ifdef SIMULATOR
1717     /* Crude logging for the sim - to aid in debugging */
1718     int logfd = open(ROCKBOX_DIR "/database.log",
1719                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1720     if (logfd >= 0) {
1721         write(logfd, path, strlen(path));
1722         write(logfd, "\n", 1);
1723         close(logfd);
1724     }
1725 #endif
1726
1727     if (cachefd < 0)
1728         return ;
1729
1730     /* Check for overlength file path. */
1731     if (path_length > TAG_MAXLEN)
1732     {
1733         /* Path can't be shortened. */
1734         logf("Too long path: %s", path);
1735         return ;
1736     }
1737
1738     /* Check if the file is supported. */
1739     if (probe_file_format(path) == AFMT_UNKNOWN)
1740         return ;
1741
1742     /* Check if the file is already cached. */
1743 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1744     if (tc_stat.ramcache && is_dircache_intact())
1745     {
1746         idx_id = find_entry_ram(path, dc);
1747     }
1748 #endif
1749
1750     /* Be sure the entry doesn't exist. */
1751     if (filenametag_fd >= 0 && idx_id < 0)
1752         idx_id = find_entry_disk(path, false);
1753
1754     /* Check if file has been modified. */
1755     if (idx_id >= 0)
1756     {
1757         struct index_entry idx;
1758
1759         /* TODO: Mark that the index exists (for fast reverse scan) */
1760         //found_idx[idx_id/8] |= idx_id%8;
1761
1762         if (!get_index(-1, idx_id, &idx, true))
1763         {
1764             logf("failed to retrieve index entry");
1765             return ;
1766         }
1767
1768         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1769         {
1770             /* No changes to file. */
1771             return ;
1772         }
1773
1774         /* Metadata might have been changed. Delete the entry. */
1775         logf("Re-adding: %s", path);
1776         if (!delete_entry(idx_id))
1777         {
1778             logf("delete_entry failed: %d", idx_id);
1779             return ;
1780         }
1781     }
1782
1783     fd = open(path, O_RDONLY);
1784     if (fd < 0)
1785     {
1786         logf("open fail: %s", path);
1787         return ;
1788     }
1789
1790     memset(&id3, 0, sizeof(struct mp3entry));
1791     memset(&entry, 0, sizeof(struct temp_file_entry));
1792     memset(&tracknumfix, 0, sizeof(tracknumfix));
1793     ret = get_metadata(&id3, fd, path);
1794     close(fd);
1795
1796     if (!ret)
1797         return ;
1798
1799     logf("-> %s", path);
1800
1801     /* Generate track number if missing. */
1802     if (id3.tracknum <= 0)
1803     {
1804         const char *p = strrchr(path, '.');
1805
1806         if (p == NULL)
1807             p = &path[strlen(path)-1];
1808
1809         while (*p != '/')
1810         {
1811             if (isdigit(*p) && isdigit(*(p-1)))
1812             {
1813                 tracknumfix[1] = *p--;
1814                 tracknumfix[0] = *p;
1815                 break;
1816             }
1817             p--;
1818         }
1819
1820         if (tracknumfix[0] != '\0')
1821         {
1822             id3.tracknum = atoi(tracknumfix);
1823             /* Set a flag to indicate track number has been generated. */
1824             entry.flag |= FLAG_TRKNUMGEN;
1825         }
1826         else
1827         {
1828             /* Unable to generate track number. */
1829             id3.tracknum = -1;
1830         }
1831     }
1832
1833     /* Numeric tags */
1834     entry.tag_offset[tag_year] = id3.year;
1835     entry.tag_offset[tag_discnumber] = id3.discnum;
1836     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1837     entry.tag_offset[tag_length] = id3.length;
1838     entry.tag_offset[tag_bitrate] = id3.bitrate;
1839     entry.tag_offset[tag_mtime] = mtime;
1840
1841     /* String tags. */
1842     has_albumartist = id3.albumartist != NULL
1843         && strlen(id3.albumartist) > 0;
1844     has_grouping = id3.grouping != NULL
1845         && strlen(id3.grouping) > 0;
1846
1847     ADD_TAG(entry, tag_filename, &path);
1848     ADD_TAG(entry, tag_title, &id3.title);
1849     ADD_TAG(entry, tag_artist, &id3.artist);
1850     ADD_TAG(entry, tag_album, &id3.album);
1851     ADD_TAG(entry, tag_genre, &id3.genre_string);
1852     ADD_TAG(entry, tag_composer, &id3.composer);
1853     ADD_TAG(entry, tag_comment, &id3.comment);
1854     if (has_albumartist)
1855     {
1856         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1857     }
1858     else
1859     {
1860         ADD_TAG(entry, tag_albumartist, &id3.artist);
1861     }
1862     if (has_grouping)
1863     {
1864         ADD_TAG(entry, tag_grouping, &id3.grouping);
1865     }
1866     else
1867     {
1868         ADD_TAG(entry, tag_grouping, &id3.title);
1869     }
1870     entry.data_length = offset;
1871
1872     /* Write the header */
1873     write(cachefd, &entry, sizeof(struct temp_file_entry));
1874
1875     /* And tags also... Correct order is critical */
1876     write_item(path);
1877     write_item(id3.title);
1878     write_item(id3.artist);
1879     write_item(id3.album);
1880     write_item(id3.genre_string);
1881     write_item(id3.composer);
1882     write_item(id3.comment);
1883     if (has_albumartist)
1884     {
1885         write_item(id3.albumartist);
1886     }
1887     else
1888     {
1889         write_item(id3.artist);
1890     }
1891     if (has_grouping)
1892     {
1893         write_item(id3.grouping);
1894     }
1895     else
1896     {
1897         write_item(id3.title);
1898     }
1899     total_entry_count++;
1900 }
1901
1902 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1903 {
1904     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1905     int len = strlen(str)+1;
1906     int i;
1907     unsigned crc32;
1908     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1909     char buf[TAG_MAXLEN+32];
1910
1911     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1912         buf[i] = tolower(str[i]);
1913     buf[i] = '\0';
1914
1915     crc32 = crc_32(buf, i, 0xffffffff);
1916
1917     if (unique)
1918     {
1919         /* Check if the crc does not exist -> entry does not exist for sure. */
1920         for (i = 0; i < tempbufidx; i++)
1921         {
1922             if (crcbuf[-i] != crc32)
1923                 continue;
1924
1925             if (!strcasecmp(str, index[i].str))
1926             {
1927                 if (id < 0 || id >= lookup_buffer_depth)
1928                 {
1929                     logf("lookup buf overf.: %d", id);
1930                     return false;
1931                 }
1932
1933                 lookup[id] = &index[i];
1934                 return true;
1935             }
1936         }
1937     }
1938
1939     /* Insert to CRC buffer. */
1940     crcbuf[-tempbufidx] = crc32;
1941     tempbuf_left -= 4;
1942
1943     /* Insert it to the buffer. */
1944     tempbuf_left -= len;
1945     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1946         return false;
1947
1948     if (id >= lookup_buffer_depth)
1949     {
1950         logf("lookup buf overf. #2: %d", id);
1951         return false;
1952     }
1953
1954     if (id >= 0)
1955     {
1956         lookup[id] = &index[tempbufidx];
1957         index[tempbufidx].idlist.id = id;
1958     }
1959     else
1960         index[tempbufidx].idlist.id = -1;
1961
1962     index[tempbufidx].idlist.next = NULL;
1963     index[tempbufidx].idx_id = idx_id;
1964     index[tempbufidx].seek = -1;
1965     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1966     memcpy(index[tempbufidx].str, str, len);
1967     tempbuf_pos += len;
1968     tempbufidx++;
1969
1970     return true;
1971 }
1972
1973 static int compare(const void *p1, const void *p2)
1974 {
1975     do_timed_yield();
1976
1977     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1978     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1979
1980     if (strcmp(e1->str, UNTAGGED) == 0)
1981     {
1982         if (strcmp(e2->str, UNTAGGED) == 0)
1983             return 0;
1984         return -1;
1985     }
1986     else if (strcmp(e2->str, UNTAGGED) == 0)
1987         return 1;
1988
1989     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1990 }
1991
1992 static int tempbuf_sort(int fd)
1993 {
1994     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1995     struct tagfile_entry fe;
1996     int i;
1997     int length;
1998
1999     /* Generate reverse lookup entries. */
2000     for (i = 0; i < lookup_buffer_depth; i++)
2001     {
2002         struct tempbuf_id_list *idlist;
2003
2004         if (!lookup[i])
2005             continue;
2006
2007         if (lookup[i]->idlist.id == i)
2008             continue;
2009
2010         idlist = &lookup[i]->idlist;
2011         while (idlist->next != NULL)
2012             idlist = idlist->next;
2013
2014         tempbuf_left -= sizeof(struct tempbuf_id_list);
2015         if (tempbuf_left - 4 < 0)
2016             return -1;
2017
2018         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2019         if (tempbuf_pos & 0x03)
2020         {
2021             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2022             tempbuf_left -= 3;
2023             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2024         }
2025         tempbuf_pos += sizeof(struct tempbuf_id_list);
2026
2027         idlist = idlist->next;
2028         idlist->id = i;
2029         idlist->next = NULL;
2030
2031         do_timed_yield();
2032     }
2033
2034     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2035     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2036
2037     for (i = 0; i < tempbufidx; i++)
2038     {
2039         struct tempbuf_id_list *idlist = &index[i].idlist;
2040
2041         /* Fix the lookup list. */
2042         while (idlist != NULL)
2043         {
2044             if (idlist->id >= 0)
2045                 lookup[idlist->id] = &index[i];
2046             idlist = idlist->next;
2047         }
2048
2049         index[i].seek = lseek(fd, 0, SEEK_CUR);
2050         length = strlen(index[i].str) + 1;
2051         fe.tag_length = length;
2052         fe.idx_id = index[i].idx_id;
2053
2054         /* Check the chunk alignment. */
2055         if ((fe.tag_length + sizeof(struct tagfile_entry))
2056             % TAGFILE_ENTRY_CHUNK_LENGTH)
2057         {
2058             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2059                 ((fe.tag_length + sizeof(struct tagfile_entry))
2060                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2061         }
2062
2063 #ifdef TAGCACHE_STRICT_ALIGN
2064         /* Make sure the entry is long aligned. */
2065         if (index[i].seek & 0x03)
2066         {
2067             logf("tempbuf_sort: alignment error!");
2068             return -3;
2069         }
2070 #endif
2071
2072         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2073             sizeof(struct tagfile_entry))
2074         {
2075             logf("tempbuf_sort: write error #1");
2076             return -1;
2077         }
2078
2079         if (write(fd, index[i].str, length) != length)
2080         {
2081             logf("tempbuf_sort: write error #2");
2082             return -2;
2083         }
2084
2085         /* Write some padding. */
2086         if (fe.tag_length - length > 0)
2087             write(fd, "XXXXXXXX", fe.tag_length - length);
2088     }
2089
2090     return i;
2091 }
2092
2093 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2094 {
2095     if (id < 0 || id >= lookup_buffer_depth)
2096         return NULL;
2097
2098     return lookup[id];
2099 }
2100
2101
2102 inline static int tempbuf_find_location(int id)
2103 {
2104     struct tempbuf_searchidx *entry;
2105
2106     entry = tempbuf_locate(id);
2107     if (entry == NULL)
2108         return -1;
2109
2110     return entry->seek;
2111 }
2112
2113 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2114 {
2115     struct master_header tcmh;
2116     struct index_entry idx;
2117     int masterfd;
2118     int masterfd_pos;
2119     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2120     int max_entries;
2121     int entries_processed = 0;
2122     int i, j;
2123     char buf[TAG_MAXLEN];
2124
2125     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2126
2127     logf("Building numeric indices...");
2128     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2129
2130     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2131         return false;
2132
2133     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2134                          SEEK_CUR);
2135     if (masterfd_pos == filesize(masterfd))
2136     {
2137         logf("we can't append!");
2138         close(masterfd);
2139         return false;
2140     }
2141
2142     while (entries_processed < h->entry_count)
2143     {
2144         int count = MIN(h->entry_count - entries_processed, max_entries);
2145
2146         /* Read in as many entries as possible. */
2147         for (i = 0; i < count; i++)
2148         {
2149             struct temp_file_entry *tfe = &entrybuf[i];
2150             int datastart;
2151
2152             /* Read in numeric data. */
2153             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2154                 sizeof(struct temp_file_entry))
2155             {
2156                 logf("read fail #1");
2157                 close(masterfd);
2158                 return false;
2159             }
2160
2161             datastart = lseek(tmpfd, 0, SEEK_CUR);
2162
2163             /**
2164              * Read string data from the following tags:
2165              * - tag_filename
2166              * - tag_artist
2167              * - tag_album
2168              * - tag_title
2169              *
2170              * A crc32 hash is calculated from the read data
2171              * and stored back to the data offset field kept in memory.
2172              */
2173 #define tmpdb_read_string_tag(tag) \
2174     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2175     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2176     { \
2177         logf("read fail: buffer overflow"); \
2178         close(masterfd); \
2179         return false; \
2180     } \
2181     \
2182     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2183         tfe->tag_length[tag]) \
2184     { \
2185         logf("read fail #2"); \
2186         close(masterfd); \
2187         return false; \
2188     } \
2189     \
2190     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2191     lseek(tmpfd, datastart, SEEK_SET)
2192
2193             tmpdb_read_string_tag(tag_filename);
2194             tmpdb_read_string_tag(tag_artist);
2195             tmpdb_read_string_tag(tag_album);
2196             tmpdb_read_string_tag(tag_title);
2197
2198             /* Seek to the end of the string data. */
2199             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2200         }
2201
2202         /* Backup the master index position. */
2203         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2204         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2205
2206         /* Check if we can resurrect some deleted runtime statistics data. */
2207         for (i = 0; i < tcmh.tch.entry_count; i++)
2208         {
2209             /* Read the index entry. */
2210             if (ecread_index_entry(masterfd, &idx)
2211                 != sizeof(struct index_entry))
2212             {
2213                 logf("read fail #3");
2214                 close(masterfd);
2215                 return false;
2216             }
2217
2218             /**
2219              * Skip unless the entry is marked as being deleted
2220              * or the data has already been resurrected.
2221              */
2222             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2223                 continue;
2224
2225             /* Now try to match the entry. */
2226             /**
2227              * To succesfully match a song, the following conditions
2228              * must apply:
2229              *
2230              * For numeric fields: tag_length
2231              * - Full identical match is required
2232              *
2233              * If tag_filename matches, no further checking necessary.
2234              *
2235              * For string hashes: tag_artist, tag_album, tag_title
2236              * - Two of these must match
2237              */
2238             for (j = 0; j < count; j++)
2239             {
2240                 struct temp_file_entry *tfe = &entrybuf[j];
2241
2242                 /* Try to match numeric fields first. */
2243                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2244                     continue;
2245
2246                 /* Now it's time to do the hash matching. */
2247                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2248                 {
2249                     int match_count = 0;
2250
2251                     /* No filename match, check if we can match two other tags. */
2252 #define tmpdb_match(tag) \
2253     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2254         match_count++
2255
2256                     tmpdb_match(tag_artist);
2257                     tmpdb_match(tag_album);
2258                     tmpdb_match(tag_title);
2259
2260                     if (match_count < 2)
2261                     {
2262                         /* Still no match found, give up. */
2263                         continue;
2264                     }
2265                 }
2266
2267                 /* A match found, now copy & resurrect the statistical data. */
2268 #define tmpdb_copy_tag(tag) \
2269     tfe->tag_offset[tag] = idx.tag_seek[tag]
2270
2271                 tmpdb_copy_tag(tag_playcount);
2272                 tmpdb_copy_tag(tag_rating);
2273                 tmpdb_copy_tag(tag_playtime);
2274                 tmpdb_copy_tag(tag_lastplayed);
2275                 tmpdb_copy_tag(tag_commitid);
2276
2277                 /* Avoid processing this entry again. */
2278                 idx.flag |= FLAG_RESURRECTED;
2279
2280                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2281                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2282                 {
2283                     logf("masterfd writeback fail #1");
2284                     close(masterfd);
2285                     return false;
2286                 }
2287
2288                 logf("Entry resurrected");
2289             }
2290         }
2291
2292
2293         /* Restore the master index position. */
2294         lseek(masterfd, masterfd_pos, SEEK_SET);
2295
2296         /* Commit the data to the index. */
2297         for (i = 0; i < count; i++)
2298         {
2299             int loc = lseek(masterfd, 0, SEEK_CUR);
2300
2301             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2302             {
2303                 logf("read fail #3");
2304                 close(masterfd);
2305                 return false;
2306             }
2307
2308             for (j = 0; j < TAG_COUNT; j++)
2309             {
2310                 if (!TAGCACHE_IS_NUMERIC(j))
2311                     continue;
2312
2313                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2314             }
2315             idx.flag = entrybuf[i].flag;
2316
2317             if (idx.tag_seek[tag_commitid])
2318             {
2319                 /* Data has been resurrected. */
2320                 idx.flag |= FLAG_DIRTYNUM;
2321             }
2322             else if (tc_stat.ready && current_tcmh.commitid > 0)
2323             {
2324                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2325                 idx.flag |= FLAG_DIRTYNUM;
2326             }
2327
2328             /* Write back the updated index. */
2329             lseek(masterfd, loc, SEEK_SET);
2330             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2331             {
2332                 logf("write fail");
2333                 close(masterfd);
2334                 return false;
2335             }
2336         }
2337
2338         entries_processed += count;
2339         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2340     }
2341
2342     close(masterfd);
2343
2344     return true;
2345 }
2346
2347 /**
2348  * Return values:
2349  *     > 0   success
2350  *    == 0   temporary failure
2351  *     < 0   fatal error
2352  */
2353 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2354 {
2355     int i;
2356     struct tagcache_header tch;
2357     struct master_header   tcmh;
2358     struct index_entry idxbuf[IDX_BUF_DEPTH];
2359     int idxbuf_pos;
2360     char buf[TAG_MAXLEN+32], path[MAX_PATH];
2361     const char *file;
2362     int fd = -1, masterfd;
2363     bool error = false;
2364     int init;
2365     int masterfd_pos;
2366
2367     logf("Building index: %d", index_type);
2368
2369     /* Check the number of entries we need to allocate ram for. */
2370     commit_entry_count = h->entry_count + 1;
2371
2372     masterfd = open_master_fd(&tcmh, false);
2373     if (masterfd >= 0)
2374     {
2375         commit_entry_count += tcmh.tch.entry_count;
2376         close(masterfd);
2377     }
2378     else
2379         remove_files(); /* Just to be sure we are clean. */
2380
2381     /* Open the index file, which contains the tag names. */
2382     fd = open_tag_fd(&tch, index_type, true);
2383     if (fd >= 0)
2384     {
2385         logf("tch.datasize=%ld", tch.datasize);
2386         lookup_buffer_depth = 1 +
2387         /* First part */ commit_entry_count +
2388         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2389     }
2390     else
2391     {
2392         lookup_buffer_depth = 1 +
2393         /* First part */ commit_entry_count +
2394         /* Second part */ 0;
2395     }
2396
2397     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2398     logf("commit_entry_count=%ld", commit_entry_count);
2399
2400     /* Allocate buffer for all index entries from both old and new
2401      * tag files. */
2402     tempbufidx = 0;
2403     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2404
2405     /* Allocate lookup buffer. The first portion of commit_entry_count
2406      * contains the new tags in the temporary file and the second
2407      * part for locating entries already in the db.
2408      *
2409      *  New tags  Old tags
2410      * +---------+---------------------------+
2411      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2412      * +---------+---------------------------+
2413      *
2414      * Old tags are inserted to a temporary buffer with position:
2415      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2416      * And new tags with index:
2417      *     tempbuf_insert(idx, ...);
2418      *
2419      * The buffer is sorted and written into tag file:
2420      *     tempbuf_sort(...);
2421      * leaving master index locations messed up.
2422      *
2423      * That is fixed using the lookup buffer for old tags:
2424      *     new_seek = tempbuf_find_location(old_seek, ...);
2425      * and for new tags:
2426      *     new_seek = tempbuf_find_location(idx);
2427      */
2428     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2429     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2430     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2431
2432     /* And calculate the remaining data space used mainly for storing
2433      * tag data (strings). */
2434     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2435     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2436     {
2437         logf("Buffer way too small!");
2438         return 0;
2439     }
2440
2441     if (fd >= 0)
2442     {
2443         /**
2444          * If tag file contains unique tags (sorted index), we will load
2445          * it entirely into memory so we can resort it later for use with
2446          * chunked browsing.
2447          */
2448         if (TAGCACHE_IS_SORTED(index_type))
2449         {
2450             logf("loading tags...");
2451             for (i = 0; i < tch.entry_count; i++)
2452             {
2453                 struct tagfile_entry entry;
2454                 int loc = lseek(fd, 0, SEEK_CUR);
2455                 bool ret;
2456
2457                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2458                 {
2459                     logf("read error #7");
2460                     close(fd);
2461                     return -2;
2462                 }
2463
2464                 if (entry.tag_length >= (int)sizeof(buf))
2465                 {
2466                     logf("too long tag #3");
2467                     close(fd);
2468                     return -2;
2469                 }
2470
2471                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2472                 {
2473                     logf("read error #8");
2474                     close(fd);
2475                     return -2;
2476                 }
2477
2478                 /* Skip deleted entries. */
2479                 if (buf[0] == '\0')
2480                     continue;
2481
2482                 /**
2483                  * Save the tag and tag id in the memory buffer. Tag id
2484                  * is saved so we can later reindex the master lookup
2485                  * table when the index gets resorted.
2486                  */
2487                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2488                                      + commit_entry_count, entry.idx_id,
2489                                      TAGCACHE_IS_UNIQUE(index_type));
2490                 if (!ret)
2491                 {
2492                     close(fd);
2493                     return -3;
2494                 }
2495                 do_timed_yield();
2496             }
2497             logf("done");
2498         }
2499         else
2500             tempbufidx = tch.entry_count;
2501     }
2502     else
2503     {
2504         /**
2505          * Creating new index file to store the tags. No need to preload
2506          * anything whether the index type is sorted or not.
2507          */
2508         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2509         file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
2510         fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2511         if (fd < 0)
2512         {
2513             logf("%s open fail", buf);
2514             return -2;
2515         }
2516
2517         tch.magic = TAGCACHE_MAGIC;
2518         tch.entry_count = 0;
2519         tch.datasize = 0;
2520
2521         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2522             != sizeof(struct tagcache_header))
2523         {
2524             logf("header write failed");
2525             close(fd);
2526             return -2;
2527         }
2528     }
2529
2530     file = get_user_file_path(TAGCACHE_FILE_MASTER,
2531                                             IS_FILE|NEED_WRITE,
2532                                             buf, sizeof(buf));
2533     /* Loading the tag lookup file as "master file". */
2534     logf("Loading index file");
2535     masterfd = open(file, O_RDWR);
2536
2537     if (masterfd < 0)
2538     {
2539         logf("Creating new DB");
2540         masterfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2541
2542         if (masterfd < 0)
2543         {
2544             logf("Failure to create index file (%s)", file);
2545             close(fd);
2546             return -2;
2547         }
2548
2549         /* Write the header (write real values later). */
2550         memset(&tcmh, 0, sizeof(struct master_header));
2551         tcmh.tch = *h;
2552         tcmh.tch.entry_count = 0;
2553         tcmh.tch.datasize = 0;
2554         tcmh.dirty = true;
2555         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2556         init = true;
2557         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2558     }
2559     else
2560     {
2561         /**
2562          * Master file already exists so we need to process the current
2563          * file first.
2564          */
2565         init = false;
2566
2567         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2568             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2569         {
2570             logf("header error");
2571             close(fd);
2572             close(masterfd);
2573             return -2;
2574         }
2575
2576         /**
2577          * If we reach end of the master file, we need to expand it to
2578          * hold new tags. If the current index is not sorted, we can
2579          * simply append new data to end of the file.
2580          * However, if the index is sorted, we need to update all tag
2581          * pointers in the master file for the current index.
2582          */
2583         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2584             SEEK_CUR);
2585         if (masterfd_pos == filesize(masterfd))
2586         {
2587             logf("appending...");
2588             init = true;
2589         }
2590     }
2591
2592     /**
2593      * Load new unique tags in memory to be sorted later and added
2594      * to the master lookup file.
2595      */
2596     if (TAGCACHE_IS_SORTED(index_type))
2597     {
2598         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2599         /* h is the header of the temporary file containing new tags. */
2600         logf("inserting new tags...");
2601         for (i = 0; i < h->entry_count; i++)
2602         {
2603             struct temp_file_entry entry;
2604
2605             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2606                 sizeof(struct temp_file_entry))
2607             {
2608                 logf("read fail #3");
2609                 error = true;
2610                 goto error_exit;
2611             }
2612
2613             /* Read data. */
2614             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2615             {
2616                 logf("too long entry!");
2617                 error = true;
2618                 goto error_exit;
2619             }
2620
2621             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2622             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2623                 entry.tag_length[index_type])
2624             {
2625                 logf("read fail #4");
2626                 error = true;
2627                 goto error_exit;
2628             }
2629
2630             if (TAGCACHE_IS_UNIQUE(index_type))
2631                 error = !tempbuf_insert(buf, i, -1, true);
2632             else
2633                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2634
2635             if (error)
2636             {
2637                 logf("insert error");
2638                 goto error_exit;
2639             }
2640
2641             /* Skip to next. */
2642             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2643                     entry.tag_length[index_type], SEEK_CUR);
2644             do_timed_yield();
2645         }
2646         logf("done");
2647
2648         /* Sort the buffer data and write it to the index file. */
2649         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2650         /**
2651          * We need to truncate the index file now. There can be junk left
2652          * at the end of file (however, we _should_ always follow the
2653          * entry_count and don't crash with that).
2654          */
2655         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2656
2657         i = tempbuf_sort(fd);
2658         if (i < 0)
2659             goto error_exit;
2660         logf("sorted %d tags", i);
2661
2662         /**
2663          * Now update all indexes in the master lookup file.
2664          */
2665         logf("updating indices...");
2666         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2667         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2668         {
2669             int j;
2670             int loc = lseek(masterfd, 0, SEEK_CUR);
2671
2672             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2673
2674             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2675                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2676             {
2677                 logf("read fail #5");
2678                 error = true;
2679                 goto error_exit ;
2680             }
2681             lseek(masterfd, loc, SEEK_SET);
2682
2683             for (j = 0; j < idxbuf_pos; j++)
2684             {
2685                 if (idxbuf[j].flag & FLAG_DELETED)
2686                 {
2687                     /* We can just ignore deleted entries. */
2688                     // idxbuf[j].tag_seek[index_type] = 0;
2689                     continue;
2690                 }
2691
2692                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2693                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2694                     + commit_entry_count);
2695
2696                 if (idxbuf[j].tag_seek[index_type] < 0)
2697                 {
2698                     logf("update error: %ld/%d/%ld",
2699                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2700                     error = true;
2701                     goto error_exit;
2702                 }
2703
2704                 do_timed_yield();
2705             }
2706
2707             /* Write back the updated index. */
2708             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2709                         index_entry_ec, tc_stat.econ) !=
2710                 (int)sizeof(struct index_entry)*idxbuf_pos)
2711             {
2712                 logf("write fail");
2713                 error = true;
2714                 goto error_exit;
2715             }
2716         }
2717         logf("done");
2718     }
2719
2720     /**
2721      * Walk through the temporary file containing the new tags.
2722      */
2723     // build_normal_index(h, tmpfd, masterfd, idx);
2724     logf("updating new indices...");
2725     lseek(masterfd, masterfd_pos, SEEK_SET);
2726     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2727     lseek(fd, 0, SEEK_END);
2728     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2729     {
2730         int j;
2731
2732         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2733         if (init)
2734         {
2735             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2736         }
2737         else
2738         {
2739             int loc = lseek(masterfd, 0, SEEK_CUR);
2740
2741             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2742                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2743             {
2744                 logf("read fail #6");
2745                 error = true;
2746                 break ;
2747             }
2748             lseek(masterfd, loc, SEEK_SET);
2749         }
2750
2751         /* Read entry headers. */
2752         for (j = 0; j < idxbuf_pos; j++)
2753         {
2754             if (!TAGCACHE_IS_SORTED(index_type))
2755             {
2756                 struct temp_file_entry entry;
2757                 struct tagfile_entry fe;
2758
2759                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2760                     sizeof(struct temp_file_entry))
2761                 {
2762                     logf("read fail #7");
2763                     error = true;
2764                     break ;
2765                 }
2766
2767                 /* Read data. */
2768                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2769                 {
2770                     logf("too long entry!");
2771                     logf("length=%d", entry.tag_length[index_type]);
2772                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2773                     error = true;
2774                     break ;
2775                 }
2776
2777                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2778                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2779                     entry.tag_length[index_type])
2780                 {
2781                     logf("read fail #8");
2782                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2783                     logf("length=0x%02x", entry.tag_length[index_type]);
2784                     error = true;
2785                     break ;
2786                 }
2787
2788                 /* Write to index file. */
2789                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2790                 fe.tag_length = entry.tag_length[index_type];
2791                 fe.idx_id = tcmh.tch.entry_count + i + j;
2792                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2793                 write(fd, buf, fe.tag_length);
2794                 tempbufidx++;
2795
2796                 /* Skip to next. */
2797                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2798                       entry.tag_length[index_type], SEEK_CUR);
2799             }
2800             else
2801             {
2802                 /* Locate the correct entry from the sorted array. */
2803                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2804                 if (idxbuf[j].tag_seek[index_type] < 0)
2805                 {
2806                     logf("entry not found (%d)", j);
2807                     error = true;
2808                     break ;
2809                 }
2810             }
2811         }
2812
2813         /* Write index. */
2814         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2815                   index_entry_ec, tc_stat.econ) !=
2816             (int)sizeof(struct index_entry)*idxbuf_pos)
2817         {
2818             logf("tagcache: write fail #4");
2819             error = true;
2820             break ;
2821         }
2822
2823         do_timed_yield();
2824     }
2825     logf("done");
2826
2827     /* Finally write the header. */
2828     tch.magic = TAGCACHE_MAGIC;
2829     tch.entry_count = tempbufidx;
2830     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2831     lseek(fd, 0, SEEK_SET);
2832     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2833
2834     if (index_type != tag_filename)
2835         h->datasize += tch.datasize;
2836     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2837     error_exit:
2838
2839     close(fd);
2840     close(masterfd);
2841
2842     if (error)
2843         return -2;
2844
2845     return 1;
2846 }
2847
2848 static bool commit(void)
2849 {
2850     struct tagcache_header tch;
2851     struct master_header   tcmh;
2852     char path[MAX_PATH];
2853     const char *file;
2854     int i, len, rc;
2855     int tmpfd;
2856     int masterfd;
2857 #ifdef HAVE_DIRCACHE
2858     bool dircache_buffer_stolen = false;
2859 #endif
2860     bool local_allocation = false;
2861
2862     logf("committing tagcache");
2863
2864     while (write_lock)
2865         sleep(1);
2866
2867     file = get_user_file_path(TAGCACHE_FILE_TEMP,
2868                             IS_FILE|NEED_WRITE, path, sizeof(path));
2869
2870     tmpfd = open(file, O_RDONLY);
2871     if (tmpfd < 0)
2872     {
2873         logf("nothing to commit");
2874         return true;
2875     }
2876
2877
2878     /* Load the header. */
2879     len = sizeof(struct tagcache_header);
2880     rc = read(tmpfd, &tch, len);
2881
2882     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2883     {
2884         logf("incorrect tmpheader");
2885         close(tmpfd);
2886         remove(file);
2887         return false;
2888     }
2889
2890     if (tch.entry_count == 0)
2891     {
2892         logf("nothing to commit");
2893         close(tmpfd);
2894         remove(file);
2895         return true;
2896     }
2897
2898     /* Fully initialize existing headers (if any) before going further. */
2899     tc_stat.ready = check_all_headers();
2900
2901 #ifdef HAVE_EEPROM_SETTINGS
2902     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
2903                 path, sizeof(path)));
2904 #endif
2905
2906     /* At first be sure to unload the ramcache! */
2907 #ifdef HAVE_TC_RAMCACHE
2908     tc_stat.ramcache = false;
2909 #endif
2910
2911     read_lock++;
2912
2913     /* Try to steal every buffer we can :) */
2914     if (tempbuf_size == 0)
2915         local_allocation = true;
2916
2917 #ifdef HAVE_DIRCACHE
2918     if (tempbuf_size == 0)
2919     {
2920         /* Try to steal the dircache buffer. */
2921         tempbuf = dircache_steal_buffer(&tempbuf_size);
2922         tempbuf_size &= ~0x03;
2923
2924         if (tempbuf_size > 0)
2925         {
2926             dircache_buffer_stolen = true;
2927         }
2928     }
2929 #endif
2930
2931 #ifdef HAVE_TC_RAMCACHE
2932     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2933     {
2934         tempbuf = (char *)(hdr + 1);
2935         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2936         tempbuf_size &= ~0x03;
2937     }
2938 #endif
2939
2940     /* And finally fail if there are no buffers available. */
2941     if (tempbuf_size == 0)
2942     {
2943         logf("delaying commit until next boot");
2944         tc_stat.commit_delayed = true;
2945         close(tmpfd);
2946         read_lock--;
2947         return false;
2948     }
2949
2950     logf("commit %ld entries...", tch.entry_count);
2951
2952     /* Mark DB dirty so it will stay disabled if commit fails. */
2953     current_tcmh.dirty = true;
2954     update_master_header();
2955
2956     /* Now create the index files. */
2957     tc_stat.commit_step = 0;
2958     tch.datasize = 0;
2959     tc_stat.commit_delayed = false;
2960
2961     for (i = 0; i < TAG_COUNT; i++)
2962     {
2963         int ret;
2964
2965         if (TAGCACHE_IS_NUMERIC(i))
2966             continue;
2967
2968         tc_stat.commit_step++;
2969         ret = build_index(i, &tch, tmpfd);
2970         if (ret <= 0)
2971         {
2972             close(tmpfd);
2973             logf("tagcache failed init");
2974             if (ret == 0)
2975                 tc_stat.commit_delayed = true;
2976
2977             tc_stat.commit_step = 0;
2978             read_lock--;
2979             return false;
2980         }
2981     }
2982
2983     if (!build_numeric_indices(&tch, tmpfd))
2984     {
2985         logf("Failure to commit numeric indices");
2986         close(tmpfd);
2987         tc_stat.commit_step = 0;
2988         read_lock--;
2989         return false;
2990     }
2991
2992     close(tmpfd);
2993     remove(file);
2994
2995     tc_stat.commit_step = 0;
2996
2997     /* Update the master index headers. */
2998     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2999     {
3000         read_lock--;
3001         return false;
3002     }
3003
3004     tcmh.tch.entry_count += tch.entry_count;
3005     tcmh.tch.datasize = sizeof(struct master_header)
3006         + sizeof(struct index_entry) * tcmh.tch.entry_count
3007         + tch.datasize;
3008     tcmh.dirty = false;
3009     tcmh.commitid++;
3010
3011     lseek(masterfd, 0, SEEK_SET);
3012     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3013     close(masterfd);
3014
3015     logf("tagcache committed");
3016     tc_stat.ready = check_all_headers();
3017     tc_stat.readyvalid = true;
3018
3019     if (local_allocation)
3020     {
3021         tempbuf = NULL;
3022         tempbuf_size = 0;
3023     }
3024
3025 #ifdef HAVE_DIRCACHE
3026     /* Rebuild the dircache, if we stole the buffer. */
3027     if (dircache_buffer_stolen)
3028         dircache_build(0);
3029 #endif
3030
3031 #ifdef HAVE_TC_RAMCACHE
3032     /* Reload tagcache. */
3033     if (tc_stat.ramcache_allocated > 0)
3034         tagcache_start_scan();
3035 #endif
3036
3037     read_lock--;
3038
3039     return true;
3040 }
3041
3042 static void allocate_tempbuf(void)
3043 {
3044     /* Yeah, malloc would be really nice now :) */
3045 #ifdef __PCTOOL__
3046     tempbuf_size = 32*1024*1024;
3047     tempbuf = malloc(tempbuf_size);
3048 #else
3049     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3050     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3051     audiobuf += tempbuf_size;
3052 #endif
3053 }
3054
3055 static void free_tempbuf(void)
3056 {
3057     if (tempbuf_size == 0)
3058         return ;
3059
3060 #ifdef __PCTOOL__
3061     free(tempbuf);
3062 #else
3063     audiobuf -= tempbuf_size;
3064 #endif
3065     tempbuf = NULL;
3066     tempbuf_size = 0;
3067 }
3068
3069 #ifndef __PCTOOL__
3070
3071 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3072 {
3073     struct index_entry idx;
3074
3075     if (!tc_stat.ready)
3076         return false;
3077
3078     if (!TAGCACHE_IS_NUMERIC(tag))
3079         return false;
3080
3081     if (!get_index(masterfd, idx_id, &idx, false))
3082         return false;
3083
3084     idx.tag_seek[tag] = data;
3085     idx.flag |= FLAG_DIRTYNUM;
3086
3087     return write_index(masterfd, idx_id, &idx);
3088 }
3089
3090 #if 0
3091 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3092                                    int tag, long data)
3093 {
3094     struct master_header myhdr;
3095
3096     if (tcs->masterfd < 0)
3097     {
3098         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3099             return false;
3100     }
3101
3102     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3103 }
3104 #endif
3105
3106 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3107
3108 static bool command_queue_is_full(void)
3109 {
3110     int next;
3111
3112     next = command_queue_widx + 1;
3113     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3114         next = 0;
3115
3116     return (next == command_queue_ridx);
3117 }
3118
3119 static void command_queue_sync_callback(void *data)
3120 {
3121     (void)data;
3122     struct master_header myhdr;
3123     int masterfd;
3124
3125     mutex_lock(&command_queue_mutex);
3126
3127     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3128         return;
3129
3130     while (command_queue_ridx != command_queue_widx)
3131     {
3132         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3133
3134         switch (ce->command)
3135         {
3136             case CMD_UPDATE_MASTER_HEADER:
3137             {
3138                 close(masterfd);
3139                 update_master_header();
3140
3141                 /* Re-open the masterfd. */
3142                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3143                     return;
3144
3145                 break;
3146             }
3147             case CMD_UPDATE_NUMERIC:
3148             {
3149                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3150                 break;
3151             }
3152         }
3153
3154         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3155             command_queue_ridx = 0;
3156     }
3157
3158     close(masterfd);
3159
3160     tc_stat.queue_length = 0;
3161     mutex_unlock(&command_queue_mutex);
3162 }
3163
3164 static void run_command_queue(bool force)
3165 {
3166     if (COMMAND_QUEUE_IS_EMPTY)
3167         return;
3168
3169     if (force || command_queue_is_full())
3170         command_queue_sync_callback(NULL);
3171     else
3172         register_storage_idle_func(command_queue_sync_callback);
3173 }
3174
3175 static void queue_command(int cmd, long idx_id, int tag, long data)
3176 {
3177     while (1)
3178     {
3179         int next;
3180
3181         mutex_lock(&command_queue_mutex);
3182         next = command_queue_widx + 1;
3183         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3184             next = 0;
3185
3186         /* Make sure queue is not full. */
3187         if (next != command_queue_ridx)
3188         {
3189             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3190
3191             ce->command = cmd;
3192             ce->idx_id = idx_id;
3193             ce->tag = tag;
3194             ce->data = data;
3195
3196             command_queue_widx = next;
3197
3198             tc_stat.queue_length++;
3199
3200             mutex_unlock(&command_queue_mutex);
3201             break;
3202         }
3203
3204         /* Queue is full, try again later... */
3205         mutex_unlock(&command_queue_mutex);
3206         sleep(1);
3207     }
3208 }
3209
3210 long tagcache_increase_serial(void)
3211 {
3212     long old;
3213
3214     if (!tc_stat.ready)
3215         return -2;
3216
3217     while (read_lock)
3218         sleep(1);
3219
3220     old = current_tcmh.serial++;
3221     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3222
3223     return old;
3224 }
3225
3226 void tagcache_update_numeric(int idx_id, int tag, long data)
3227 {
3228     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3229 }
3230 #endif /* !__PCTOOL__ */
3231
3232 long tagcache_get_serial(void)
3233 {
3234     return current_tcmh.serial;
3235 }
3236
3237 long tagcache_get_commitid(void)
3238 {
3239     return current_tcmh.commitid;
3240 }
3241
3242 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3243 {
3244     char buf[512];
3245     int i;
3246
3247     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3248     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3249     {
3250         if (*datastr == '\0')
3251             break;
3252
3253         if (*datastr == '"' || *datastr == '\\')
3254             buf[i++] = '\\';
3255
3256         buf[i] = *(datastr++);
3257     }
3258
3259     strcpy(&buf[i], "\" ");
3260
3261     write(fd, buf, i + 2);
3262
3263     return true;
3264 }
3265
3266 #ifndef __PCTOOL__
3267
3268 static bool read_tag(char *dest, long size,
3269                      const char *src, const char *tagstr)
3270 {
3271     int pos;
3272     char current_tag[32];
3273
3274     while (*src != '\0')
3275     {
3276         /* Skip all whitespace */
3277         while (*src == ' ')
3278             src++;
3279
3280         if (*src == '\0')
3281             break;
3282
3283         pos = 0;
3284         /* Read in tag name */
3285         while (*src != '=' && *src != ' ')
3286         {
3287             current_tag[pos] = *src;
3288             src++;
3289             pos++;
3290
3291             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3292                 return false;
3293         }
3294         current_tag[pos] = '\0';
3295
3296         /* Read in tag data */
3297
3298         /* Find the start. */
3299         while (*src != '"' && *src != '\0')
3300             src++;
3301
3302         if (*src == '\0' || *(++src) == '\0')
3303             return false;
3304
3305         /* Read the data. */
3306         for (pos = 0; pos < size; pos++)
3307         {
3308             if (*src == '\0')
3309                 break;
3310
3311             if (*src == '\\')
3312             {
3313                 dest[pos] = *(src+1);
3314                 src += 2;
3315                 continue;
3316             }
3317
3318             dest[pos] = *src;
3319
3320             if (*src == '"')
3321             {
3322                 src++;
3323                 break;
3324             }
3325
3326             if (*src == '\0')
3327                 break;
3328
3329             src++;
3330         }
3331         dest[pos] = '\0';
3332
3333         if (!strcasecmp(tagstr, current_tag))
3334             return true;
3335     }
3336
3337     return false;
3338 }
3339
3340 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3341 {
3342     struct index_entry idx;
3343     char tag_data[TAG_MAXLEN+32];
3344     int idx_id;
3345     long masterfd = (long)parameters;
3346     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3347         tag_commitid };
3348     int i;
3349     (void)line_n;
3350
3351     if (*buf == '#')
3352         return 0;
3353
3354     logf("%d/%s", line_n, buf);
3355     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3356     {
3357         logf("filename missing");
3358         logf("-> %s", buf);
3359         return 0;
3360     }
3361
3362     idx_id = find_index(tag_data);
3363     if (idx_id < 0)
3364     {
3365         logf("entry not found");
3366         return 0;
3367     }
3368
3369     if (!get_index(masterfd, idx_id, &idx, false))
3370     {
3371         logf("failed to retrieve index entry");
3372         return 0;
3373     }
3374
3375     /* Stop if tag has already been modified. */
3376     if (idx.flag & FLAG_DIRTYNUM)
3377         return 0;
3378
3379     logf("import: %s", tag_data);
3380
3381     idx.flag |= FLAG_DIRTYNUM;
3382     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3383     {
3384         int data;
3385
3386         if (!read_tag(tag_data, sizeof tag_data, buf,
3387                       tagcache_tag_to_str(import_tags[i])))
3388         {
3389             continue;
3390         }
3391
3392         data = atoi(tag_data);
3393         if (data < 0)
3394             continue;
3395
3396         idx.tag_seek[import_tags[i]] = data;
3397
3398         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3399             current_tcmh.serial = data + 1;
3400         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3401             current_tcmh.commitid = data + 1;
3402     }
3403
3404     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3405 }
3406
3407 bool tagcache_import_changelog(void)
3408 {
3409     struct master_header myhdr;
3410     struct tagcache_header tch;
3411     int clfd;
3412     long masterfd;
3413     char buf[MAX(MAX_PATH, 2048)];
3414     const char *file;
3415
3416     if (!tc_stat.ready)
3417         return false;
3418
3419     while (read_lock)
3420         sleep(1);
3421
3422     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG,
3423                                    IS_FILE|NEED_WRITE, buf, sizeof(buf));
3424     clfd = open(file, O_RDONLY);
3425     if (clfd < 0)
3426     {
3427         logf("failure to open changelog");
3428         return false;
3429     }
3430
3431     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3432     {
3433         close(clfd);
3434         return false;
3435     }
3436
3437     write_lock++;
3438
3439     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3440
3441     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3442                   parse_changelog_line);
3443
3444     close(clfd);
3445     close(masterfd);
3446
3447     if (filenametag_fd >= 0)
3448     {
3449         close(filenametag_fd);
3450         filenametag_fd = -1;
3451     }
3452
3453     write_lock--;
3454
3455     update_master_header();
3456
3457     return true;
3458 }
3459
3460 #endif /* !__PCTOOL__ */
3461
3462 bool tagcache_create_changelog(struct tagcache_search *tcs)
3463 {
3464     struct master_header myhdr;
3465     struct index_entry idx;
3466     const char *file;
3467     char buf[MAX(TAG_MAXLEN+32, MAX_PATH)];
3468     char temp[32];
3469     int clfd;
3470     int i, j;
3471
3472     if (!tc_stat.ready)
3473         return false;
3474
3475     if (!tagcache_search(tcs, tag_filename))
3476         return false;
3477
3478     /* Initialize the changelog */
3479     file = get_user_file_path(TAGCACHE_FILE_CHANGELOG, IS_FILE | NEED_WRITE,
3480                               buf, sizeof(buf));
3481     clfd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3482     if (clfd < 0)
3483     {
3484         logf("failure to open changelog");
3485         return false;
3486     }
3487
3488     if (tcs->masterfd < 0)
3489     {
3490         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3491             return false;
3492     }
3493     else
3494     {
3495         lseek(tcs->masterfd, 0, SEEK_SET);
3496         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3497     }
3498
3499     write(clfd, "## Changelog version 1\n", 23);
3500
3501     for (i = 0; i < myhdr.tch.entry_count; i++)
3502     {
3503         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3504         {
3505             logf("read error #9");
3506             tagcache_search_finish(tcs);
3507             close(clfd);
3508             return false;
3509         }
3510
3511         /* Skip until the entry found has been modified. */
3512         if (! (idx.flag & FLAG_DIRTYNUM) )
3513             continue;
3514
3515         /* Skip deleted entries too. */
3516         if (idx.flag & FLAG_DELETED)
3517             continue;
3518
3519         /* Now retrieve all tags. */
3520         for (j = 0; j < TAG_COUNT; j++)
3521         {
3522             if (TAGCACHE_IS_NUMERIC(j))
3523             {
3524                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3525                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3526                 continue;
3527             }
3528
3529             tcs->type = j;
3530             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3531             write_tag(clfd, tagcache_tag_to_str(j), buf);
3532         }
3533
3534         write(clfd, "\n", 1);
3535         do_timed_yield();
3536     }
3537
3538     close(clfd);
3539
3540     tagcache_search_finish(tcs);
3541
3542     return true;
3543 }
3544
3545 static bool delete_entry(long idx_id)
3546 {
3547     int fd = -1;
3548     int masterfd = -1;
3549     int tag, i;
3550     struct index_entry idx, myidx;
3551     struct master_header myhdr;
3552     char buf[TAG_MAXLEN+32];
3553     int in_use[TAG_COUNT];
3554
3555     logf("delete_entry(): %ld", idx_id);
3556
3557 #ifdef HAVE_TC_RAMCACHE
3558     /* At first mark the entry removed from ram cache. */
3559     if (tc_stat.ramcache)
3560         hdr->indices[idx_id].flag |= FLAG_DELETED;
3561 #endif
3562
3563     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3564         return false;
3565
3566     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3567     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3568     {
3569         logf("delete_entry(): read error");
3570         goto cleanup;
3571     }
3572
3573     if (myidx.flag & FLAG_DELETED)
3574     {
3575         logf("delete_entry(): already deleted!");
3576         goto cleanup;
3577     }
3578
3579     myidx.flag |= FLAG_DELETED;
3580     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3581     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3582     {
3583         logf("delete_entry(): write_error #1");
3584         goto cleanup;
3585     }
3586
3587     /* Now check which tags are no longer in use (if any) */
3588     for (tag = 0; tag < TAG_COUNT; tag++)
3589         in_use[tag] = 0;
3590
3591     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3592     for (i = 0; i < myhdr.tch.entry_count; i++)
3593     {
3594         struct index_entry *idxp;
3595
3596 #ifdef HAVE_TC_RAMCACHE
3597         /* Use RAM DB if available for greater speed */
3598         if (tc_stat.ramcache)
3599             idxp = &hdr->indices[i];
3600         else
3601 #endif
3602         {
3603             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3604             {
3605                 logf("delete_entry(): read error #2");
3606                 goto cleanup;
3607             }
3608             idxp = &idx;
3609         }
3610
3611         if (idxp->flag & FLAG_DELETED)
3612             continue;
3613
3614         for (tag = 0; tag < TAG_COUNT; tag++)
3615         {
3616             if (TAGCACHE_IS_NUMERIC(tag))
3617                 continue;
3618
3619             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3620                 in_use[tag]++;
3621         }
3622     }
3623
3624     /* Now delete all tags no longer in use. */
3625     for (tag = 0; tag < TAG_COUNT; tag++)
3626     {
3627         struct tagcache_header tch;
3628         int oldseek = myidx.tag_seek[tag];
3629
3630         if (TAGCACHE_IS_NUMERIC(tag))
3631             continue;
3632
3633         /**
3634          * Replace tag seek with a hash value of the field string data.
3635          * That way runtime statistics of moved or altered files can be
3636          * resurrected.
3637          */
3638 #ifdef HAVE_TC_RAMCACHE
3639         if (tc_stat.ramcache && tag != tag_filename)
3640         {
3641             struct tagfile_entry *tfe;
3642             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3643
3644             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3645             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3646             myidx.tag_seek[tag] = *seek;
3647         }
3648         else
3649 #endif
3650         {
3651             struct tagfile_entry tfe;
3652
3653             /* Open the index file, which contains the tag names. */
3654             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3655                 goto cleanup;
3656
3657             /* Skip the header block */
3658             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3659             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3660             {
3661                 logf("delete_entry(): read error #3");
3662                 goto cleanup;
3663             }
3664
3665             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3666             {
3667                 logf("delete_entry(): read error #4");
3668                 goto cleanup;
3669             }
3670
3671             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3672         }
3673
3674         if (in_use[tag])
3675         {
3676             logf("in use: %d/%d", tag, in_use[tag]);
3677             if (fd >= 0)
3678             {
3679                 close(fd);
3680                 fd = -1;
3681             }
3682             continue;
3683         }
3684
3685 #ifdef HAVE_TC_RAMCACHE
3686         /* Delete from ram. */
3687         if (tc_stat.ramcache && tag != tag_filename)
3688         {
3689             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3690             tagentry->tag_data[0] = '\0';
3691         }
3692 #endif
3693
3694         /* Open the index file, which contains the tag names. */
3695         if (fd < 0)
3696         {
3697             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3698                 goto cleanup;
3699         }
3700
3701         /* Skip the header block */
3702         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3703
3704         /* Debug, print 10 first characters of the tag
3705         read(fd, buf, 10);
3706         buf[10]='\0';
3707         logf("TAG:%s", buf);
3708         lseek(fd, -10, SEEK_CUR);
3709         */
3710
3711         /* Write first data byte in tag as \0 */
3712         write(fd, "", 1);
3713
3714         /* Now tag data has been removed */
3715         close(fd);
3716         fd = -1;
3717     }
3718
3719     /* Write index entry back into master index. */
3720     lseek(masterfd, sizeof(struct master_header) +
3721           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3722     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3723     {
3724         logf("delete_entry(): write_error #2");
3725         goto cleanup;
3726     }
3727
3728     close(masterfd);
3729
3730     return true;
3731
3732     cleanup:
3733     if (fd >= 0)
3734         close(fd);
3735     if (masterfd >= 0)
3736         close(masterfd);
3737
3738     return false;
3739 }
3740
3741 #ifndef __PCTOOL__
3742 /**
3743  * Returns true if there is an event waiting in the queue
3744  * that requires the current operation to be aborted.
3745  */
3746 static bool check_event_queue(void)
3747 {
3748     struct queue_event ev;
3749
3750     if(!queue_peek(&tagcache_queue, &ev))
3751         return false;
3752
3753     switch (ev.id)
3754     {
3755         case Q_STOP_SCAN:
3756         case SYS_POWEROFF:
3757         case SYS_USB_CONNECTED:
3758             return true;
3759     }
3760
3761     return false;
3762 }
3763 #endif
3764
3765 #ifdef HAVE_TC_RAMCACHE
3766 static bool allocate_tagcache(void)
3767 {
3768     struct master_header tcmh;
3769     int fd;
3770
3771     /* Load the header. */
3772     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3773     {
3774         hdr = NULL;
3775         return false;
3776     }
3777
3778     close(fd);
3779
3780     /**
3781      * Now calculate the required cache size plus
3782      * some extra space for alignment fixes.
3783      */
3784     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3785         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3786     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3787     memset(hdr, 0, sizeof(struct ramcache_header));
3788     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3789     hdr->indices = (struct index_entry *)(hdr + 1);
3790     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3791
3792     return true;
3793 }
3794
3795 # ifdef HAVE_EEPROM_SETTINGS
3796 static bool tagcache_dumpload(void)
3797 {
3798     struct statefile_header shdr;
3799     char path[MAX_PATH];
3800     const char *file;
3801     int fd, rc;
3802     long offpos;
3803     int i;
3804
3805     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3806                                     path, sizeof(path));
3807     fd = open(file, O_RDONLY);
3808     if (fd < 0)
3809     {
3810         logf("no tagcache statedump");
3811         return false;
3812     }
3813
3814     /* Check the statefile memory placement */
3815     hdr = buffer_alloc(0);
3816     rc = read(fd, &shdr, sizeof(struct statefile_header));
3817     if (rc != sizeof(struct statefile_header)
3818         /* || (long)hdr != (long)shdr.hdr */)
3819     {
3820         logf("incorrect statefile");
3821         hdr = NULL;
3822         close(fd);
3823         return false;
3824     }
3825
3826     offpos = (long)hdr - (long)shdr.hdr;
3827
3828     /* Lets allocate real memory and load it */
3829     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3830     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3831     close(fd);
3832
3833     if (rc != shdr.tc_stat.ramcache_allocated)
3834     {
3835         logf("read failure!");
3836         hdr = NULL;
3837         return false;
3838     }
3839
3840     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3841
3842     /* Now fix the pointers */
3843     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3844     for (i = 0; i < TAG_COUNT; i++)
3845         hdr->tags[i] += offpos;
3846
3847     return true;
3848 }
3849
3850 static bool tagcache_dumpsave(void)
3851 {
3852     struct statefile_header shdr;
3853     char path[MAX_PATH];
3854     const char *file;
3855     int fd;
3856
3857     if (!tc_stat.ramcache)
3858         return false;
3859
3860     file = get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
3861                                     path, sizeof(path));
3862     fd = open(file, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3863     if (fd < 0)
3864     {
3865         logf("failed to create a statedump");
3866         return false;
3867     }
3868
3869     /* Create the header */
3870     shdr.hdr = hdr;
3871     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3872     write(fd, &shdr, sizeof(struct statefile_header));
3873
3874     /* And dump the data too */
3875     write(fd, hdr, tc_stat.ramcache_allocated);
3876     close(fd);
3877
3878     return true;
3879 }
3880 # endif
3881
3882 static bool load_tagcache(void)
3883 {
3884     struct tagcache_header *tch;
3885     long bytesleft = tc_stat.ramcache_allocated;
3886     struct index_entry *idx;
3887     int rc, fd;
3888     char *p, path[MAX_PATH];
3889     const char *file;
3890     int i, tag;
3891
3892 # ifdef HAVE_DIRCACHE
3893     while (dircache_is_initializing())
3894         sleep(1);
3895
3896     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3897 # endif
3898
3899     logf("loading tagcache to ram...");
3900
3901     file = get_user_file_path(TAGCACHE_FILE_MASTER,
3902                               IS_FILE|NEED_WRITE,
3903                               path, sizeof(path));
3904     fd = open(file, O_RDONLY);
3905     if (fd < 0)
3906     {
3907         logf("tagcache open failed");
3908         return false;
3909     }
3910
3911     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3912         != sizeof(struct master_header)
3913         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3914     {
3915         logf("incorrect header");
3916         return false;
3917     }
3918
3919     idx = hdr->indices;
3920
3921     /* Load the master index table. */
3922     for (i = 0; i < hdr->h.tch.entry_count; i++)
3923     {
3924         rc = ecread_index_entry(fd, idx);
3925         if (rc != sizeof(struct index_entry))
3926         {
3927             logf("read error #10");
3928             close(fd);
3929             return false;
3930         }
3931
3932         bytesleft -= sizeof(struct index_entry);
3933         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3934         {
3935             logf("too big tagcache.");
3936             close(fd);
3937             return false;
3938         }
3939
3940         idx++;
3941     }
3942
3943     close(fd);
3944
3945     /* Load the tags. */
3946     p = (char *)idx;
3947     for (tag = 0; tag < TAG_COUNT; tag++)
3948     {
3949         struct tagfile_entry *fe;
3950         char buf[TAG_MAXLEN+32];
3951
3952         if (TAGCACHE_IS_NUMERIC(tag))
3953             continue ;
3954
3955         //p = ((void *)p+1);
3956         p = (char *)((long)p & ~0x03) + 0x04;
3957         hdr->tags[tag] = p;
3958
3959         /* Check the header. */
3960         tch = (struct tagcache_header *)p;
3961         p += sizeof(struct tagcache_header);
3962
3963         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3964             return false;
3965
3966         for (hdr->entry_count[tag] = 0;
3967              hdr->entry_count[tag] < tch->entry_count;
3968              hdr->entry_count[tag]++)
3969         {
3970             long pos;
3971
3972             if (do_timed_yield())
3973             {
3974                 /* Abort if we got a critical event in queue */
3975                 if (check_event_queue())
3976                     return false;
3977             }
3978
3979             fe = (struct tagfile_entry *)p;
3980             pos = lseek(fd, 0, SEEK_CUR);
3981             rc = ecread_tagfile_entry(fd, fe);
3982             if (rc != sizeof(struct tagfile_entry))
3983             {
3984                 /* End of lookup table. */
3985                 logf("read error #11");
3986                 close(fd);
3987                 return false;
3988             }
3989
3990             /* We have a special handling for the filename tags. */
3991             if (tag == tag_filename)
3992             {
3993 # ifdef HAVE_DIRCACHE
3994                 const struct dircache_entry *dc;
3995 # endif
3996
3997                 // FIXME: This is wrong!
3998                 // idx = &hdr->indices[hdr->entry_count[i]];
3999                 idx = &hdr->indices[fe->idx_id];
4000
4001                 if (fe->tag_length >= (long)sizeof(buf)-1)
4002                 {
4003                     read(fd, buf, 10);
4004                     buf[10] = '\0';
4005                     logf("TAG:%s", buf);
4006                     logf("too long filename");
4007                     close(fd);
4008                     return false;
4009                 }
4010
4011                 rc = read(fd, buf, fe->tag_length);
4012                 if (rc != fe->tag_length)
4013                 {
4014                     logf("read error #12");
4015                     close(fd);
4016                     return false;
4017                 }
4018
4019                 /* Check if the entry has already been removed */
4020                 if (idx->flag & FLAG_DELETED)
4021                     continue;
4022
4023                 /* This flag must not be used yet. */
4024                 if (idx->flag & FLAG_DIRCACHE)
4025                 {
4026                     logf("internal error!");
4027                     close(fd);
4028                     return false;
4029                 }
4030
4031                 if (idx->tag_seek[tag] != pos)
4032                 {
4033                     logf("corrupt data structures!");
4034                     close(fd);
4035                     return false;
4036                 }
4037
4038 # ifdef HAVE_DIRCACHE
4039                 if (dircache_is_enabled())
4040                 {
4041                     dc = dircache_get_entry_ptr(buf);
4042                     if (dc == NULL)
4043                     {
4044                         logf("Entry no longer valid.");
4045                         logf("-> %s", buf);
4046                         if (global_settings.tagcache_autoupdate)
4047                             delete_entry(fe->idx_id);
4048                         continue ;
4049                     }
4050
4051                     idx->flag |= FLAG_DIRCACHE;
4052                     idx->tag_seek[tag_filename] = (long)dc;
4053                 }
4054                 else
4055 # endif
4056                 {
4057                     /* This will be very slow unless dircache is enabled
4058                        or target is flash based, but do it anyway for
4059                        consistency. */
4060                     /* Check if entry has been removed. */
4061                     if (global_settings.tagcache_autoupdate)
4062                     {
4063                         if (!file_exists(buf))
4064                         {
4065                             logf("Entry no longer valid.");
4066                             logf("-> %s", buf);
4067                             delete_entry(fe->idx_id);
4068                             continue;
4069                         }
4070                     }
4071                 }
4072
4073                 continue ;
4074             }
4075
4076             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4077             if (bytesleft < 0)
4078             {
4079                 logf("too big tagcache #2");
4080                 logf("tl: %ld", fe->tag_length);
4081                 logf("bl: %ld", bytesleft);
4082                 close(fd);
4083                 return false;
4084             }
4085
4086             p = fe->tag_data;
4087             rc = read(fd, fe->tag_data, fe->tag_length);
4088             p += rc;
4089
4090             if (rc != fe->tag_length)
4091             {
4092                 logf("read error #13");
4093                 logf("rc=0x%04x", rc); // 0x431
4094                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4095                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4096                 logf("tag=0x%02x", tag); // 0x00
4097                 close(fd);
4098                 return false;
4099             }
4100         }
4101         close(fd);
4102     }
4103
4104     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4105     logf("tagcache loaded into ram!");
4106
4107     return true;
4108 }
4109 #endif /* HAVE_TC_RAMCACHE */
4110
4111 static bool check_deleted_files(void)
4112 {
4113     int fd;
4114     char buf[TAG_MAXLEN+32], path[MAX_PATH];
4115     const char *file;
4116     struct tagfile_entry tfe;
4117
4118     logf("reverse scan...");
4119     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4120     file = get_user_file_path(buf, IS_FILE | NEED_WRITE, path, sizeof(path));
4121     fd = open(file, O_RDONLY);
4122
4123     if (fd < 0)
4124     {
4125         logf("%s open fail", buf);
4126         return false;
4127     }
4128
4129     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4130     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4131 #ifndef __PCTOOL__
4132            && !check_event_queue()
4133 #endif
4134            )
4135     {
4136         if (tfe.tag_length >= (long)sizeof(buf)-1)
4137         {
4138             logf("too long tag");
4139             close(fd);
4140             return false;
4141         }
4142
4143         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4144         {
4145             logf("read error #14");
4146             close(fd);
4147             return false;
4148         }
4149
4150         /* Check if the file has already deleted from the db. */
4151         if (*buf == '\0')
4152             continue;
4153
4154         /* Now check if the file exists. */
4155         if (!file_exists(buf))
4156         {
4157             logf("Entry no longer valid.");
4158             logf("-> %s / %ld", buf, tfe.tag_length);
4159             delete_entry(tfe.idx_id);
4160         }
4161     }
4162
4163     close(fd);
4164
4165     logf("done");
4166
4167     return true;
4168 }
4169
4170
4171 /* Note that this function must not be inlined, otherwise the whole point
4172  * of having the code in a separate function is lost.
4173  */
4174 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4175     int *ignore, int *unignore)
4176 {
4177     char newpath[MAX_PATH];
4178
4179     /* check for a database.ignore file */
4180     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4181     *ignore = file_exists(newpath);
4182     /* check for a database.unignore file */
4183     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4184     *unignore = file_exists(newpath);
4185 }
4186
4187
4188 static bool check_dir(const char *dirname, int add_files)
4189 {
4190     DIR *dir;
4191     int len;
4192     int success = false;
4193     int ignore, unignore;
4194
4195     dir = opendir(dirname);
4196     if (!dir)
4197     {
4198         logf("tagcache: opendir(%s) failed", dirname);
4199         return false;
4200     }
4201
4202     /* check for a database.ignore and database.unignore */
4203     check_ignore(dirname, &ignore, &unignore);
4204
4205     /* don't do anything if both ignore and unignore are there */
4206     if (ignore != unignore)
4207         add_files = unignore;
4208
4209     /* Recursively scan the dir. */
4210 #ifdef __PCTOOL__
4211     while (1)
4212 #else
4213     while (!check_event_queue())
4214 #endif
4215     {
4216         struct dirent *entry;
4217
4218         entry = readdir(dir);
4219
4220         if (entry == NULL)
4221         {
4222             success = true;
4223             break ;
4224         }
4225
4226         if (!strcmp((char *)entry->d_name, ".") ||
4227             !strcmp((char *)entry->d_name, ".."))
4228             continue;
4229
4230         yield();
4231
4232         len = strlen(curpath);
4233         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4234                  entry->d_name);
4235
4236         processed_dir_count++;
4237         if (entry->attribute & ATTR_DIRECTORY)
4238             check_dir(curpath, add_files);
4239         else if (add_files)
4240         {
4241             tc_stat.curentry = curpath;
4242
4243             /* Add a new entry to the temporary db file. */
4244             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4245 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4246                          , dir->internal_entry
4247 #endif
4248                          );
4249
4250             /* Wait until current path for debug screen is read and unset. */
4251             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4252                 yield();
4253
4254             tc_stat.curentry = NULL;
4255         }
4256
4257         curpath[len] = '\0';
4258     }
4259
4260     closedir(dir);
4261
4262     return success;
4263 }
4264
4265 void tagcache_screensync_event(void)
4266 {
4267     tc_stat.curentry = NULL;
4268 }
4269
4270 void tagcache_screensync_enable(bool state)
4271 {
4272     tc_stat.syncscreen = state;
4273 }
4274
4275 void tagcache_build(const char *path)
4276 {
4277     struct tagcache_header header;
4278     bool ret;
4279     char buf[MAX_PATH];
4280     const char *file;
4281
4282     curpath[0] = '\0';
4283     data_size = 0;
4284     total_entry_count = 0;
4285     processed_dir_count = 0;
4286
4287 #ifdef HAVE_DIRCACHE
4288     while (dircache_is_initializing())
4289         sleep(1);
4290 #endif
4291
4292     logf("updating tagcache");
4293
4294     file = get_user_file_path(TAGCACHE_FILE_TEMP,
4295                                     IS_FILE|NEED_WRITE, buf, sizeof(buf));
4296
4297
4298     if (file_exists(file))
4299     {
4300         logf("skipping, cache already waiting for commit");
4301         return ;
4302     }
4303
4304     cachefd = open(file, O_RDWR | O_CREAT | O_TRUNC, 0666);
4305     if (cachefd < 0)
4306     {
4307         logf("master file open failed: %s", file);
4308         return ;
4309     }
4310
4311     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4312
4313     cpu_boost(true);
4314
4315     logf("Scanning files...");
4316     /* Scan for new files. */
4317     memset(&header, 0, sizeof(struct tagcache_header));
4318     write(cachefd, &header, sizeof(struct tagcache_header));
4319
4320     if (strcmp("/", path) != 0)
4321         strcpy(curpath, path);
4322     ret = check_dir(path, true);
4323
4324     /* Write the header. */
4325     header.magic = TAGCACHE_MAGIC;
4326     header.datasize = data_size;
4327     header.entry_count = total_entry_count;
4328     lseek(cachefd, 0, SEEK_SET);
4329     write(cachefd, &header, sizeof(struct tagcache_header));
4330     close(cachefd);
4331
4332     if (filenametag_fd >= 0)
4333     {
4334         close(filenametag_fd);
4335         filenametag_fd = -1;
4336     }
4337
4338     if (!ret)
4339     {
4340         logf("Aborted.");
4341         cpu_boost(false);
4342         return ;
4343     }
4344
4345     /* Commit changes to the database. */
4346 #ifdef __PCTOOL__
4347     allocate_tempbuf();
4348 #endif
4349     if (commit())
4350     {
4351         remove(file);
4352         logf("tagcache built!");
4353     }
4354 #ifdef __PCTOOL__
4355     free_tempbuf();
4356 #endif
4357
4358 #ifdef HAVE_TC_RAMCACHE
4359     if (hdr)
4360     {
4361         /* Import runtime statistics if we just initialized the db. */
4362         if (hdr->h.serial == 0)
4363             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4364     }
4365 #endif
4366
4367     cpu_boost(false);
4368 }
4369
4370 #ifdef HAVE_TC_RAMCACHE
4371 static void load_ramcache(void)
4372 {
4373     if (!hdr)
4374         return ;
4375
4376     cpu_boost(true);
4377
4378     /* At first we should load the cache (if exists). */
4379     tc_stat.ramcache = load_tagcache();
4380
4381     if (!tc_stat.ramcache)
4382     {
4383         /* If loading failed, it must indicate some problem with the db
4384          * so disable it entirely to prevent further issues. */
4385         tc_stat.ready = false;
4386         hdr = NULL;
4387     }
4388
4389     cpu_boost(false);
4390 }
4391
4392 void tagcache_unload_ramcache(void)
4393 {
4394     tc_stat.ramcache = false;
4395     /* Just to make sure there is no statefile present. */
4396
4397 #if 0
4398     char path[MAX_PATH];
4399     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4400                 path, sizeof(path)));
4401 #endif
4402 }
4403 #endif
4404
4405 #ifndef __PCTOOL__
4406 static void tagcache_thread(void)
4407 {
4408     struct queue_event ev;
4409     bool check_done = false;
4410     char path[MAX_PATH];
4411
4412     /* If the previous cache build/update was interrupted, commit
4413      * the changes first in foreground. */
4414     cpu_boost(true);
4415     allocate_tempbuf();
4416     commit();
4417     free_tempbuf();
4418
4419 #ifdef HAVE_TC_RAMCACHE
4420 # ifdef HAVE_EEPROM_SETTINGS
4421     if (firmware_settings.initialized && firmware_settings.disk_clean
4422         && global_settings.tagcache_ram)
4423     {
4424         check_done = tagcache_dumpload();
4425     }
4426
4427     remove(get_user_file_path(TAGCACHE_STATEFILE, IS_FILE | NEED_WRITE,
4428                 buf, sizeof(buf)));
4429 # endif
4430
4431     /* Allocate space for the tagcache if found on disk. */
4432     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4433         allocate_tagcache();
4434 #endif
4435
4436     cpu_boost(false);
4437     tc_stat.initialized = true;
4438
4439     /* Don't delay bootup with the header check but do it on background. */
4440     if (!tc_stat.ready)
4441     {
4442         sleep(HZ);
4443         tc_stat.ready = check_all_headers();
4444         tc_stat.readyvalid = true;
4445     }
4446
4447     while (1)
4448     {
4449         run_command_queue(false);
4450
4451         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4452
4453         switch (ev.id)
4454         {
4455             case Q_IMPORT_CHANGELOG:
4456                 tagcache_import_changelog();
4457                 break;
4458
4459             case Q_REBUILD:
4460                 remove_files();
4461                 remove(get_user_file_path(TAGCACHE_FILE_TEMP,
4462                                 IS_FILE|NEED_WRITE, path, sizeof(path)));
4463                 tagcache_build("/");
4464                 break;
4465
4466             case Q_UPDATE:
4467                 tagcache_build("/");
4468 #ifdef HAVE_TC_RAMCACHE
4469                 load_ramcache();
4470 #endif
4471                 check_deleted_files();
4472                 break ;
4473
4474             case Q_START_SCAN:
4475                 check_done = false;
4476             case SYS_TIMEOUT:
4477                 if (check_done || !tc_stat.ready)
4478                     break ;
4479
4480 #ifdef HAVE_TC_RAMCACHE
4481                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4482                 {
4483                     load_ramcache();
4484                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4485                         tagcache_build("/");
4486                 }
4487                 else
4488 #endif
4489                 if (global_settings.tagcache_autoupdate)
4490                 {
4491                     tagcache_build("/");
4492
4493                     /* This will be very slow unless dircache is enabled
4494                        or target is flash based, but do it anyway for
4495                        consistency. */
4496                     check_deleted_files();
4497                 }
4498
4499                 logf("tagcache check done");
4500
4501                 check_done = true;
4502                 break ;
4503
4504             case Q_STOP_SCAN:
4505                 break ;
4506
4507             case SYS_POWEROFF:
4508                 break ;
4509
4510 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4511             case SYS_USB_CONNECTED:
4512                 logf("USB: TagCache");
4513                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4514                 usb_wait_for_disconnect(&tagcache_queue);
4515                 break ;
4516 #endif
4517         }
4518     }
4519 }
4520
4521 bool tagcache_prepare_shutdown(void)
4522 {
4523     if (tagcache_get_commit_step() > 0)
4524         return false;
4525
4526     tagcache_stop_scan();
4527     while (read_lock || write_lock)
4528         sleep(1);
4529
4530     return true;
4531 }
4532
4533 void tagcache_shutdown(void)
4534 {
4535     /* Flush the command queue. */
4536     run_command_queue(true);
4537
4538 #ifdef HAVE_EEPROM_SETTINGS
4539     if (tc_stat.ramcache)
4540         tagcache_dumpsave();
4541 #endif
4542 }
4543
4544 static int get_progress(void)
4545 {
4546     int total_count = -1;
4547
4548 #ifdef HAVE_DIRCACHE
4549     if (dircache_is_enabled())
4550     {
4551         total_count = dircache_get_entry_count();
4552     }
4553     else
4554 #endif
4555 #ifdef HAVE_TC_RAMCACHE
4556     {
4557         if (hdr && tc_stat.ramcache)
4558             total_count = hdr->h.tch.entry_count;
4559     }
4560 #endif
4561
4562     if (total_count < 0)
4563         return -1;
4564
4565     return processed_dir_count * 100 / total_count;
4566 }
4567
4568 struct tagcache_stat* tagcache_get_stat(void)
4569 {
4570     tc_stat.progress = get_progress();
4571     tc_stat.processed_entries = processed_dir_count;
4572
4573     return &tc_stat;
4574 }
4575
4576 void tagcache_start_scan(void)
4577 {
4578     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4579 }
4580
4581 bool tagcache_update(void)
4582 {
4583     if (!tc_stat.ready)
4584         return false;
4585
4586     queue_post(&tagcache_queue, Q_UPDATE, 0);
4587     return false;
4588 }
4589
4590 bool tagcache_rebuild()
4591 {
4592     queue_post(&tagcache_queue, Q_REBUILD, 0);
4593     return false;
4594 }
4595
4596 void tagcache_stop_scan(void)
4597 {
4598     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4599 }
4600
4601 #ifdef HAVE_TC_RAMCACHE
4602 bool tagcache_is_ramcache(void)
4603 {
4604     return tc_stat.ramcache;
4605 }
4606 #endif
4607
4608 #endif /* !__PCTOOL__ */
4609
4610
4611 void tagcache_init(void)
4612 {
4613     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4614     memset(&current_tcmh, 0, sizeof(struct master_header));
4615     filenametag_fd = -1;
4616     write_lock = read_lock = 0;
4617
4618 #ifndef __PCTOOL__
4619     mutex_init(&command_queue_mutex);
4620     queue_init(&tagcache_queue, true);
4621     create_thread(tagcache_thread, tagcache_stack,
4622                   sizeof(tagcache_stack), 0, tagcache_thread_name
4623                   IF_PRIO(, PRIORITY_BACKGROUND)
4624                   IF_COP(, CPU));
4625 #else
4626     tc_stat.initialized = true;
4627     allocate_tempbuf();
4628     commit();
4629     free_tempbuf();
4630     tc_stat.ready = check_all_headers();
4631 #endif
4632 }
4633
4634 #ifdef __PCTOOL__
4635 void tagcache_reverse_scan(void)
4636 {
4637     logf("Checking for deleted files");
4638     check_deleted_files();
4639 }
4640 #endif
4641
4642 bool tagcache_is_initialized(void)
4643 {
4644     return tc_stat.initialized;
4645 }
4646 bool tagcache_is_usable(void)
4647 {
4648     return tc_stat.initialized && tc_stat.ready;
4649 }
4650 int tagcache_get_commit_step(void)
4651 {
4652     return tc_stat.commit_step;
4653 }
4654 int tagcache_get_max_commit_step(void)
4655 {
4656     return (int)(SORTED_TAGS_COUNT)+1;
4657 }
4658