apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #ifdef APPLICATION
  64 #include <unistd.h> /* readlink() */
  65 #include <limits.h> /* PATH_MAX */
  66 #endif
  67 #include "config.h"
  68 #include "ata_idle_notify.h"
  69 #include "thread.h"
  70 #include "kernel.h"
  71 #include "system.h"
  72 #include "logf.h"
  73 #include "string-extra.h"
  74 #include "usb.h"
  75 #include "metadata.h"
  76 #include "tagcache.h"
  77 #include "buffer.h"
  78 #include "crc32.h"
  79 #include "misc.h"
  80 #include "settings.h"
  81 #include "dir.h"
  82 #include "filefuncs.h"
  83 #include "structec.h"
  84 #include "debug.h"
  85
  86 #ifndef __PCTOOL__
  87 #include "lang.h"
  88 #include "eeprom_settings.h"
  89 #endif
  90
  91 #ifdef __PCTOOL__
  92 #define yield() do { } while(0)
  93 #define sim_sleep(timeout) do { } while(0)
  94 #define do_timed_yield() do { } while(0)
  95 #endif
  96
  97 #ifndef __PCTOOL__
  98 /* Tag Cache thread. */
  99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
 100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
 101 static const char tagcache_thread_name[] = "tagcache";
 102 #endif
 103
 104 /* Previous path when scanning directory tree recursively. */
 105 static char curpath[TAG_MAXLEN+32];
 106
 107 /* Used when removing duplicates. */
 108 static char *tempbuf;     /* Allocated when needed. */
 109 static long tempbufidx;   /* Current location in buffer. */
 110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 111 static long tempbuf_left; /* Buffer space left. */
 112 static long tempbuf_pos;
 113
 114 #define SORTED_TAGS_COUNT 8
 115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 118     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 119 /* Tags we want to get sorted (loaded to the tempbuf). */
 120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 123
 124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 126     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 127     (1LU << tag_albumartist) | (1LU << tag_grouping))
 128
 129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 130 static const char *tags_str[] = { "artist", "album", "genre", "title",
 131     "filename", "composer", "comment", "albumartist", "grouping", "year",
 132     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 133     "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
 134
 135 /* Status information of the tagcache. */
 136 static struct tagcache_stat tc_stat;
 137
 138 /* Queue commands. */
 139 enum tagcache_queue {
 140     Q_STOP_SCAN = 0,
 141     Q_START_SCAN,
 142     Q_IMPORT_CHANGELOG,
 143     Q_UPDATE,
 144     Q_REBUILD,
 145
 146     /* Internal tagcache command queue. */
 147     CMD_UPDATE_MASTER_HEADER,
 148     CMD_UPDATE_NUMERIC,
 149 };
 150
 151 struct tagcache_command_entry {
 152     int32_t command;
 153     int32_t idx_id;
 154     int32_t tag;
 155     int32_t data;
 156 };
 157
 158 #ifndef __PCTOOL__
 159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 160 static volatile int command_queue_widx = 0;
 161 static volatile int command_queue_ridx = 0;
 162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
 163 #endif
 164
 165 /* Tag database structures. */
 166
 167 /* Variable-length tag entry in tag files. */
 168 struct tagfile_entry {
 169     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 170     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 171     char tag_data[0];  /* Begin of the tag data */
 172 };
 173
 174 /* Fixed-size tag entry in master db index. */
 175 struct index_entry {
 176     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 177     int32_t flag;                /* Status flags */
 178 };
 179
 180 /* Header is the same in every file. */
 181 struct tagcache_header {
 182     int32_t magic;       /* Header version number */
 183     int32_t datasize;    /* Data size in bytes */
 184     int32_t entry_count; /* Number of entries in this file */
 185 };
 186
 187 struct master_header {
 188     struct tagcache_header tch;
 189     int32_t serial; /* Increasing counting number */
 190     int32_t commitid; /* Number of commits so far */
 191     int32_t dirty;
 192 };
 193
 194 /* For the endianess correction */
 195 static const char *tagfile_entry_ec   = "ll";
 196 /**
 197  Note: This should be (1 + TAG_COUNT) amount of l's.
 198  */
 199 static const char *index_entry_ec     = "llllllllllllllllllllll";
 200
 201 static const char *tagcache_header_ec = "lll";
 202 static const char *master_header_ec   = "llllll";
 203
 204 static struct master_header current_tcmh;
 205
 206 #ifdef HAVE_TC_RAMCACHE
 207 /* Header is created when loading database to ram. */
 208 struct ramcache_header {
 209     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 210     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 211     struct index_entry indices[0]; /* Master index file content */
 212 };
 213
 214 # ifdef HAVE_EEPROM_SETTINGS
 215 struct statefile_header {
 216     int32_t magic;                /* Statefile version number */
 217     struct master_header mh;      /* Header from the master index */
 218     struct ramcache_header *hdr;  /* Old load address of hdr for relocation */
 219     struct tagcache_stat tc_stat;
 220 };
 221 # endif
 222
 223 /* Pointer to allocated ramcache_header */
 224 static struct ramcache_header *hdr;
 225 #endif
 226
 227 /**
 228  * Full tag entries stored in a temporary file waiting
 229  * for commit to the cache. */
 230 struct temp_file_entry {
 231     long tag_offset[TAG_COUNT];
 232     short tag_length[TAG_COUNT];
 233     long flag;
 234
 235     long data_length;
 236 };
 237
 238 struct tempbuf_id_list {
 239     long id;
 240     struct tempbuf_id_list *next;
 241 };
 242
 243 struct tempbuf_searchidx {
 244     long idx_id;
 245     char *str;
 246     int seek;
 247     struct tempbuf_id_list idlist;
 248 };
 249
 250 /* Lookup buffer for fixing messed up index while after sorting. */
 251 static long commit_entry_count;
 252 static long lookup_buffer_depth;
 253 static struct tempbuf_searchidx **lookup;
 254
 255 /* Used when building the temporary file. */
 256 static int cachefd = -1, filenametag_fd;
 257 static int total_entry_count = 0;
 258 static int data_size = 0;
 259 static int processed_dir_count;
 260
 261 /* Thread safe locking */
 262 static volatile int write_lock;
 263 static volatile int read_lock;
 264
 265 static bool delete_entry(long idx_id);
 266
 267 const char* tagcache_tag_to_str(int tag)
 268 {
 269     return tags_str[tag];
 270 }
 271
 272 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 273 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 274 {
 275     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 276 }
 277
 278 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 279 {
 280     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 281 }
 282
 283 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 284 {
 285     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 286 }
 287
 288 #ifdef HAVE_DIRCACHE
 289 /**
 290  * Returns true if specified flag is still present, i.e., dircache
 291  * has not been reloaded.
 292  */
 293 static bool is_dircache_intact(void)
 294 {
 295     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 296 }
 297 #endif
 298
 299 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 300 {
 301     int fd;
 302     char buf[MAX_PATH];
 303     int rc;
 304
 305     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 306         return -1;
 307
 308     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 309
 310     fd = open(buf, write ? O_RDWR : O_RDONLY);
 311     if (fd < 0)
 312     {
 313         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 314         tc_stat.ready = false;
 315         return fd;
 316     }
 317
 318     /* Check the header. */
 319     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 320     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 321     {
 322         logf("header error");
 323         tc_stat.ready = false;
 324         close(fd);
 325         return -2;
 326     }
 327
 328     return fd;
 329 }
 330
 331 static int open_master_fd(struct master_header *hdr, bool write)
 332 {
 333     int fd;
 334     int rc;
 335
 336     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 337     if (fd < 0)
 338     {
 339         logf("master file open failed for R/W");
 340         tc_stat.ready = false;
 341         return fd;
 342     }
 343
 344     tc_stat.econ = false;
 345
 346     /* Check the header. */
 347     rc = read(fd, hdr, sizeof(struct master_header));
 348     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 349     {
 350         /* Success. */
 351         return fd;
 352     }
 353
 354     /* Trying to read again, this time with endianess correction enabled. */
 355     lseek(fd, 0, SEEK_SET);
 356
 357     rc = ecread(fd, hdr, 1, master_header_ec, true);
 358     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 359     {
 360         logf("header error");
 361         tc_stat.ready = false;
 362         close(fd);
 363         return -2;
 364     }
 365
 366     tc_stat.econ = true;
 367
 368     return fd;
 369 }
 370
 371 #ifndef __PCTOOL__
 372 static bool do_timed_yield(void)
 373 {
 374     /* Sorting can lock up for quite a while, so yield occasionally */
 375     static long wakeup_tick = 0;
 376     if (TIME_AFTER(current_tick, wakeup_tick))
 377     {
 378         wakeup_tick = current_tick + (HZ/4);
 379         yield();
 380         return true;
 381     }
 382     return false;
 383 }
 384 #endif
 385
 386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 387 /* find the ramcache entry corresponding to the file indicated by
 388  * filename and dc (it's corresponding dircache id). */
 389 static long find_entry_ram(const char *filename, int dc)
 390 {
 391     static long last_pos = 0;
 392     int i;
 393
 394     /* Check if tagcache is loaded into ram. */
 395     if (!tc_stat.ramcache)
 396         return -1;
 397
 398     if (dc < 0)
 399         dc = dircache_get_entry_id(filename);
 400
 401     if (dc < 0)
 402     {
 403         logf("tagcache: file not found.");
 404         return -1;
 405     }
 406
 407     try_again:
 408
 409     if (last_pos > 0)
 410         i = last_pos;
 411     else
 412         i = 0;
 413
 414     for (; i < current_tcmh.tch.entry_count; i++)
 415     {
 416         if (hdr->indices[i].tag_seek[tag_filename] == dc)
 417         {
 418             last_pos = MAX(0, i - 3);
 419             return i;
 420         }
 421
 422         do_timed_yield();
 423     }
 424
 425     if (last_pos > 0)
 426     {
 427         last_pos = 0;
 428         goto try_again;
 429     }
 430
 431     return -1;
 432 }
 433 #endif
 434
 435 static long find_entry_disk(const char *filename_raw, bool localfd)
 436 {
 437     struct tagcache_header tch;
 438     static long last_pos = -1;
 439     long pos_history[POS_HISTORY_COUNT];
 440     long pos_history_idx = 0;
 441     bool found = false;
 442     struct tagfile_entry tfe;
 443     int fd;
 444     char buf[TAG_MAXLEN+32];
 445     int i;
 446     int pos = -1;
 447
 448     const char *filename = filename_raw;
 449 #ifdef APPLICATION
 450     char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
 451     if (realpath(filename, pathbuf) == pathbuf)
 452         filename = pathbuf;
 453 #endif
 454
 455     if (!tc_stat.ready)
 456         return -2;
 457
 458     fd = filenametag_fd;
 459     if (fd < 0 || localfd)
 460     {
 461         last_pos = -1;
 462         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 463             return -1;
 464     }
 465
 466     check_again:
 467
 468     if (last_pos > 0)
 469         lseek(fd, last_pos, SEEK_SET);
 470     else
 471         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 472
 473     while (true)
 474     {
 475         pos = lseek(fd, 0, SEEK_CUR);
 476         for (i = pos_history_idx-1; i >= 0; i--)
 477             pos_history[i+1] = pos_history[i];
 478         pos_history[0] = pos;
 479
 480         if (ecread_tagfile_entry(fd, &tfe)
 481             != sizeof(struct tagfile_entry))
 482         {
 483             break ;
 484         }
 485
 486         if (tfe.tag_length >= (long)sizeof(buf))
 487         {
 488             logf("too long tag #1");
 489             close(fd);
 490             if (!localfd)
 491                 filenametag_fd = -1;
 492             last_pos = -1;
 493             return -2;
 494         }
 495
 496         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 497         {
 498             logf("read error #2");
 499             close(fd);
 500             if (!localfd)
 501                 filenametag_fd = -1;
 502             last_pos = -1;
 503             return -3;
 504         }
 505
 506         if (!strcasecmp(filename, buf))
 507         {
 508             last_pos = pos_history[pos_history_idx];
 509             found = true;
 510             break ;
 511         }
 512
 513         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 514             pos_history_idx++;
 515     }
 516
 517     /* Not found? */
 518     if (!found)
 519     {
 520         if (last_pos > 0)
 521         {
 522             last_pos = -1;
 523             logf("seek again");
 524             goto check_again;
 525         }
 526
 527         if (fd != filenametag_fd || localfd)
 528             close(fd);
 529         return -4;
 530     }
 531
 532     if (fd != filenametag_fd || localfd)
 533         close(fd);
 534
 535     return tfe.idx_id;
 536 }
 537
 538 static int find_index(const char *filename)
 539 {
 540     long idx_id = -1;
 541
 542 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 543     if (tc_stat.ramcache && is_dircache_intact())
 544         idx_id = find_entry_ram(filename, -1);
 545 #endif
 546
 547     if (idx_id < 0)
 548         idx_id = find_entry_disk(filename, true);
 549
 550     return idx_id;
 551 }
 552
 553 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 554 {
 555     int idx_id;
 556
 557     if (!tc_stat.ready)
 558         return false;
 559
 560     idx_id = find_index(filename);
 561     if (idx_id < 0)
 562         return false;
 563
 564     if (!tagcache_search(tcs, tag_filename))
 565         return false;
 566
 567     tcs->entry_count = 0;
 568     tcs->idx_id = idx_id;
 569
 570     return true;
 571 }
 572
 573 static bool get_index(int masterfd, int idxid,
 574                       struct index_entry *idx, bool use_ram)
 575 {
 576     bool localfd = false;
 577
 578     if (idxid < 0)
 579     {
 580         logf("Incorrect idxid: %d", idxid);
 581         return false;
 582     }
 583
 584 #ifdef HAVE_TC_RAMCACHE
 585     if (tc_stat.ramcache && use_ram)
 586     {
 587         if (hdr->indices[idxid].flag & FLAG_DELETED)
 588             return false;
 589
 590 # ifdef HAVE_DIRCACHE
 591         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 592             || is_dircache_intact())
 593 #endif
 594         {
 595             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 596             return true;
 597         }
 598     }
 599 #else
 600     (void)use_ram;
 601 #endif
 602
 603     if (masterfd < 0)
 604     {
 605         struct master_header tcmh;
 606
 607         localfd = true;
 608         masterfd = open_master_fd(&tcmh, false);
 609         if (masterfd < 0)
 610             return false;
 611     }
 612
 613     lseek(masterfd, idxid * sizeof(struct index_entry)
 614           + sizeof(struct master_header), SEEK_SET);
 615     if (ecread_index_entry(masterfd, idx)
 616         != sizeof(struct index_entry))
 617     {
 618         logf("read error #3");
 619         if (localfd)
 620             close(masterfd);
 621
 622         return false;
 623     }
 624
 625     if (localfd)
 626         close(masterfd);
 627
 628     if (idx->flag & FLAG_DELETED)
 629         return false;
 630
 631     return true;
 632 }
 633
 634 #ifndef __PCTOOL__
 635
 636 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 637 {
 638     /* We need to exclude all memory only flags & tags when writing to disk. */
 639     if (idx->flag & FLAG_DIRCACHE)
 640     {
 641         logf("memory only flags!");
 642         return false;
 643     }
 644
 645 #ifdef HAVE_TC_RAMCACHE
 646     /* Only update numeric data. Writing the whole index to RAM by memcpy
 647      * destroys dircache pointers!
 648      */
 649     if (tc_stat.ramcache)
 650     {
 651         int tag;
 652         struct index_entry *idx_ram = &hdr->indices[idxid];
 653
 654         for (tag = 0; tag < TAG_COUNT; tag++)
 655         {
 656             if (TAGCACHE_IS_NUMERIC(tag))
 657             {
 658                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 659             }
 660         }
 661
 662         /* Don't touch the dircache flag or attributes. */
 663         idx_ram->flag = (idx->flag & 0x0000ffff)
 664             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 665     }
 666 #endif
 667
 668     lseek(masterfd, idxid * sizeof(struct index_entry)
 669           + sizeof(struct master_header), SEEK_SET);
 670     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 671     {
 672         logf("write error #3");
 673         logf("idxid: %d", idxid);
 674         return false;
 675     }
 676
 677     return true;
 678 }
 679
 680 #endif /* !__PCTOOL__ */
 681
 682 static bool open_files(struct tagcache_search *tcs, int tag)
 683 {
 684     if (tcs->idxfd[tag] < 0)
 685     {
 686         char fn[MAX_PATH];
 687
 688         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 689         tcs->idxfd[tag] = open(fn, O_RDONLY);
 690     }
 691
 692     if (tcs->idxfd[tag] < 0)
 693     {
 694         logf("File not open!");
 695         return false;
 696     }
 697
 698     return true;
 699 }
 700
 701 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 702                      int tag, char *buf, long size)
 703 {
 704     struct tagfile_entry tfe;
 705     long seek;
 706
 707     *buf = '\0';
 708
 709     if (TAGCACHE_IS_NUMERIC(tag))
 710         return false;
 711
 712     seek = idx->tag_seek[tag];
 713     if (seek < 0)
 714     {
 715         logf("Retrieve failed");
 716         return false;
 717     }
 718
 719 #ifdef HAVE_TC_RAMCACHE
 720     if (tcs->ramsearch)
 721     {
 722         struct tagfile_entry *ep;
 723
 724 # ifdef HAVE_DIRCACHE
 725         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 726             && is_dircache_intact())
 727         {
 728             /* for tag_filename, seek is a dircache index */
 729             dircache_copy_path(seek, buf, size);
 730             return true;
 731         }
 732         else
 733 # endif
 734         if (tag != tag_filename)
 735         {
 736             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 737             strlcpy(buf, ep->tag_data, size);
 738
 739             return true;
 740         }
 741     }
 742 #endif
 743
 744     if (!open_files(tcs, tag))
 745         return false;
 746
 747     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 748     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 749         != sizeof(struct tagfile_entry))
 750     {
 751         logf("read error #5");
 752         return false;
 753     }
 754
 755     if (tfe.tag_length >= size)
 756     {
 757         logf("too small buffer");
 758         return false;
 759     }
 760
 761     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 762         tfe.tag_length)
 763     {
 764         logf("read error #6");
 765         return false;
 766     }
 767
 768     buf[tfe.tag_length] = '\0';
 769
 770     return true;
 771 }
 772
 773 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 774
 775 static long find_tag(int tag, int idx_id, const struct index_entry *idx)
 776 {
 777 #ifndef __PCTOOL__
 778     if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag))
 779     {
 780         /* Attempt to find tag data through store-to-load forwarding in
 781            command queue */
 782         long result = -1;
 783
 784         mutex_lock(&command_queue_mutex);
 785
 786         int ridx = command_queue_widx;
 787
 788         while (ridx != command_queue_ridx)
 789         {
 790             if (--ridx < 0)
 791                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 792
 793             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 794                 && command_queue[ridx].idx_id == idx_id
 795                 && command_queue[ridx].tag == tag)
 796             {
 797                 result = command_queue[ridx].data;
 798                 break;
 799             }
 800         }
 801
 802         mutex_unlock(&command_queue_mutex);
 803
 804         if (result >= 0)
 805         {
 806             logf("find_tag: "
 807                  "Recovered tag %d value %lX from write queue",
 808                  tag, result);
 809             return result;
 810         }
 811     }
 812 #endif
 813
 814     return idx->tag_seek[tag];
 815 }
 816
 817
 818 static long check_virtual_tags(int tag, int idx_id,
 819                                const struct index_entry *idx)
 820 {
 821     long data = 0;
 822
 823     switch (tag)
 824     {
 825         case tag_virt_length_sec:
 826             data = (find_tag(tag_length, idx_id, idx)/1000) % 60;
 827             break;
 828
 829         case tag_virt_length_min:
 830             data = (find_tag(tag_length, idx_id, idx)/1000) / 60;
 831             break;
 832
 833         case tag_virt_playtime_sec:
 834             data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60;
 835             break;
 836
 837         case tag_virt_playtime_min:
 838             data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60;
 839             break;
 840
 841         case tag_virt_autoscore:
 842             if (find_tag(tag_length, idx_id, idx) == 0
 843                 || find_tag(tag_playcount, idx_id, idx) == 0)
 844             {
 845                 data = 0;
 846             }
 847             else
 848             {
 849                 /* A straight calculus gives:
 850                      autoscore = 100 * playtime / length / playcout (1)
 851                    Now, consider the euclidian division of playtime by length:
 852                      playtime = alpha * length + beta
 853                    With:
 854                      0 <= beta < length
 855                    Now, (1) becomes:
 856                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 857                    Both terms should be small enough to avoid any overflow
 858                 */
 859                 data = 100 * (find_tag(tag_playtime, idx_id, idx)
 860                               / find_tag(tag_length, idx_id, idx))
 861                        + (100 * (find_tag(tag_playtime, idx_id, idx)
 862                                  % find_tag(tag_length, idx_id, idx)))
 863                          / find_tag(tag_length, idx_id, idx);
 864                 data /=  find_tag(tag_playcount, idx_id, idx);
 865             }
 866             break;
 867
 868         /* How many commits before the file has been added to the DB. */
 869         case tag_virt_entryage:
 870             data = current_tcmh.commitid
 871                    - find_tag(tag_commitid, idx_id, idx) - 1;
 872             break;
 873
 874         default:
 875             data = find_tag(tag, idx_id, idx);
 876     }
 877
 878     return data;
 879 }
 880
 881 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 882 {
 883     struct index_entry idx;
 884
 885     if (!tc_stat.ready)
 886         return false;
 887
 888     if (!TAGCACHE_IS_NUMERIC(tag))
 889         return -1;
 890
 891     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 892         return -2;
 893
 894     return check_virtual_tags(tag, tcs->idx_id, &idx);
 895 }
 896
 897 inline static bool str_ends_with(const char *str1, const char *str2)
 898 {
 899     int str_len = strlen(str1);
 900     int clause_len = strlen(str2);
 901
 902     if (clause_len > str_len)
 903         return false;
 904
 905     return !strcasecmp(&str1[str_len - clause_len], str2);
 906 }
 907
 908 inline static bool str_oneof(const char *str, const char *list)
 909 {
 910     const char *sep;
 911     int l, len = strlen(str);
 912
 913     while (*list)
 914     {
 915         sep = strchr(list, '|');
 916         l = sep ? (long)sep - (long)list : (int)strlen(list);
 917         if ((l==len) && !strncasecmp(str, list, len))
 918             return true;
 919         list += sep ? l + 1 : l;
 920     }
 921
 922     return false;
 923 }
 924
 925 static bool check_against_clause(long numeric, const char *str,
 926                                  const struct tagcache_search_clause *clause)
 927 {
 928     if (clause->numeric)
 929     {
 930         switch (clause->type)
 931         {
 932             case clause_is:
 933                 return numeric == clause->numeric_data;
 934             case clause_is_not:
 935                 return numeric != clause->numeric_data;
 936             case clause_gt:
 937                 return numeric > clause->numeric_data;
 938             case clause_gteq:
 939                 return numeric >= clause->numeric_data;
 940             case clause_lt:
 941                 return numeric < clause->numeric_data;
 942             case clause_lteq:
 943                 return numeric <= clause->numeric_data;
 944             default:
 945                 logf("Incorrect numeric tag: %d", clause->type);
 946         }
 947     }
 948     else
 949     {
 950         switch (clause->type)
 951         {
 952             case clause_is:
 953                 return !strcasecmp(clause->str, str);
 954             case clause_is_not:
 955                 return strcasecmp(clause->str, str);
 956             case clause_gt:
 957                 return 0>strcasecmp(clause->str, str);
 958             case clause_gteq:
 959                 return 0>=strcasecmp(clause->str, str);
 960             case clause_lt:
 961                 return 0<strcasecmp(clause->str, str);
 962             case clause_lteq:
 963                 return 0<=strcasecmp(clause->str, str);
 964             case clause_contains:
 965                 return (strcasestr(str, clause->str) != NULL);
 966             case clause_not_contains:
 967                 return (strcasestr(str, clause->str) == NULL);
 968             case clause_begins_with:
 969                 return (strcasestr(str, clause->str) == str);
 970             case clause_not_begins_with:
 971                 return (strcasestr(str, clause->str) != str);
 972             case clause_ends_with:
 973                 return str_ends_with(str, clause->str);
 974             case clause_not_ends_with:
 975                 return !str_ends_with(str, clause->str);
 976             case clause_oneof:
 977                 return str_oneof(str, clause->str);
 978
 979             default:
 980                 logf("Incorrect tag: %d", clause->type);
 981         }
 982     }
 983
 984     return false;
 985 }
 986
 987 static bool check_clauses(struct tagcache_search *tcs,
 988                           struct index_entry *idx,
 989                           struct tagcache_search_clause **clauses, int count)
 990 {
 991     int i;
 992
 993     /* Go through all conditional clauses. */
 994     for (i = 0; i < count; i++)
 995     {
 996         int seek;
 997         char buf[256];
 998         char *str = buf;
 999         struct tagcache_search_clause *clause = clauses[i];
1000
1001         if (clause->type == clause_logical_or)
1002             break; /* all conditions before logical-or satisfied --
1003                       stop processing clauses */
1004
1005         seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1006
1007 #ifdef HAVE_TC_RAMCACHE
1008         if (tcs->ramsearch)
1009         {
1010             struct tagfile_entry *tfe;
1011
1012             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1013             {
1014                 if (clause->tag == tag_filename)
1015                 {
1016                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1017                 }
1018                 else
1019                 {
1020                     tfe = (struct tagfile_entry *)&hdr->tags[clause->tag][seek];
1021                     str = tfe->tag_data;
1022                 }
1023             }
1024         }
1025         else
1026 #endif
1027         {
1028             struct tagfile_entry tfe;
1029
1030             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1031             {
1032                 int fd = tcs->idxfd[clause->tag];
1033                 lseek(fd, seek, SEEK_SET);
1034                 ecread_tagfile_entry(fd, &tfe);
1035                 if (tfe.tag_length >= (int)sizeof(buf))
1036                 {
1037                     logf("Too long tag read!");
1038                     return false;
1039                 }
1040
1041                 read(fd, str, tfe.tag_length);
1042                 str[tfe.tag_length] = '\0';
1043
1044                 /* Check if entry has been deleted. */
1045                 if (str[0] == '\0')
1046                     return false;
1047             }
1048         }
1049
1050         if (!check_against_clause(seek, str, clause))
1051         {
1052             /* Clause failed -- try finding a logical-or clause */
1053             while (++i < count)
1054             {
1055                 if (clauses[i]->type == clause_logical_or)
1056                     break;
1057             }
1058
1059             if (i < count)        /* Found logical-or? */
1060                 continue;         /* Check clauses after logical-or */
1061
1062             return false;
1063         }
1064     }
1065
1066     return true;
1067 }
1068
1069 bool tagcache_check_clauses(struct tagcache_search *tcs,
1070                             struct tagcache_search_clause **clause, int count)
1071 {
1072     struct index_entry idx;
1073
1074     if (count == 0)
1075         return true;
1076
1077     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1078         return false;
1079
1080     return check_clauses(tcs, &idx, clause, count);
1081 }
1082
1083 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1084 {
1085     int i;
1086
1087     /* If uniq buffer is not defined we must return true for search to work. */
1088     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1089                                      && !TAGCACHE_IS_NUMERIC(tcs->type)))
1090     {
1091         return true;
1092     }
1093
1094     for (i = 0; i < tcs->unique_list_count; i++)
1095     {
1096         /* Return false if entry is found. */
1097         if (tcs->unique_list[i] == id)
1098             return false;
1099     }
1100
1101     if (tcs->unique_list_count < tcs->unique_list_capacity)
1102     {
1103         tcs->unique_list[i] = id;
1104         tcs->unique_list_count++;
1105     }
1106
1107     return true;
1108 }
1109
1110 static bool build_lookup_list(struct tagcache_search *tcs)
1111 {
1112     struct index_entry entry;
1113     int i, j;
1114
1115     tcs->seek_list_count = 0;
1116
1117 #ifdef HAVE_TC_RAMCACHE
1118     if (tcs->ramsearch
1119 # ifdef HAVE_DIRCACHE
1120         && (tcs->type != tag_filename || is_dircache_intact())
1121 # endif
1122         )
1123     {
1124         for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++)
1125         {
1126             struct tagcache_seeklist_entry *seeklist;
1127             struct index_entry *idx = &hdr->indices[i];
1128             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1129                 break ;
1130
1131             /* Skip deleted files. */
1132             if (idx->flag & FLAG_DELETED)
1133                 continue;
1134
1135             /* Go through all filters.. */
1136             for (j = 0; j < tcs->filter_count; j++)
1137             {
1138                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1139                 {
1140                     break ;
1141                 }
1142             }
1143
1144             if (j < tcs->filter_count)
1145                 continue ;
1146
1147             /* Check for conditions. */
1148             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1149                 continue;
1150
1151             /* Add to the seek list if not already in uniq buffer. */
1152             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1153                 continue;
1154
1155             /* Lets add it. */
1156             seeklist = &tcs->seeklist[tcs->seek_list_count];
1157             seeklist->seek = idx->tag_seek[tcs->type];
1158             seeklist->flag = idx->flag;
1159             seeklist->idx_id = i;
1160             tcs->seek_list_count++;
1161         }
1162
1163         tcs->seek_pos = i;
1164
1165         return tcs->seek_list_count > 0;
1166     }
1167 #endif
1168
1169     if (tcs->masterfd < 0)
1170     {
1171         struct master_header tcmh;
1172         tcs->masterfd = open_master_fd(&tcmh, false);
1173     }
1174
1175     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1176             sizeof(struct master_header), SEEK_SET);
1177
1178     while (ecread_index_entry(tcs->masterfd, &entry)
1179            == sizeof(struct index_entry))
1180     {
1181         struct tagcache_seeklist_entry *seeklist;
1182
1183         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1184             break ;
1185
1186         i = tcs->seek_pos;
1187         tcs->seek_pos++;
1188
1189         /* Check if entry has been deleted. */
1190         if (entry.flag & FLAG_DELETED)
1191             continue;
1192
1193         /* Go through all filters.. */
1194         for (j = 0; j < tcs->filter_count; j++)
1195         {
1196             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1197                 break ;
1198         }
1199
1200         if (j < tcs->filter_count)
1201             continue ;
1202
1203         /* Check for conditions. */
1204         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1205             continue;
1206
1207         /* Add to the seek list if not already in uniq buffer. */
1208         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1209             continue;
1210
1211         /* Lets add it. */
1212         seeklist = &tcs->seeklist[tcs->seek_list_count];
1213         seeklist->seek = entry.tag_seek[tcs->type];
1214         seeklist->flag = entry.flag;
1215         seeklist->idx_id = i;
1216         tcs->seek_list_count++;
1217
1218         yield();
1219     }
1220
1221     return tcs->seek_list_count > 0;
1222 }
1223
1224
1225 static void remove_files(void)
1226 {
1227     int i;
1228     char buf[MAX_PATH];
1229
1230     tc_stat.ready = false;
1231     tc_stat.ramcache = false;
1232     tc_stat.econ = false;
1233     remove(TAGCACHE_FILE_MASTER);
1234     for (i = 0; i < TAG_COUNT; i++)
1235     {
1236         if (TAGCACHE_IS_NUMERIC(i))
1237             continue;
1238
1239         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1240         remove(buf);
1241     }
1242 }
1243
1244
1245 static bool check_all_headers(void)
1246 {
1247     struct master_header myhdr;
1248     struct tagcache_header tch;
1249     int tag;
1250     int fd;
1251
1252     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1253         return false;
1254
1255     close(fd);
1256     if (myhdr.dirty)
1257     {
1258         logf("tagcache is dirty!");
1259         return false;
1260     }
1261
1262     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1263
1264     for (tag = 0; tag < TAG_COUNT; tag++)
1265     {
1266         if (TAGCACHE_IS_NUMERIC(tag))
1267             continue;
1268
1269         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1270             return false;
1271
1272         close(fd);
1273     }
1274
1275     return true;
1276 }
1277
1278 bool tagcache_is_busy(void)
1279 {
1280     return read_lock || write_lock;
1281 }
1282
1283 bool tagcache_search(struct tagcache_search *tcs, int tag)
1284 {
1285     struct tagcache_header tag_hdr;
1286     struct master_header   master_hdr;
1287     int i;
1288
1289     while (read_lock)
1290         sleep(1);
1291
1292     memset(tcs, 0, sizeof(struct tagcache_search));
1293     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1294         return false;
1295
1296     tcs->position = sizeof(struct tagcache_header);
1297     tcs->type = tag;
1298     tcs->seek_pos = 0;
1299     tcs->list_position = 0;
1300     tcs->seek_list_count = 0;
1301     tcs->filter_count = 0;
1302     tcs->masterfd = -1;
1303
1304     for (i = 0; i < TAG_COUNT; i++)
1305         tcs->idxfd[i] = -1;
1306
1307 #ifndef HAVE_TC_RAMCACHE
1308     tcs->ramsearch = false;
1309 #else
1310     tcs->ramsearch = tc_stat.ramcache;
1311     if (tcs->ramsearch)
1312     {
1313         tcs->entry_count = hdr->entry_count[tcs->type];
1314     }
1315     else
1316 #endif
1317     {
1318         /* Always open as R/W so we can pass tcs to functions that modify data also
1319          * without failing. */
1320         tcs->masterfd = open_master_fd(&master_hdr, true);
1321         if (tcs->masterfd < 0)
1322             return false;
1323
1324         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1325         {
1326             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1327             if (tcs->idxfd[tcs->type] < 0)
1328                 return false;
1329
1330             tcs->entry_count = tag_hdr.entry_count;
1331         }
1332         else
1333         {
1334             tcs->entry_count = master_hdr.tch.entry_count;
1335         }
1336     }
1337
1338     tcs->valid = true;
1339     tcs->initialized = true;
1340     write_lock++;
1341
1342     return true;
1343 }
1344
1345 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1346                                  void *buffer, long length)
1347 {
1348     tcs->unique_list = (unsigned long *)buffer;
1349     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1350     tcs->unique_list_count = 0;
1351 }
1352
1353 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1354                                 int tag, int seek)
1355 {
1356     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1357         return false;
1358
1359     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1360         return false;
1361
1362     tcs->filter_tag[tcs->filter_count] = tag;
1363     tcs->filter_seek[tcs->filter_count] = seek;
1364     tcs->filter_count++;
1365
1366     return true;
1367 }
1368
1369 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1370                                 struct tagcache_search_clause *clause)
1371 {
1372     int i;
1373
1374     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1375     {
1376         logf("Too many clauses");
1377         return false;
1378     }
1379
1380     if (clause->type != clause_logical_or)
1381     {
1382         /* Check if there is already a similar filter in present (filters are
1383          * much faster than clauses).
1384          */
1385         for (i = 0; i < tcs->filter_count; i++)
1386         {
1387             if (tcs->filter_tag[i] == clause->tag)
1388                 return true;
1389         }
1390
1391         if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1392         {
1393             char buf[MAX_PATH];
1394
1395             snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1396             tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1397         }
1398     }
1399
1400     tcs->clause[tcs->clause_count] = clause;
1401     tcs->clause_count++;
1402
1403     return true;
1404 }
1405
1406 static bool get_next(struct tagcache_search *tcs)
1407 {
1408     static char buf[TAG_MAXLEN+32];
1409     struct tagfile_entry entry;
1410 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1411     long flag = 0;
1412 #endif
1413
1414     if (!tcs->valid || !tc_stat.ready)
1415         return false;
1416
1417     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1418 #ifdef HAVE_TC_RAMCACHE
1419         && !tcs->ramsearch
1420 #endif
1421         )
1422         return false;
1423
1424     /* Relative fetch. */
1425     if (tcs->filter_count > 0 || tcs->clause_count > 0
1426         || TAGCACHE_IS_NUMERIC(tcs->type)
1427 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1428         /* We need to retrieve flag status for dircache. */
1429         || (tcs->ramsearch && tcs->type == tag_filename)
1430 #endif
1431         )
1432     {
1433         struct tagcache_seeklist_entry *seeklist;
1434
1435         /* Check for end of list. */
1436         if (tcs->list_position == tcs->seek_list_count)
1437         {
1438             tcs->list_position = 0;
1439
1440             /* Try to fetch more. */
1441             if (!build_lookup_list(tcs))
1442             {
1443                 tcs->valid = false;
1444                 return false;
1445             }
1446         }
1447
1448         seeklist = &tcs->seeklist[tcs->list_position];
1449 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1450         flag = seeklist->flag;
1451 #endif
1452         tcs->position = seeklist->seek;
1453         tcs->idx_id = seeklist->idx_id;
1454         tcs->list_position++;
1455     }
1456     else
1457     {
1458         if (tcs->entry_count == 0)
1459         {
1460             tcs->valid = false;
1461             return false;
1462         }
1463
1464         tcs->entry_count--;
1465     }
1466
1467     tcs->result_seek = tcs->position;
1468
1469     if (TAGCACHE_IS_NUMERIC(tcs->type))
1470     {
1471         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1472         tcs->result = buf;
1473         tcs->result_len = strlen(buf) + 1;
1474         return true;
1475     }
1476
1477     /* Direct fetch. */
1478 #ifdef HAVE_TC_RAMCACHE
1479     if (tcs->ramsearch)
1480     {
1481
1482 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1483         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1484             && is_dircache_intact())
1485         {
1486             size_t len = dircache_copy_path(tcs->position, buf, sizeof buf);
1487             tcs->result_len = len + 1;
1488             tcs->result = buf;
1489             tcs->ramresult = false;
1490
1491             return true;
1492         }
1493         else
1494 #endif
1495         if (tcs->type != tag_filename)
1496         {
1497             struct tagfile_entry *ep;
1498
1499             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1500             tcs->result = ep->tag_data;
1501             tcs->result_len = strlen(tcs->result) + 1;
1502             tcs->idx_id = ep->idx_id;
1503             tcs->ramresult = true;
1504
1505             /* Increase position for the next run. This may get overwritten. */
1506             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1507
1508             return true;
1509         }
1510     }
1511 #endif
1512
1513     if (!open_files(tcs, tcs->type))
1514     {
1515         tcs->valid = false;
1516         return false;
1517     }
1518
1519     /* Seek stream to the correct position and continue to direct fetch. */
1520     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1521
1522     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1523     {
1524         logf("read error #5");
1525         tcs->valid = false;
1526         return false;
1527     }
1528
1529     if (entry.tag_length > (long)sizeof(buf))
1530     {
1531         tcs->valid = false;
1532         logf("too long tag #2");
1533         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1534         return false;
1535     }
1536
1537     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1538     {
1539         tcs->valid = false;
1540         logf("read error #4");
1541         return false;
1542     }
1543
1544     /**
1545      Update the position for the next read (this may be overridden
1546      if filters or clauses are being used).
1547      */
1548     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1549     tcs->result = buf;
1550     tcs->result_len = strlen(tcs->result) + 1;
1551     tcs->idx_id = entry.idx_id;
1552     tcs->ramresult = false;
1553
1554     return true;
1555 }
1556
1557 bool tagcache_get_next(struct tagcache_search *tcs)
1558 {
1559     while (get_next(tcs))
1560     {
1561         if (tcs->result_len > 1)
1562             return true;
1563     }
1564
1565     return false;
1566 }
1567
1568 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1569                        int tag, char *buf, long size)
1570 {
1571     struct index_entry idx;
1572
1573     *buf = '\0';
1574     if (!get_index(tcs->masterfd, idxid, &idx, true))
1575         return false;
1576
1577     return retrieve(tcs, &idx, tag, buf, size);
1578 }
1579
1580 static bool update_master_header(void)
1581 {
1582     struct master_header myhdr;
1583     int fd;
1584
1585     if (!tc_stat.ready)
1586         return false;
1587
1588     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1589         return false;
1590
1591     myhdr.serial = current_tcmh.serial;
1592     myhdr.commitid = current_tcmh.commitid;
1593     myhdr.dirty = current_tcmh.dirty;
1594
1595     /* Write it back */
1596     lseek(fd, 0, SEEK_SET);
1597     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1598     close(fd);
1599
1600     return true;
1601 }
1602
1603 void tagcache_search_finish(struct tagcache_search *tcs)
1604 {
1605     int i;
1606
1607     if (!tcs->initialized)
1608         return;
1609
1610     if (tcs->masterfd >= 0)
1611     {
1612         close(tcs->masterfd);
1613         tcs->masterfd = -1;
1614     }
1615
1616     for (i = 0; i < TAG_COUNT; i++)
1617     {
1618         if (tcs->idxfd[i] >= 0)
1619         {
1620             close(tcs->idxfd[i]);
1621             tcs->idxfd[i] = -1;
1622         }
1623     }
1624
1625     tcs->ramsearch = false;
1626     tcs->valid = false;
1627     tcs->initialized = 0;
1628     if (write_lock > 0)
1629         write_lock--;
1630 }
1631
1632 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1633 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1634 {
1635     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1636 }
1637
1638 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1639 {
1640     return check_virtual_tags(tag, idx_id, entry);
1641 }
1642
1643 static char* get_tag_string(const struct index_entry *entry, int tag)
1644 {
1645     char* s = get_tag(entry, tag)->tag_data;
1646     return strcmp(s, UNTAGGED) ? s : NULL;
1647 }
1648
1649 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1650 {
1651     struct index_entry *entry;
1652     int idx_id;
1653
1654     if (!tc_stat.ready || !tc_stat.ramcache)
1655         return false;
1656
1657     /* Find the corresponding entry in tagcache. */
1658     idx_id = find_entry_ram(filename, -1);
1659     if (idx_id < 0)
1660         return false;
1661
1662     entry = &hdr->indices[idx_id];
1663
1664     memset(id3, 0, sizeof(struct mp3entry));
1665
1666     id3->title        = get_tag_string(entry, tag_title);
1667     id3->artist       = get_tag_string(entry, tag_artist);
1668     id3->album        = get_tag_string(entry, tag_album);
1669     id3->genre_string = get_tag_string(entry, tag_genre);
1670     id3->composer     = get_tag_string(entry, tag_composer);
1671     id3->comment      = get_tag_string(entry, tag_comment);
1672     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1673     id3->grouping     = get_tag_string(entry, tag_grouping);
1674
1675     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1676     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1677     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1678     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1679     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1680     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1681
1682     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1683     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1684     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1685     if (id3->bitrate == 0)
1686         id3->bitrate = 1;
1687
1688 #if CONFIG_CODEC == SWCODEC
1689     if (global_settings.autoresume_enable)
1690     {
1691         id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1692         logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1693              id3->title, id3->offset);
1694     }
1695 #endif
1696
1697     return true;
1698 }
1699 #endif
1700
1701 static inline void write_item(const char *item)
1702 {
1703     int len = strlen(item) + 1;
1704
1705     data_size += len;
1706     write(cachefd, item, len);
1707 }
1708
1709 static int check_if_empty(char **tag)
1710 {
1711     int length;
1712
1713     if (*tag == NULL || **tag == '\0')
1714     {
1715         *tag = UNTAGGED;
1716         return sizeof(UNTAGGED); /* Tag length */
1717     }
1718
1719     length = strlen(*tag);
1720     if (length > TAG_MAXLEN)
1721     {
1722         logf("over length tag: %s", *tag);
1723         length = TAG_MAXLEN;
1724         (*tag)[length] = '\0';
1725     }
1726
1727     return length + 1;
1728 }
1729
1730 #define ADD_TAG(entry,tag,data) \
1731     /* Adding tag */ \
1732     entry.tag_offset[tag] = offset; \
1733     entry.tag_length[tag] = check_if_empty(data); \
1734     offset += entry.tag_length[tag]
1735 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1736  * idea, as it uses lots of stack and is called from a recursive function
1737  * (check_dir).
1738  */
1739 static void __attribute__ ((noinline)) add_tagcache(char *path,
1740                                                     unsigned long mtime
1741 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1742                                                     ,int dc
1743 #endif
1744                                                    )
1745 {
1746     struct mp3entry id3;
1747     struct temp_file_entry entry;
1748     bool ret;
1749     int fd;
1750     int idx_id = -1;
1751     char tracknumfix[3];
1752     int offset = 0;
1753     int path_length = strlen(path);
1754     bool has_albumartist;
1755     bool has_grouping;
1756
1757 #ifdef SIMULATOR
1758     /* Crude logging for the sim - to aid in debugging */
1759     int logfd = open(ROCKBOX_DIR "/database.log",
1760                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1761     if (logfd >= 0) {
1762         write(logfd, path, strlen(path));
1763         write(logfd, "\n", 1);
1764         close(logfd);
1765     }
1766 #endif
1767
1768     if (cachefd < 0)
1769         return ;
1770
1771     /* Check for overlength file path. */
1772     if (path_length > TAG_MAXLEN)
1773     {
1774         /* Path can't be shortened. */
1775         logf("Too long path: %s", path);
1776         return ;
1777     }
1778
1779     /* Check if the file is supported. */
1780     if (probe_file_format(path) == AFMT_UNKNOWN)
1781         return ;
1782
1783     /* Check if the file is already cached. */
1784 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1785     if (tc_stat.ramcache && is_dircache_intact())
1786     {
1787         idx_id = find_entry_ram(path, dc);
1788     }
1789 #endif
1790
1791     /* Be sure the entry doesn't exist. */
1792     if (filenametag_fd >= 0 && idx_id < 0)
1793         idx_id = find_entry_disk(path, false);
1794
1795     /* Check if file has been modified. */
1796     if (idx_id >= 0)
1797     {
1798         struct index_entry idx;
1799
1800         /* TODO: Mark that the index exists (for fast reverse scan) */
1801         //found_idx[idx_id/8] |= idx_id%8;
1802
1803         if (!get_index(-1, idx_id, &idx, true))
1804         {
1805             logf("failed to retrieve index entry");
1806             return ;
1807         }
1808
1809         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1810         {
1811             /* No changes to file. */
1812             return ;
1813         }
1814
1815         /* Metadata might have been changed. Delete the entry. */
1816         logf("Re-adding: %s", path);
1817         if (!delete_entry(idx_id))
1818         {
1819             logf("delete_entry failed: %d", idx_id);
1820             return ;
1821         }
1822     }
1823
1824     fd = open(path, O_RDONLY);
1825     if (fd < 0)
1826     {
1827         logf("open fail: %s", path);
1828         return ;
1829     }
1830
1831     memset(&id3, 0, sizeof(struct mp3entry));
1832     memset(&entry, 0, sizeof(struct temp_file_entry));
1833     memset(&tracknumfix, 0, sizeof(tracknumfix));
1834     ret = get_metadata(&id3, fd, path);
1835     close(fd);
1836
1837     if (!ret)
1838         return ;
1839
1840     logf("-> %s", path);
1841
1842     if (id3.tracknum <= 0)              /* Track number missing? */
1843     {
1844         id3.tracknum = -1;
1845     }
1846
1847     /* Numeric tags */
1848     entry.tag_offset[tag_year] = id3.year;
1849     entry.tag_offset[tag_discnumber] = id3.discnum;
1850     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1851     entry.tag_offset[tag_length] = id3.length;
1852     entry.tag_offset[tag_bitrate] = id3.bitrate;
1853     entry.tag_offset[tag_mtime] = mtime;
1854
1855     /* String tags. */
1856     has_albumartist = id3.albumartist != NULL
1857         && strlen(id3.albumartist) > 0;
1858     has_grouping = id3.grouping != NULL
1859         && strlen(id3.grouping) > 0;
1860
1861     ADD_TAG(entry, tag_filename, &path);
1862     ADD_TAG(entry, tag_title, &id3.title);
1863     ADD_TAG(entry, tag_artist, &id3.artist);
1864     ADD_TAG(entry, tag_album, &id3.album);
1865     ADD_TAG(entry, tag_genre, &id3.genre_string);
1866     ADD_TAG(entry, tag_composer, &id3.composer);
1867     ADD_TAG(entry, tag_comment, &id3.comment);
1868     if (has_albumartist)
1869     {
1870         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1871     }
1872     else
1873     {
1874         ADD_TAG(entry, tag_albumartist, &id3.artist);
1875     }
1876     if (has_grouping)
1877     {
1878         ADD_TAG(entry, tag_grouping, &id3.grouping);
1879     }
1880     else
1881     {
1882         ADD_TAG(entry, tag_grouping, &id3.title);
1883     }
1884     entry.data_length = offset;
1885
1886     /* Write the header */
1887     write(cachefd, &entry, sizeof(struct temp_file_entry));
1888
1889     /* And tags also... Correct order is critical */
1890     write_item(path);
1891     write_item(id3.title);
1892     write_item(id3.artist);
1893     write_item(id3.album);
1894     write_item(id3.genre_string);
1895     write_item(id3.composer);
1896     write_item(id3.comment);
1897     if (has_albumartist)
1898     {
1899         write_item(id3.albumartist);
1900     }
1901     else
1902     {
1903         write_item(id3.artist);
1904     }
1905     if (has_grouping)
1906     {
1907         write_item(id3.grouping);
1908     }
1909     else
1910     {
1911         write_item(id3.title);
1912     }
1913     total_entry_count++;
1914 }
1915
1916 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1917 {
1918     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1919     int len = strlen(str)+1;
1920     int i;
1921     unsigned crc32;
1922     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1923     char buf[TAG_MAXLEN+32];
1924
1925     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1926         buf[i] = tolower(str[i]);
1927     buf[i] = '\0';
1928
1929     crc32 = crc_32(buf, i, 0xffffffff);
1930
1931     if (unique)
1932     {
1933         /* Check if the crc does not exist -> entry does not exist for sure. */
1934         for (i = 0; i < tempbufidx; i++)
1935         {
1936             if (crcbuf[-i] != crc32)
1937                 continue;
1938
1939             if (!strcasecmp(str, index[i].str))
1940             {
1941                 if (id < 0 || id >= lookup_buffer_depth)
1942                 {
1943                     logf("lookup buf overf.: %d", id);
1944                     return false;
1945                 }
1946
1947                 lookup[id] = &index[i];
1948                 return true;
1949             }
1950         }
1951     }
1952
1953     /* Insert to CRC buffer. */
1954     crcbuf[-tempbufidx] = crc32;
1955     tempbuf_left -= 4;
1956
1957     /* Insert it to the buffer. */
1958     tempbuf_left -= len;
1959     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1960         return false;
1961
1962     if (id >= lookup_buffer_depth)
1963     {
1964         logf("lookup buf overf. #2: %d", id);
1965         return false;
1966     }
1967
1968     if (id >= 0)
1969     {
1970         lookup[id] = &index[tempbufidx];
1971         index[tempbufidx].idlist.id = id;
1972     }
1973     else
1974         index[tempbufidx].idlist.id = -1;
1975
1976     index[tempbufidx].idlist.next = NULL;
1977     index[tempbufidx].idx_id = idx_id;
1978     index[tempbufidx].seek = -1;
1979     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1980     memcpy(index[tempbufidx].str, str, len);
1981     tempbuf_pos += len;
1982     tempbufidx++;
1983
1984     return true;
1985 }
1986
1987 static int compare(const void *p1, const void *p2)
1988 {
1989     do_timed_yield();
1990
1991     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1992     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1993
1994     if (strcmp(e1->str, UNTAGGED) == 0)
1995     {
1996         if (strcmp(e2->str, UNTAGGED) == 0)
1997             return 0;
1998         return -1;
1999     }
2000     else if (strcmp(e2->str, UNTAGGED) == 0)
2001         return 1;
2002
2003     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2004 }
2005
2006 static int tempbuf_sort(int fd)
2007 {
2008     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2009     struct tagfile_entry fe;
2010     int i;
2011     int length;
2012
2013     /* Generate reverse lookup entries. */
2014     for (i = 0; i < lookup_buffer_depth; i++)
2015     {
2016         struct tempbuf_id_list *idlist;
2017
2018         if (!lookup[i])
2019             continue;
2020
2021         if (lookup[i]->idlist.id == i)
2022             continue;
2023
2024         idlist = &lookup[i]->idlist;
2025         while (idlist->next != NULL)
2026             idlist = idlist->next;
2027
2028         tempbuf_left -= sizeof(struct tempbuf_id_list);
2029         if (tempbuf_left - 4 < 0)
2030             return -1;
2031
2032         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2033         if (tempbuf_pos & 0x03)
2034         {
2035             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2036             tempbuf_left -= 3;
2037             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2038         }
2039         tempbuf_pos += sizeof(struct tempbuf_id_list);
2040
2041         idlist = idlist->next;
2042         idlist->id = i;
2043         idlist->next = NULL;
2044
2045         do_timed_yield();
2046     }
2047
2048     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2049     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2050
2051     for (i = 0; i < tempbufidx; i++)
2052     {
2053         struct tempbuf_id_list *idlist = &index[i].idlist;
2054
2055         /* Fix the lookup list. */
2056         while (idlist != NULL)
2057         {
2058             if (idlist->id >= 0)
2059                 lookup[idlist->id] = &index[i];
2060             idlist = idlist->next;
2061         }
2062
2063         index[i].seek = lseek(fd, 0, SEEK_CUR);
2064         length = strlen(index[i].str) + 1;
2065         fe.tag_length = length;
2066         fe.idx_id = index[i].idx_id;
2067
2068         /* Check the chunk alignment. */
2069         if ((fe.tag_length + sizeof(struct tagfile_entry))
2070             % TAGFILE_ENTRY_CHUNK_LENGTH)
2071         {
2072             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2073                 ((fe.tag_length + sizeof(struct tagfile_entry))
2074                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2075         }
2076
2077 #ifdef TAGCACHE_STRICT_ALIGN
2078         /* Make sure the entry is long aligned. */
2079         if (index[i].seek & 0x03)
2080         {
2081             logf("tempbuf_sort: alignment error!");
2082             return -3;
2083         }
2084 #endif
2085
2086         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2087             sizeof(struct tagfile_entry))
2088         {
2089             logf("tempbuf_sort: write error #1");
2090             return -1;
2091         }
2092
2093         if (write(fd, index[i].str, length) != length)
2094         {
2095             logf("tempbuf_sort: write error #2");
2096             return -2;
2097         }
2098
2099         /* Write some padding. */
2100         if (fe.tag_length - length > 0)
2101             write(fd, "XXXXXXXX", fe.tag_length - length);
2102     }
2103
2104     return i;
2105 }
2106
2107 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2108 {
2109     if (id < 0 || id >= lookup_buffer_depth)
2110         return NULL;
2111
2112     return lookup[id];
2113 }
2114
2115
2116 inline static int tempbuf_find_location(int id)
2117 {
2118     struct tempbuf_searchidx *entry;
2119
2120     entry = tempbuf_locate(id);
2121     if (entry == NULL)
2122         return -1;
2123
2124     return entry->seek;
2125 }
2126
2127 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2128 {
2129     struct master_header tcmh;
2130     struct index_entry idx;
2131     int masterfd;
2132     int masterfd_pos;
2133     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2134     int max_entries;
2135     int entries_processed = 0;
2136     int i, j;
2137     char buf[TAG_MAXLEN];
2138
2139     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2140
2141     logf("Building numeric indices...");
2142     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2143
2144     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2145         return false;
2146
2147     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2148                          SEEK_CUR);
2149     if (masterfd_pos == filesize(masterfd))
2150     {
2151         logf("we can't append!");
2152         close(masterfd);
2153         return false;
2154     }
2155
2156     while (entries_processed < h->entry_count)
2157     {
2158         int count = MIN(h->entry_count - entries_processed, max_entries);
2159
2160         /* Read in as many entries as possible. */
2161         for (i = 0; i < count; i++)
2162         {
2163             struct temp_file_entry *tfe = &entrybuf[i];
2164             int datastart;
2165
2166             /* Read in numeric data. */
2167             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2168                 sizeof(struct temp_file_entry))
2169             {
2170                 logf("read fail #1");
2171                 close(masterfd);
2172                 return false;
2173             }
2174
2175             datastart = lseek(tmpfd, 0, SEEK_CUR);
2176
2177             /**
2178              * Read string data from the following tags:
2179              * - tag_filename
2180              * - tag_artist
2181              * - tag_album
2182              * - tag_title
2183              *
2184              * A crc32 hash is calculated from the read data
2185              * and stored back to the data offset field kept in memory.
2186              */
2187 #define tmpdb_read_string_tag(tag) \
2188     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2189     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2190     { \
2191         logf("read fail: buffer overflow"); \
2192         close(masterfd); \
2193         return false; \
2194     } \
2195     \
2196     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2197         tfe->tag_length[tag]) \
2198     { \
2199         logf("read fail #2"); \
2200         close(masterfd); \
2201         return false; \
2202     } \
2203     \
2204     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2205     lseek(tmpfd, datastart, SEEK_SET)
2206
2207             tmpdb_read_string_tag(tag_filename);
2208             tmpdb_read_string_tag(tag_artist);
2209             tmpdb_read_string_tag(tag_album);
2210             tmpdb_read_string_tag(tag_title);
2211
2212             /* Seek to the end of the string data. */
2213             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2214         }
2215
2216         /* Backup the master index position. */
2217         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2218         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2219
2220         /* Check if we can resurrect some deleted runtime statistics data. */
2221         for (i = 0; i < tcmh.tch.entry_count; i++)
2222         {
2223             /* Read the index entry. */
2224             if (ecread_index_entry(masterfd, &idx)
2225                 != sizeof(struct index_entry))
2226             {
2227                 logf("read fail #3");
2228                 close(masterfd);
2229                 return false;
2230             }
2231
2232             /**
2233              * Skip unless the entry is marked as being deleted
2234              * or the data has already been resurrected.
2235              */
2236             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2237                 continue;
2238
2239             /* Now try to match the entry. */
2240             /**
2241              * To succesfully match a song, the following conditions
2242              * must apply:
2243              *
2244              * For numeric fields: tag_length
2245              * - Full identical match is required
2246              *
2247              * If tag_filename matches, no further checking necessary.
2248              *
2249              * For string hashes: tag_artist, tag_album, tag_title
2250              * - All three of these must match
2251              */
2252             for (j = 0; j < count; j++)
2253             {
2254                 struct temp_file_entry *tfe = &entrybuf[j];
2255
2256                 /* Try to match numeric fields first. */
2257                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2258                     continue;
2259
2260                 /* Now it's time to do the hash matching. */
2261                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2262                 {
2263                     int match_count = 0;
2264
2265                     /* No filename match, check if we can match two other tags. */
2266 #define tmpdb_match(tag) \
2267     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2268         match_count++
2269
2270                     tmpdb_match(tag_artist);
2271                     tmpdb_match(tag_album);
2272                     tmpdb_match(tag_title);
2273
2274                     if (match_count < 3)
2275                     {
2276                         /* Still no match found, give up. */
2277                         continue;
2278                     }
2279                 }
2280
2281                 /* A match found, now copy & resurrect the statistical data. */
2282 #define tmpdb_copy_tag(tag) \
2283     tfe->tag_offset[tag] = idx.tag_seek[tag]
2284
2285                 tmpdb_copy_tag(tag_playcount);
2286                 tmpdb_copy_tag(tag_rating);
2287                 tmpdb_copy_tag(tag_playtime);
2288                 tmpdb_copy_tag(tag_lastplayed);
2289                 tmpdb_copy_tag(tag_commitid);
2290                 tmpdb_copy_tag(tag_lastoffset);
2291
2292                 /* Avoid processing this entry again. */
2293                 idx.flag |= FLAG_RESURRECTED;
2294
2295                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2296                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2297                 {
2298                     logf("masterfd writeback fail #1");
2299                     close(masterfd);
2300                     return false;
2301                 }
2302
2303                 logf("Entry resurrected");
2304             }
2305         }
2306
2307
2308         /* Restore the master index position. */
2309         lseek(masterfd, masterfd_pos, SEEK_SET);
2310
2311         /* Commit the data to the index. */
2312         for (i = 0; i < count; i++)
2313         {
2314             int loc = lseek(masterfd, 0, SEEK_CUR);
2315
2316             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2317             {
2318                 logf("read fail #3");
2319                 close(masterfd);
2320                 return false;
2321             }
2322
2323             for (j = 0; j < TAG_COUNT; j++)
2324             {
2325                 if (!TAGCACHE_IS_NUMERIC(j))
2326                     continue;
2327
2328                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2329             }
2330             idx.flag = entrybuf[i].flag;
2331
2332             if (idx.tag_seek[tag_commitid])
2333             {
2334                 /* Data has been resurrected. */
2335                 idx.flag |= FLAG_DIRTYNUM;
2336             }
2337             else if (tc_stat.ready && current_tcmh.commitid > 0)
2338             {
2339                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2340                 idx.flag |= FLAG_DIRTYNUM;
2341             }
2342
2343             /* Write back the updated index. */
2344             lseek(masterfd, loc, SEEK_SET);
2345             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2346             {
2347                 logf("write fail");
2348                 close(masterfd);
2349                 return false;
2350             }
2351         }
2352
2353         entries_processed += count;
2354         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2355     }
2356
2357     close(masterfd);
2358
2359     return true;
2360 }
2361
2362 /**
2363  * Return values:
2364  *     > 0   success
2365  *    == 0   temporary failure
2366  *     < 0   fatal error
2367  */
2368 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2369 {
2370     int i;
2371     struct tagcache_header tch;
2372     struct master_header   tcmh;
2373     struct index_entry idxbuf[IDX_BUF_DEPTH];
2374     int idxbuf_pos;
2375     char buf[TAG_MAXLEN+32];
2376     int fd = -1, masterfd;
2377     bool error = false;
2378     int init;
2379     int masterfd_pos;
2380
2381     logf("Building index: %d", index_type);
2382
2383     /* Check the number of entries we need to allocate ram for. */
2384     commit_entry_count = h->entry_count + 1;
2385
2386     masterfd = open_master_fd(&tcmh, false);
2387     if (masterfd >= 0)
2388     {
2389         commit_entry_count += tcmh.tch.entry_count;
2390         close(masterfd);
2391     }
2392     else
2393         remove_files(); /* Just to be sure we are clean. */
2394
2395     /* Open the index file, which contains the tag names. */
2396     fd = open_tag_fd(&tch, index_type, true);
2397     if (fd >= 0)
2398     {
2399         logf("tch.datasize=%ld", tch.datasize);
2400         lookup_buffer_depth = 1 +
2401         /* First part */ commit_entry_count +
2402         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2403     }
2404     else
2405     {
2406         lookup_buffer_depth = 1 +
2407         /* First part */ commit_entry_count +
2408         /* Second part */ 0;
2409     }
2410
2411     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2412     logf("commit_entry_count=%ld", commit_entry_count);
2413
2414     /* Allocate buffer for all index entries from both old and new
2415      * tag files. */
2416     tempbufidx = 0;
2417     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2418
2419     /* Allocate lookup buffer. The first portion of commit_entry_count
2420      * contains the new tags in the temporary file and the second
2421      * part for locating entries already in the db.
2422      *
2423      *  New tags  Old tags
2424      * +---------+---------------------------+
2425      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2426      * +---------+---------------------------+
2427      *
2428      * Old tags are inserted to a temporary buffer with position:
2429      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2430      * And new tags with index:
2431      *     tempbuf_insert(idx, ...);
2432      *
2433      * The buffer is sorted and written into tag file:
2434      *     tempbuf_sort(...);
2435      * leaving master index locations messed up.
2436      *
2437      * That is fixed using the lookup buffer for old tags:
2438      *     new_seek = tempbuf_find_location(old_seek, ...);
2439      * and for new tags:
2440      *     new_seek = tempbuf_find_location(idx);
2441      */
2442     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2443     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2444     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2445
2446     /* And calculate the remaining data space used mainly for storing
2447      * tag data (strings). */
2448     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2449     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2450     {
2451         logf("Buffer way too small!");
2452         return 0;
2453     }
2454
2455     if (fd >= 0)
2456     {
2457         /**
2458          * If tag file contains unique tags (sorted index), we will load
2459          * it entirely into memory so we can resort it later for use with
2460          * chunked browsing.
2461          */
2462         if (TAGCACHE_IS_SORTED(index_type))
2463         {
2464             logf("loading tags...");
2465             for (i = 0; i < tch.entry_count; i++)
2466             {
2467                 struct tagfile_entry entry;
2468                 int loc = lseek(fd, 0, SEEK_CUR);
2469                 bool ret;
2470
2471                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2472                 {
2473                     logf("read error #7");
2474                     close(fd);
2475                     return -2;
2476                 }
2477
2478                 if (entry.tag_length >= (int)sizeof(buf))
2479                 {
2480                     logf("too long tag #3");
2481                     close(fd);
2482                     return -2;
2483                 }
2484
2485                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2486                 {
2487                     logf("read error #8");
2488                     close(fd);
2489                     return -2;
2490                 }
2491
2492                 /* Skip deleted entries. */
2493                 if (buf[0] == '\0')
2494                     continue;
2495
2496                 /**
2497                  * Save the tag and tag id in the memory buffer. Tag id
2498                  * is saved so we can later reindex the master lookup
2499                  * table when the index gets resorted.
2500                  */
2501                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2502                                      + commit_entry_count, entry.idx_id,
2503                                      TAGCACHE_IS_UNIQUE(index_type));
2504                 if (!ret)
2505                 {
2506                     close(fd);
2507                     return -3;
2508                 }
2509                 do_timed_yield();
2510             }
2511             logf("done");
2512         }
2513         else
2514             tempbufidx = tch.entry_count;
2515     }
2516     else
2517     {
2518         /**
2519          * Creating new index file to store the tags. No need to preload
2520          * anything whether the index type is sorted or not.
2521          */
2522         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2523         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2524         if (fd < 0)
2525         {
2526             logf("%s open fail", buf);
2527             return -2;
2528         }
2529
2530         tch.magic = TAGCACHE_MAGIC;
2531         tch.entry_count = 0;
2532         tch.datasize = 0;
2533
2534         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2535             != sizeof(struct tagcache_header))
2536         {
2537             logf("header write failed");
2538             close(fd);
2539             return -2;
2540         }
2541     }
2542
2543     /* Loading the tag lookup file as "master file". */
2544     logf("Loading index file");
2545     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2546
2547     if (masterfd < 0)
2548     {
2549         logf("Creating new DB");
2550         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2551
2552         if (masterfd < 0)
2553         {
2554             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2555             close(fd);
2556             return -2;
2557         }
2558
2559         /* Write the header (write real values later). */
2560         memset(&tcmh, 0, sizeof(struct master_header));
2561         tcmh.tch = *h;
2562         tcmh.tch.entry_count = 0;
2563         tcmh.tch.datasize = 0;
2564         tcmh.dirty = true;
2565         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2566         init = true;
2567         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2568     }
2569     else
2570     {
2571         /**
2572          * Master file already exists so we need to process the current
2573          * file first.
2574          */
2575         init = false;
2576
2577         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2578             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2579         {
2580             logf("header error");
2581             close(fd);
2582             close(masterfd);
2583             return -2;
2584         }
2585
2586         /**
2587          * If we reach end of the master file, we need to expand it to
2588          * hold new tags. If the current index is not sorted, we can
2589          * simply append new data to end of the file.
2590          * However, if the index is sorted, we need to update all tag
2591          * pointers in the master file for the current index.
2592          */
2593         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2594             SEEK_CUR);
2595         if (masterfd_pos == filesize(masterfd))
2596         {
2597             logf("appending...");
2598             init = true;
2599         }
2600     }
2601
2602     /**
2603      * Load new unique tags in memory to be sorted later and added
2604      * to the master lookup file.
2605      */
2606     if (TAGCACHE_IS_SORTED(index_type))
2607     {
2608         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2609         /* h is the header of the temporary file containing new tags. */
2610         logf("inserting new tags...");
2611         for (i = 0; i < h->entry_count; i++)
2612         {
2613             struct temp_file_entry entry;
2614
2615             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2616                 sizeof(struct temp_file_entry))
2617             {
2618                 logf("read fail #3");
2619                 error = true;
2620                 goto error_exit;
2621             }
2622
2623             /* Read data. */
2624             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2625             {
2626                 logf("too long entry!");
2627                 error = true;
2628                 goto error_exit;
2629             }
2630
2631             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2632             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2633                 entry.tag_length[index_type])
2634             {
2635                 logf("read fail #4");
2636                 error = true;
2637                 goto error_exit;
2638             }
2639
2640             if (TAGCACHE_IS_UNIQUE(index_type))
2641                 error = !tempbuf_insert(buf, i, -1, true);
2642             else
2643                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2644
2645             if (error)
2646             {
2647                 logf("insert error");
2648                 goto error_exit;
2649             }
2650
2651             /* Skip to next. */
2652             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2653                     entry.tag_length[index_type], SEEK_CUR);
2654             do_timed_yield();
2655         }
2656         logf("done");
2657
2658         /* Sort the buffer data and write it to the index file. */
2659         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2660         /**
2661          * We need to truncate the index file now. There can be junk left
2662          * at the end of file (however, we _should_ always follow the
2663          * entry_count and don't crash with that).
2664          */
2665         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2666
2667         i = tempbuf_sort(fd);
2668         if (i < 0)
2669             goto error_exit;
2670         logf("sorted %d tags", i);
2671
2672         /**
2673          * Now update all indexes in the master lookup file.
2674          */
2675         logf("updating indices...");
2676         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2677         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2678         {
2679             int j;
2680             int loc = lseek(masterfd, 0, SEEK_CUR);
2681
2682             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2683
2684             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2685                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2686             {
2687                 logf("read fail #5");
2688                 error = true;
2689                 goto error_exit ;
2690             }
2691             lseek(masterfd, loc, SEEK_SET);
2692
2693             for (j = 0; j < idxbuf_pos; j++)
2694             {
2695                 if (idxbuf[j].flag & FLAG_DELETED)
2696                 {
2697                     /* We can just ignore deleted entries. */
2698                     // idxbuf[j].tag_seek[index_type] = 0;
2699                     continue;
2700                 }
2701
2702                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2703                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2704                     + commit_entry_count);
2705
2706                 if (idxbuf[j].tag_seek[index_type] < 0)
2707                 {
2708                     logf("update error: %ld/%d/%ld",
2709                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2710                     error = true;
2711                     goto error_exit;
2712                 }
2713
2714                 do_timed_yield();
2715             }
2716
2717             /* Write back the updated index. */
2718             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2719                         index_entry_ec, tc_stat.econ) !=
2720                 (int)sizeof(struct index_entry)*idxbuf_pos)
2721             {
2722                 logf("write fail");
2723                 error = true;
2724                 goto error_exit;
2725             }
2726         }
2727         logf("done");
2728     }
2729
2730     /**
2731      * Walk through the temporary file containing the new tags.
2732      */
2733     // build_normal_index(h, tmpfd, masterfd, idx);
2734     logf("updating new indices...");
2735     lseek(masterfd, masterfd_pos, SEEK_SET);
2736     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2737     lseek(fd, 0, SEEK_END);
2738     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2739     {
2740         int j;
2741
2742         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2743         if (init)
2744         {
2745             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2746         }
2747         else
2748         {
2749             int loc = lseek(masterfd, 0, SEEK_CUR);
2750
2751             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2752                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2753             {
2754                 logf("read fail #6");
2755                 error = true;
2756                 break ;
2757             }
2758             lseek(masterfd, loc, SEEK_SET);
2759         }
2760
2761         /* Read entry headers. */
2762         for (j = 0; j < idxbuf_pos; j++)
2763         {
2764             if (!TAGCACHE_IS_SORTED(index_type))
2765             {
2766                 struct temp_file_entry entry;
2767                 struct tagfile_entry fe;
2768
2769                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2770                     sizeof(struct temp_file_entry))
2771                 {
2772                     logf("read fail #7");
2773                     error = true;
2774                     break ;
2775                 }
2776
2777                 /* Read data. */
2778                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2779                 {
2780                     logf("too long entry!");
2781                     logf("length=%d", entry.tag_length[index_type]);
2782                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2783                     error = true;
2784                     break ;
2785                 }
2786
2787                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2788                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2789                     entry.tag_length[index_type])
2790                 {
2791                     logf("read fail #8");
2792                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2793                     logf("length=0x%02x", entry.tag_length[index_type]);
2794                     error = true;
2795                     break ;
2796                 }
2797
2798                 /* Write to index file. */
2799                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2800                 fe.tag_length = entry.tag_length[index_type];
2801                 fe.idx_id = tcmh.tch.entry_count + i + j;
2802                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2803                 write(fd, buf, fe.tag_length);
2804                 tempbufidx++;
2805
2806                 /* Skip to next. */
2807                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2808                       entry.tag_length[index_type], SEEK_CUR);
2809             }
2810             else
2811             {
2812                 /* Locate the correct entry from the sorted array. */
2813                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2814                 if (idxbuf[j].tag_seek[index_type] < 0)
2815                 {
2816                     logf("entry not found (%d)", j);
2817                     error = true;
2818                     break ;
2819                 }
2820             }
2821         }
2822
2823         /* Write index. */
2824         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2825                   index_entry_ec, tc_stat.econ) !=
2826             (int)sizeof(struct index_entry)*idxbuf_pos)
2827         {
2828             logf("tagcache: write fail #4");
2829             error = true;
2830             break ;
2831         }
2832
2833         do_timed_yield();
2834     }
2835     logf("done");
2836
2837     /* Finally write the header. */
2838     tch.magic = TAGCACHE_MAGIC;
2839     tch.entry_count = tempbufidx;
2840     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2841     lseek(fd, 0, SEEK_SET);
2842     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2843
2844     if (index_type != tag_filename)
2845         h->datasize += tch.datasize;
2846     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2847     error_exit:
2848
2849     close(fd);
2850     close(masterfd);
2851
2852     if (error)
2853         return -2;
2854
2855     return 1;
2856 }
2857
2858 static bool commit(void)
2859 {
2860     struct tagcache_header tch;
2861     struct master_header   tcmh;
2862     int i, len, rc;
2863     int tmpfd;
2864     int masterfd;
2865 #ifdef HAVE_DIRCACHE
2866     bool dircache_buffer_stolen = false;
2867 #endif
2868     bool local_allocation = false;
2869
2870     logf("committing tagcache");
2871
2872     while (write_lock)
2873         sleep(1);
2874
2875     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2876     if (tmpfd < 0)
2877     {
2878         logf("nothing to commit");
2879         return true;
2880     }
2881
2882
2883     /* Load the header. */
2884     len = sizeof(struct tagcache_header);
2885     rc = read(tmpfd, &tch, len);
2886
2887     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2888     {
2889         logf("incorrect tmpheader");
2890         close(tmpfd);
2891         remove(TAGCACHE_FILE_TEMP);
2892         return false;
2893     }
2894
2895     if (tch.entry_count == 0)
2896     {
2897         logf("nothing to commit");
2898         close(tmpfd);
2899         remove(TAGCACHE_FILE_TEMP);
2900         return true;
2901     }
2902
2903     /* Fully initialize existing headers (if any) before going further. */
2904     tc_stat.ready = check_all_headers();
2905
2906 #ifdef HAVE_EEPROM_SETTINGS
2907     remove(TAGCACHE_STATEFILE);
2908 #endif
2909
2910     /* At first be sure to unload the ramcache! */
2911 #ifdef HAVE_TC_RAMCACHE
2912     tc_stat.ramcache = false;
2913 #endif
2914
2915     read_lock++;
2916
2917     /* Try to steal every buffer we can :) */
2918     if (tempbuf_size == 0)
2919         local_allocation = true;
2920
2921 #ifdef HAVE_DIRCACHE
2922     if (tempbuf_size == 0)
2923     {
2924         /* Try to steal the dircache buffer. */
2925         tempbuf = dircache_steal_buffer(&tempbuf_size);
2926         tempbuf_size &= ~0x03;
2927
2928         if (tempbuf_size > 0)
2929         {
2930             dircache_buffer_stolen = true;
2931         }
2932     }
2933 #endif
2934
2935 #ifdef HAVE_TC_RAMCACHE
2936     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2937     {
2938         tempbuf = (char *)(hdr + 1);
2939         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2940         tempbuf_size &= ~0x03;
2941     }
2942 #endif
2943
2944     /* And finally fail if there are no buffers available. */
2945     if (tempbuf_size == 0)
2946     {
2947         logf("delaying commit until next boot");
2948         tc_stat.commit_delayed = true;
2949         close(tmpfd);
2950         read_lock--;
2951         return false;
2952     }
2953
2954     logf("commit %ld entries...", tch.entry_count);
2955
2956     /* Mark DB dirty so it will stay disabled if commit fails. */
2957     current_tcmh.dirty = true;
2958     update_master_header();
2959
2960     /* Now create the index files. */
2961     tc_stat.commit_step = 0;
2962     tch.datasize = 0;
2963     tc_stat.commit_delayed = false;
2964
2965     for (i = 0; i < TAG_COUNT; i++)
2966     {
2967         int ret;
2968
2969         if (TAGCACHE_IS_NUMERIC(i))
2970             continue;
2971
2972         tc_stat.commit_step++;
2973         ret = build_index(i, &tch, tmpfd);
2974         if (ret <= 0)
2975         {
2976             close(tmpfd);
2977             logf("tagcache failed init");
2978             if (ret == 0)
2979                 tc_stat.commit_delayed = true;
2980
2981             tc_stat.commit_step = 0;
2982             read_lock--;
2983             return false;
2984         }
2985     }
2986
2987     if (!build_numeric_indices(&tch, tmpfd))
2988     {
2989         logf("Failure to commit numeric indices");
2990         close(tmpfd);
2991         tc_stat.commit_step = 0;
2992         read_lock--;
2993         return false;
2994     }
2995
2996     close(tmpfd);
2997
2998     tc_stat.commit_step = 0;
2999
3000     /* Update the master index headers. */
3001     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3002     {
3003         read_lock--;
3004         return false;
3005     }
3006
3007     remove(TAGCACHE_FILE_TEMP);
3008
3009     tcmh.tch.entry_count += tch.entry_count;
3010     tcmh.tch.datasize = sizeof(struct master_header)
3011         + sizeof(struct index_entry) * tcmh.tch.entry_count
3012         + tch.datasize;
3013     tcmh.dirty = false;
3014     tcmh.commitid++;
3015
3016     lseek(masterfd, 0, SEEK_SET);
3017     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3018     close(masterfd);
3019
3020     logf("tagcache committed");
3021     tc_stat.ready = check_all_headers();
3022     tc_stat.readyvalid = true;
3023
3024     if (local_allocation)
3025     {
3026         tempbuf = NULL;
3027         tempbuf_size = 0;
3028     }
3029
3030 #ifdef HAVE_DIRCACHE
3031     /* Rebuild the dircache, if we stole the buffer. */
3032     if (dircache_buffer_stolen)
3033         dircache_build(0);
3034 #endif
3035
3036 #ifdef HAVE_TC_RAMCACHE
3037     /* Reload tagcache. */
3038     if (tc_stat.ramcache_allocated > 0)
3039         tagcache_start_scan();
3040 #endif
3041
3042     read_lock--;
3043
3044     return true;
3045 }
3046
3047 static void allocate_tempbuf(void)
3048 {
3049     /* Yeah, malloc would be really nice now :) */
3050 #ifdef __PCTOOL__
3051     tempbuf_size = 32*1024*1024;
3052     tempbuf = malloc(tempbuf_size);
3053 #else
3054     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3055     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3056     audiobuf += tempbuf_size;
3057 #endif
3058 }
3059
3060 static void free_tempbuf(void)
3061 {
3062     if (tempbuf_size == 0)
3063         return ;
3064
3065 #ifdef __PCTOOL__
3066     free(tempbuf);
3067 #else
3068     audiobuf -= tempbuf_size;
3069 #endif
3070     tempbuf = NULL;
3071     tempbuf_size = 0;
3072 }
3073
3074 #ifndef __PCTOOL__
3075
3076 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3077 {
3078     struct index_entry idx;
3079
3080     if (!tc_stat.ready)
3081         return false;
3082
3083     if (!TAGCACHE_IS_NUMERIC(tag))
3084         return false;
3085
3086     if (!get_index(masterfd, idx_id, &idx, false))
3087         return false;
3088
3089     idx.tag_seek[tag] = data;
3090     idx.flag |= FLAG_DIRTYNUM;
3091
3092     return write_index(masterfd, idx_id, &idx);
3093 }
3094
3095 #if 0
3096 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3097                                    int tag, long data)
3098 {
3099     struct master_header myhdr;
3100
3101     if (tcs->masterfd < 0)
3102     {
3103         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3104             return false;
3105     }
3106
3107     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3108 }
3109 #endif
3110
3111 static bool command_queue_is_full(void)
3112 {
3113     int next;
3114
3115     next = command_queue_widx + 1;
3116     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3117         next = 0;
3118
3119     return (next == command_queue_ridx);
3120 }
3121
3122 static void command_queue_sync_callback(void *data)
3123 {
3124     (void)data;
3125     struct master_header myhdr;
3126     int masterfd;
3127
3128     mutex_lock(&command_queue_mutex);
3129
3130     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3131         return;
3132
3133     while (command_queue_ridx != command_queue_widx)
3134     {
3135         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3136
3137         switch (ce->command)
3138         {
3139             case CMD_UPDATE_MASTER_HEADER:
3140             {
3141                 close(masterfd);
3142                 update_master_header();
3143
3144                 /* Re-open the masterfd. */
3145                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3146                     return;
3147
3148                 break;
3149             }
3150             case CMD_UPDATE_NUMERIC:
3151             {
3152                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3153                 break;
3154             }
3155         }
3156
3157         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3158             command_queue_ridx = 0;
3159     }
3160
3161     close(masterfd);
3162
3163     tc_stat.queue_length = 0;
3164     mutex_unlock(&command_queue_mutex);
3165 }
3166
3167 static void run_command_queue(bool force)
3168 {
3169     if (COMMAND_QUEUE_IS_EMPTY)
3170         return;
3171
3172     if (force || command_queue_is_full())
3173         command_queue_sync_callback(NULL);
3174     else
3175         register_storage_idle_func(command_queue_sync_callback);
3176 }
3177
3178 static void queue_command(int cmd, long idx_id, int tag, long data)
3179 {
3180     while (1)
3181     {
3182         int next;
3183
3184         mutex_lock(&command_queue_mutex);
3185         next = command_queue_widx + 1;
3186         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3187             next = 0;
3188
3189         /* Make sure queue is not full. */
3190         if (next != command_queue_ridx)
3191         {
3192             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3193
3194             ce->command = cmd;
3195             ce->idx_id = idx_id;
3196             ce->tag = tag;
3197             ce->data = data;
3198
3199             command_queue_widx = next;
3200
3201             tc_stat.queue_length++;
3202
3203             mutex_unlock(&command_queue_mutex);
3204             break;
3205         }
3206
3207         /* Queue is full, try again later... */
3208         mutex_unlock(&command_queue_mutex);
3209         sleep(1);
3210     }
3211 }
3212
3213 long tagcache_increase_serial(void)
3214 {
3215     long old;
3216
3217     if (!tc_stat.ready)
3218         return -2;
3219
3220     while (read_lock)
3221         sleep(1);
3222
3223     old = current_tcmh.serial++;
3224     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3225
3226     return old;
3227 }
3228
3229 void tagcache_update_numeric(int idx_id, int tag, long data)
3230 {
3231     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3232 }
3233 #endif /* !__PCTOOL__ */
3234
3235 long tagcache_get_serial(void)
3236 {
3237     return current_tcmh.serial;
3238 }
3239
3240 long tagcache_get_commitid(void)
3241 {
3242     return current_tcmh.commitid;
3243 }
3244
3245 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3246 {
3247     char buf[512];
3248     int i;
3249
3250     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3251     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3252     {
3253         if (*datastr == '\0')
3254             break;
3255
3256         if (*datastr == '"' || *datastr == '\\')
3257             buf[i++] = '\\';
3258
3259         else if (*datastr == '\n')
3260         {
3261             buf[i++] = '\\';
3262             buf[i] = 'n';
3263             continue;
3264         }
3265
3266         buf[i] = *(datastr++);
3267     }
3268
3269     strcpy(&buf[i], "\" ");
3270
3271     write(fd, buf, i + 2);
3272
3273     return true;
3274 }
3275
3276 #ifndef __PCTOOL__
3277
3278 static bool read_tag(char *dest, long size,
3279                      const char *src, const char *tagstr)
3280 {
3281     int pos;
3282     char current_tag[32];
3283
3284     while (*src != '\0')
3285     {
3286         /* Skip all whitespace */
3287         while (*src == ' ')
3288             src++;
3289
3290         if (*src == '\0')
3291             break;
3292
3293         pos = 0;
3294         /* Read in tag name */
3295         while (*src != '=' && *src != ' ')
3296         {
3297             current_tag[pos] = *src;
3298             src++;
3299             pos++;
3300
3301             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3302                 return false;
3303         }
3304         current_tag[pos] = '\0';
3305
3306         /* Read in tag data */
3307
3308         /* Find the start. */
3309         while (*src != '"' && *src != '\0')
3310             src++;
3311
3312         if (*src == '\0' || *(++src) == '\0')
3313             return false;
3314
3315         /* Read the data. */
3316         for (pos = 0; pos < size; pos++)
3317         {
3318             if (*src == '\0')
3319                 break;
3320
3321             if (*src == '\\')
3322             {
3323                 src++;
3324                 if (*src == 'n')
3325                     dest[pos] = '\n';
3326                 else
3327                     dest[pos] = *src;
3328
3329                 src++;
3330                 continue;
3331             }
3332
3333             if (*src == '\0')
3334                 break;
3335
3336             if (*src == '"')
3337             {
3338                 src++;
3339                 break;
3340             }
3341
3342             dest[pos] = *(src++);
3343         }
3344
3345         dest[pos] = '\0';
3346
3347         if (!strcasecmp(tagstr, current_tag))
3348             return true;
3349     }
3350
3351     return false;
3352 }
3353
3354 static int parse_changelog_line(int line_n, char *buf, void *parameters)
3355 {
3356     struct index_entry idx;
3357     char tag_data[TAG_MAXLEN+32];
3358     int idx_id;
3359     long masterfd = (long)parameters;
3360     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3361                                 tag_lastplayed, tag_commitid, tag_lastoffset };
3362     int i;
3363     (void)line_n;
3364
3365     if (*buf == '#')
3366         return 0;
3367
3368     /* logf("%d/%s", line_n, buf); */
3369     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3370     {
3371         logf("%d/filename missing", line_n);
3372         logf("-> %s", buf);
3373         return 0;
3374     }
3375
3376     idx_id = find_index(tag_data);
3377     if (idx_id < 0)
3378     {
3379         logf("%d/entry not found", line_n);
3380         return 0;
3381     }
3382
3383     if (!get_index(masterfd, idx_id, &idx, false))
3384     {
3385         logf("%d/failed to retrieve index entry", line_n);
3386         return 0;
3387     }
3388
3389     /* Stop if tag has already been modified. */
3390     if (idx.flag & FLAG_DIRTYNUM)
3391         return 0;
3392
3393     logf("%d/import: %s", line_n, tag_data);
3394
3395     idx.flag |= FLAG_DIRTYNUM;
3396     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3397     {
3398         int data;
3399
3400         if (!read_tag(tag_data, sizeof tag_data, buf,
3401                       tagcache_tag_to_str(import_tags[i])))
3402         {
3403             continue;
3404         }
3405
3406         data = atoi(tag_data);
3407         if (data < 0)
3408             continue;
3409
3410         idx.tag_seek[import_tags[i]] = data;
3411
3412         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3413             current_tcmh.serial = data + 1;
3414         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3415             current_tcmh.commitid = data + 1;
3416     }
3417
3418     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3419 }
3420
3421 bool tagcache_import_changelog(void)
3422 {
3423     struct master_header myhdr;
3424     struct tagcache_header tch;
3425     int clfd;
3426     long masterfd;
3427     char buf[2048];
3428
3429     if (!tc_stat.ready)
3430         return false;
3431
3432     while (read_lock)
3433         sleep(1);
3434
3435     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3436     if (clfd < 0)
3437     {
3438         logf("failure to open changelog");
3439         return false;
3440     }
3441
3442     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3443     {
3444         close(clfd);
3445         return false;
3446     }
3447
3448     write_lock++;
3449
3450     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3451
3452     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3453                   parse_changelog_line);
3454
3455     close(clfd);
3456     close(masterfd);
3457
3458     if (filenametag_fd >= 0)
3459     {
3460         close(filenametag_fd);
3461         filenametag_fd = -1;
3462     }
3463
3464     write_lock--;
3465
3466     update_master_header();
3467
3468     return true;
3469 }
3470
3471 #endif /* !__PCTOOL__ */
3472
3473 bool tagcache_create_changelog(struct tagcache_search *tcs)
3474 {
3475     struct master_header myhdr;
3476     struct index_entry idx;
3477     char buf[TAG_MAXLEN+32];
3478     char temp[32];
3479     int clfd;
3480     int i, j;
3481
3482     if (!tc_stat.ready)
3483         return false;
3484
3485     if (!tagcache_search(tcs, tag_filename))
3486         return false;
3487
3488     /* Initialize the changelog */
3489     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3490     if (clfd < 0)
3491     {
3492         logf("failure to open changelog");
3493         return false;
3494     }
3495
3496     if (tcs->masterfd < 0)
3497     {
3498         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3499             return false;
3500     }
3501     else
3502     {
3503         lseek(tcs->masterfd, 0, SEEK_SET);
3504         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3505     }
3506
3507     write(clfd, "## Changelog version 1\n", 23);
3508
3509     for (i = 0; i < myhdr.tch.entry_count; i++)
3510     {
3511         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3512         {
3513             logf("read error #9");
3514             tagcache_search_finish(tcs);
3515             close(clfd);
3516             return false;
3517         }
3518
3519         /* Skip until the entry found has been modified. */
3520         if (! (idx.flag & FLAG_DIRTYNUM) )
3521             continue;
3522
3523         /* Skip deleted entries too. */
3524         if (idx.flag & FLAG_DELETED)
3525             continue;
3526
3527         /* Now retrieve all tags. */
3528         for (j = 0; j < TAG_COUNT; j++)
3529         {
3530             if (TAGCACHE_IS_NUMERIC(j))
3531             {
3532                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3533                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3534                 continue;
3535             }
3536
3537             tcs->type = j;
3538             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3539             write_tag(clfd, tagcache_tag_to_str(j), buf);
3540         }
3541
3542         write(clfd, "\n", 1);
3543         do_timed_yield();
3544     }
3545
3546     close(clfd);
3547
3548     tagcache_search_finish(tcs);
3549
3550     return true;
3551 }
3552
3553 static bool delete_entry(long idx_id)
3554 {
3555     int fd = -1;
3556     int masterfd = -1;
3557     int tag, i;
3558     struct index_entry idx, myidx;
3559     struct master_header myhdr;
3560     char buf[TAG_MAXLEN+32];
3561     int in_use[TAG_COUNT];
3562
3563     logf("delete_entry(): %ld", idx_id);
3564
3565 #ifdef HAVE_TC_RAMCACHE
3566     /* At first mark the entry removed from ram cache. */
3567     if (tc_stat.ramcache)
3568         hdr->indices[idx_id].flag |= FLAG_DELETED;
3569 #endif
3570
3571     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3572         return false;
3573
3574     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3575     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3576     {
3577         logf("delete_entry(): read error");
3578         goto cleanup;
3579     }
3580
3581     if (myidx.flag & FLAG_DELETED)
3582     {
3583         logf("delete_entry(): already deleted!");
3584         goto cleanup;
3585     }
3586
3587     myidx.flag |= FLAG_DELETED;
3588     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3589     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3590     {
3591         logf("delete_entry(): write_error #1");
3592         goto cleanup;
3593     }
3594
3595     /* Now check which tags are no longer in use (if any) */
3596     for (tag = 0; tag < TAG_COUNT; tag++)
3597         in_use[tag] = 0;
3598
3599     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3600     for (i = 0; i < myhdr.tch.entry_count; i++)
3601     {
3602         struct index_entry *idxp;
3603
3604 #ifdef HAVE_TC_RAMCACHE
3605         /* Use RAM DB if available for greater speed */
3606         if (tc_stat.ramcache)
3607             idxp = &hdr->indices[i];
3608         else
3609 #endif
3610         {
3611             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3612             {
3613                 logf("delete_entry(): read error #2");
3614                 goto cleanup;
3615             }
3616             idxp = &idx;
3617         }
3618
3619         if (idxp->flag & FLAG_DELETED)
3620             continue;
3621
3622         for (tag = 0; tag < TAG_COUNT; tag++)
3623         {
3624             if (TAGCACHE_IS_NUMERIC(tag))
3625                 continue;
3626
3627             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3628                 in_use[tag]++;
3629         }
3630     }
3631
3632     /* Now delete all tags no longer in use. */
3633     for (tag = 0; tag < TAG_COUNT; tag++)
3634     {
3635         struct tagcache_header tch;
3636         int oldseek = myidx.tag_seek[tag];
3637
3638         if (TAGCACHE_IS_NUMERIC(tag))
3639             continue;
3640
3641         /**
3642          * Replace tag seek with a hash value of the field string data.
3643          * That way runtime statistics of moved or altered files can be
3644          * resurrected.
3645          */
3646 #ifdef HAVE_TC_RAMCACHE
3647         if (tc_stat.ramcache && tag != tag_filename)
3648         {
3649             struct tagfile_entry *tfe;
3650             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3651
3652             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3653             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3654             myidx.tag_seek[tag] = *seek;
3655         }
3656         else
3657 #endif
3658         {
3659             struct tagfile_entry tfe;
3660
3661             /* Open the index file, which contains the tag names. */
3662             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3663                 goto cleanup;
3664
3665             /* Skip the header block */
3666             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3667             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3668             {
3669                 logf("delete_entry(): read error #3");
3670                 goto cleanup;
3671             }
3672
3673             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3674             {
3675                 logf("delete_entry(): read error #4");
3676                 goto cleanup;
3677             }
3678
3679             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3680         }
3681
3682         if (in_use[tag])
3683         {
3684             logf("in use: %d/%d", tag, in_use[tag]);
3685             if (fd >= 0)
3686             {
3687                 close(fd);
3688                 fd = -1;
3689             }
3690             continue;
3691         }
3692
3693 #ifdef HAVE_TC_RAMCACHE
3694         /* Delete from ram. */
3695         if (tc_stat.ramcache && tag != tag_filename)
3696         {
3697             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3698             tagentry->tag_data[0] = '\0';
3699         }
3700 #endif
3701
3702         /* Open the index file, which contains the tag names. */
3703         if (fd < 0)
3704         {
3705             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3706                 goto cleanup;
3707         }
3708
3709         /* Skip the header block */
3710         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3711
3712         /* Debug, print 10 first characters of the tag
3713         read(fd, buf, 10);
3714         buf[10]='\0';
3715         logf("TAG:%s", buf);
3716         lseek(fd, -10, SEEK_CUR);
3717         */
3718
3719         /* Write first data byte in tag as \0 */
3720         write(fd, "", 1);
3721
3722         /* Now tag data has been removed */
3723         close(fd);
3724         fd = -1;
3725     }
3726
3727     /* Write index entry back into master index. */
3728     lseek(masterfd, sizeof(struct master_header) +
3729           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3730     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3731     {
3732         logf("delete_entry(): write_error #2");
3733         goto cleanup;
3734     }
3735
3736     close(masterfd);
3737
3738     return true;
3739
3740     cleanup:
3741     if (fd >= 0)
3742         close(fd);
3743     if (masterfd >= 0)
3744         close(masterfd);
3745
3746     return false;
3747 }
3748
3749 #ifndef __PCTOOL__
3750 /**
3751  * Returns true if there is an event waiting in the queue
3752  * that requires the current operation to be aborted.
3753  */
3754 static bool check_event_queue(void)
3755 {
3756     struct queue_event ev;
3757
3758     if(!queue_peek(&tagcache_queue, &ev))
3759         return false;
3760
3761     switch (ev.id)
3762     {
3763         case Q_STOP_SCAN:
3764         case SYS_POWEROFF:
3765         case SYS_USB_CONNECTED:
3766             return true;
3767     }
3768
3769     return false;
3770 }
3771 #endif
3772
3773 #ifdef HAVE_TC_RAMCACHE
3774 static bool allocate_tagcache(void)
3775 {
3776     struct master_header tcmh;
3777     int fd;
3778
3779     /* Load the header. */
3780     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3781     {
3782         hdr = NULL;
3783         return false;
3784     }
3785
3786     close(fd);
3787
3788     /**
3789      * Now calculate the required cache size plus
3790      * some extra space for alignment fixes.
3791      */
3792     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3793         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3794     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3795     memset(hdr, 0, sizeof(struct ramcache_header));
3796     memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3797     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3798
3799     return true;
3800 }
3801
3802 # ifdef HAVE_EEPROM_SETTINGS
3803 static bool tagcache_dumpload(void)
3804 {
3805     struct statefile_header shdr;
3806     int fd, rc;
3807     long offpos;
3808     int i;
3809
3810     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3811     if (fd < 0)
3812     {
3813         logf("no tagcache statedump");
3814         return false;
3815     }
3816
3817     /* Check the statefile memory placement */
3818     hdr = buffer_alloc(0);
3819     rc = read(fd, &shdr, sizeof(struct statefile_header));
3820     if (rc != sizeof(struct statefile_header)
3821         || shdr.magic != TAGCACHE_STATEFILE_MAGIC
3822         || shdr.mh.tch.magic != TAGCACHE_MAGIC)
3823     {
3824         logf("incorrect statefile");
3825         hdr = NULL;
3826         close(fd);
3827         return false;
3828     }
3829
3830     offpos = (long)hdr - (long)shdr.hdr;
3831
3832     /* Lets allocate real memory and load it */
3833     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3834     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3835     close(fd);
3836
3837     if (rc != shdr.tc_stat.ramcache_allocated)
3838     {
3839         logf("read failure!");
3840         hdr = NULL;
3841         return false;
3842     }
3843
3844     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3845
3846     /* Now fix the pointers */
3847     for (i = 0; i < TAG_COUNT; i++)
3848         hdr->tags[i] += offpos;
3849
3850     /* Load the tagcache master header (should match the actual DB file header). */
3851     memcpy(&current_tcmh, &shdr.mh, sizeof current_tcmh);
3852
3853     return true;
3854 }
3855
3856 static bool tagcache_dumpsave(void)
3857 {
3858     struct statefile_header shdr;
3859     int fd;
3860
3861     if (!tc_stat.ramcache)
3862         return false;
3863
3864     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3865     if (fd < 0)
3866     {
3867         logf("failed to create a statedump");
3868         return false;
3869     }
3870
3871     /* Create the header */
3872     shdr.magic = TAGCACHE_STATEFILE_MAGIC;
3873     shdr.hdr = hdr;
3874     memcpy(&shdr.mh, &current_tcmh, sizeof current_tcmh);
3875     memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat);
3876     write(fd, &shdr, sizeof shdr);
3877
3878     /* And dump the data too */
3879     write(fd, hdr, tc_stat.ramcache_allocated);
3880     close(fd);
3881
3882     return true;
3883 }
3884 # endif
3885
3886 static bool load_tagcache(void)
3887 {
3888     struct tagcache_header *tch;
3889     struct master_header tcmh;
3890     long bytesleft = tc_stat.ramcache_allocated;
3891     struct index_entry *idx;
3892     int rc, fd;
3893     char *p;
3894     int i, tag;
3895
3896 # ifdef HAVE_DIRCACHE
3897     while (dircache_is_initializing())
3898         sleep(1);
3899
3900     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3901 # endif
3902
3903     logf("loading tagcache to ram...");
3904
3905     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3906     if (fd < 0)
3907     {
3908         logf("tagcache open failed");
3909         return false;
3910     }
3911
3912     if (ecread(fd, &tcmh, 1, master_header_ec, tc_stat.econ)
3913         != sizeof(struct master_header)
3914         || tcmh.tch.magic != TAGCACHE_MAGIC)
3915     {
3916         logf("incorrect header");
3917         return false;
3918     }
3919
3920     /* Master header copy should already match, this can be redundant to do. */
3921     memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3922
3923     idx = hdr->indices;
3924
3925     /* Load the master index table. */
3926     for (i = 0; i < tcmh.tch.entry_count; i++)
3927     {
3928         /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
3929          * may become corrupt. */
3930         rc = ecread_index_entry(fd, idx);
3931         if (rc != sizeof(struct index_entry))
3932         {
3933             logf("read error #10");
3934             close(fd);
3935             return false;
3936         }
3937
3938         bytesleft -= sizeof(struct index_entry);
3939         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3940         {
3941             logf("too big tagcache.");
3942             close(fd);
3943             return false;
3944         }
3945
3946         idx++;
3947     }
3948
3949     close(fd);
3950
3951     /* Load the tags. */
3952     p = (char *)idx;
3953     for (tag = 0; tag < TAG_COUNT; tag++)
3954     {
3955         struct tagfile_entry *fe;
3956         char buf[TAG_MAXLEN+32];
3957
3958         if (TAGCACHE_IS_NUMERIC(tag))
3959             continue ;
3960
3961         //p = ((void *)p+1);
3962         p = (char *)((long)p & ~0x03) + 0x04;
3963         hdr->tags[tag] = p;
3964
3965         /* Check the header. */
3966         tch = (struct tagcache_header *)p;
3967         p += sizeof(struct tagcache_header);
3968
3969         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3970             return false;
3971
3972         for (hdr->entry_count[tag] = 0;
3973              hdr->entry_count[tag] < tch->entry_count;
3974              hdr->entry_count[tag]++)
3975         {
3976             long pos;
3977
3978             if (do_timed_yield())
3979             {
3980                 /* Abort if we got a critical event in queue */
3981                 if (check_event_queue())
3982                     return false;
3983             }
3984
3985             fe = (struct tagfile_entry *)p;
3986             pos = lseek(fd, 0, SEEK_CUR);
3987             rc = ecread_tagfile_entry(fd, fe);
3988             if (rc != sizeof(struct tagfile_entry))
3989             {
3990                 /* End of lookup table. */
3991                 logf("read error #11");
3992                 close(fd);
3993                 return false;
3994             }
3995
3996             /* We have a special handling for the filename tags. */
3997             if (tag == tag_filename)
3998             {
3999 # ifdef HAVE_DIRCACHE
4000                 int dc;
4001 # endif
4002
4003                 idx = &hdr->indices[fe->idx_id];
4004
4005                 if (fe->tag_length >= (long)sizeof(buf)-1)
4006                 {
4007                     read(fd, buf, 10);
4008                     buf[10] = '\0';
4009                     logf("TAG:%s", buf);
4010                     logf("too long filename");
4011                     close(fd);
4012                     return false;
4013                 }
4014
4015                 rc = read(fd, buf, fe->tag_length);
4016                 if (rc != fe->tag_length)
4017                 {
4018                     logf("read error #12");
4019                     close(fd);
4020                     return false;
4021                 }
4022
4023                 /* Check if the entry has already been removed */
4024                 if (idx->flag & FLAG_DELETED)
4025                     continue;
4026
4027                 /* This flag must not be used yet. */
4028                 if (idx->flag & FLAG_DIRCACHE)
4029                 {
4030                     logf("internal error!");
4031                     close(fd);
4032                     return false;
4033                 }
4034
4035                 if (idx->tag_seek[tag] != pos)
4036                 {
4037                     logf("corrupt data structures!");
4038                     close(fd);
4039                     return false;
4040                 }
4041
4042 # ifdef HAVE_DIRCACHE
4043                 if (dircache_is_enabled())
4044                 {
4045                     dc = dircache_get_entry_id(buf);
4046                     if (dc < 0)
4047                     {
4048                         logf("Entry no longer valid.");
4049                         logf("-> %s", buf);
4050                         if (global_settings.tagcache_autoupdate)
4051                             delete_entry(fe->idx_id);
4052                         continue ;
4053                     }
4054
4055                     idx->flag |= FLAG_DIRCACHE;
4056                     idx->tag_seek[tag_filename] = dc;
4057                 }
4058                 else
4059 # endif
4060                 {
4061                     /* This will be very slow unless dircache is enabled
4062                        or target is flash based, but do it anyway for
4063                        consistency. */
4064                     /* Check if entry has been removed. */
4065                     if (global_settings.tagcache_autoupdate)
4066                     {
4067                         if (!file_exists(buf))
4068                         {
4069                             logf("Entry no longer valid.");
4070                             logf("-> %s", buf);
4071                             delete_entry(fe->idx_id);
4072                             continue;
4073                         }
4074                     }
4075                 }
4076
4077                 continue ;
4078             }
4079
4080             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4081             if (bytesleft < 0)
4082             {
4083                 logf("too big tagcache #2");
4084                 logf("tl: %ld", fe->tag_length);
4085                 logf("bl: %ld", bytesleft);
4086                 close(fd);
4087                 return false;
4088             }
4089
4090             p = fe->tag_data;
4091             rc = read(fd, fe->tag_data, fe->tag_length);
4092             p += rc;
4093
4094             if (rc != fe->tag_length)
4095             {
4096                 logf("read error #13");
4097                 logf("rc=0x%04x", rc); // 0x431
4098                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4099                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4100                 logf("tag=0x%02x", tag); // 0x00
4101                 close(fd);
4102                 return false;
4103             }
4104         }
4105         close(fd);
4106     }
4107
4108     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4109     logf("tagcache loaded into ram!");
4110
4111     return true;
4112 }
4113 #endif /* HAVE_TC_RAMCACHE */
4114
4115 static bool check_deleted_files(void)
4116 {
4117     int fd;
4118     char buf[TAG_MAXLEN+32];
4119     struct tagfile_entry tfe;
4120
4121     logf("reverse scan...");
4122     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4123     fd = open(buf, O_RDONLY);
4124
4125     if (fd < 0)
4126     {
4127         logf("%s open fail", buf);
4128         return false;
4129     }
4130
4131     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4132     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4133 #ifndef __PCTOOL__
4134            && !check_event_queue()
4135 #endif
4136            )
4137     {
4138         if (tfe.tag_length >= (long)sizeof(buf)-1)
4139         {
4140             logf("too long tag");
4141             close(fd);
4142             return false;
4143         }
4144
4145         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4146         {
4147             logf("read error #14");
4148             close(fd);
4149             return false;
4150         }
4151
4152         /* Check if the file has already deleted from the db. */
4153         if (*buf == '\0')
4154             continue;
4155
4156         /* Now check if the file exists. */
4157         if (!file_exists(buf))
4158         {
4159             logf("Entry no longer valid.");
4160             logf("-> %s / %ld", buf, tfe.tag_length);
4161             delete_entry(tfe.idx_id);
4162         }
4163     }
4164
4165     close(fd);
4166
4167     logf("done");
4168
4169     return true;
4170 }
4171
4172
4173 /* Note that this function must not be inlined, otherwise the whole point
4174  * of having the code in a separate function is lost.
4175  */
4176 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4177     int *ignore, int *unignore)
4178 {
4179     char newpath[MAX_PATH];
4180
4181     /* check for a database.ignore file */
4182     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4183     *ignore = file_exists(newpath);
4184     /* check for a database.unignore file */
4185     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4186     *unignore = file_exists(newpath);
4187 }
4188
4189 static struct search_roots_ll {
4190     const char *path;
4191     struct search_roots_ll * next;
4192 } roots_ll;
4193
4194 #ifdef APPLICATION
4195 /*
4196  * This adds a path to the search roots, possibly during traveling through
4197  * the filesystem. It only adds if the path is not inside an already existing
4198  * search root.
4199  *
4200  * Returns true if it added the path to the search roots
4201  *
4202  * Windows 2000 and greater supports symlinks, but they don't provide
4203  * realpath() or readlink(), and symlinks are rarely used on them so
4204  * ignore this for windows for now
4205  **/
4206 static bool add_search_root(const char *name)
4207 {
4208     (void)name;
4209 #ifndef WIN32
4210     struct search_roots_ll *this, *prev = NULL;
4211     char target[MAX_PATH];
4212     /* Okay, realpath() is almost completely broken on android
4213      *
4214      * It doesn't accept NULL for resolved_name to dynamically allocate
4215      * the resulting path; and it assumes resolved_name to be PATH_MAX
4216      * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4217      * and blindly writes to the end if it
4218      *
4219      * therefore use sufficiently large static storage here
4220      * Note that PATH_MAX != MAX_PATH
4221      **/
4222     static char abs_target[PATH_MAX];
4223     ssize_t len;
4224
4225     len = readlink(name, target, sizeof(target));
4226     if (len < 0)
4227         return false;
4228
4229     target[len] = '\0';
4230     if (realpath(target, abs_target) == NULL)
4231         return false;
4232
4233     for(this = &roots_ll; this; prev = this, this = this->next)
4234     {
4235         size_t root_len = strlen(this->path);
4236         /* check if the link target is inside of an existing search root
4237          * don't add if target is inside, we'll scan it later */
4238         if (!strncmp(this->path, abs_target, root_len))
4239             return false;
4240     }
4241
4242     if (prev)
4243     {
4244         size_t len = strlen(abs_target) + 1; /* count \0 */
4245         this = malloc(sizeof(struct search_roots_ll) + len );
4246         if (!this || len > MAX_PATH)
4247         {
4248             logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4249             free(this);
4250             prev->next = NULL;
4251             return false;
4252         }
4253         this->path = ((char*)this) + sizeof(struct search_roots_ll);
4254         strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4255         this->next = NULL;
4256         prev->next = this;
4257         logf("Added %s to the search roots\n", abs_target);
4258         return true;
4259     }
4260 #endif
4261     return false;
4262 }
4263
4264 static int free_search_roots(struct search_roots_ll * start)
4265 {
4266     int ret = 0;
4267     if (start->next)
4268     {
4269         ret += free_search_roots(start->next);
4270         ret += sizeof(struct search_roots_ll);
4271         free(start->next);
4272     }
4273     return ret;
4274 }
4275 #else /* native, simulator */
4276 #define add_search_root(a) do {} while(0)
4277 #define free_search_roots(a) do {} while(0)
4278 #endif
4279
4280 static bool check_dir(const char *dirname, int add_files)
4281 {
4282     DIR *dir;
4283     int len;
4284     int success = false;
4285     int ignore, unignore;
4286
4287     dir = opendir(dirname);
4288     if (!dir)
4289     {
4290         logf("tagcache: opendir(%s) failed", dirname);
4291         return false;
4292     }
4293     /* check for a database.ignore and database.unignore */
4294     check_ignore(dirname, &ignore, &unignore);
4295
4296     /* don't do anything if both ignore and unignore are there */
4297     if (ignore != unignore)
4298         add_files = unignore;
4299
4300     /* Recursively scan the dir. */
4301 #ifdef __PCTOOL__
4302     while (1)
4303 #else
4304     while (!check_event_queue())
4305 #endif
4306     {
4307         struct dirent *entry = readdir(dir);
4308         if (entry == NULL)
4309         {
4310             success = true;
4311             break;
4312         }
4313
4314         if (!strcmp((char *)entry->d_name, ".") ||
4315             !strcmp((char *)entry->d_name, ".."))
4316             continue;
4317
4318         struct dirinfo info = dir_get_info(dir, entry);
4319
4320         yield();
4321
4322         len = strlen(curpath);
4323         /* don't add an extra / for curpath == / */
4324         if (len <= 1) len = 0;
4325         snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4326
4327         processed_dir_count++;
4328         if (info.attribute & ATTR_DIRECTORY)
4329 #ifndef SIMULATOR
4330         {   /* don't follow symlinks to dirs, but try to add it as a search root
4331              * this makes able to avoid looping in recursive symlinks */
4332             if (info.attribute & ATTR_LINK)
4333                 add_search_root(curpath);
4334             else
4335                 check_dir(curpath, add_files);
4336         }
4337 #else
4338             check_dir(curpath, add_files);
4339 #endif
4340         else if (add_files)
4341         {
4342             tc_stat.curentry = curpath;
4343
4344             /* Add a new entry to the temporary db file. */
4345             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4346 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4347                          , dir->internal_entry
4348 #endif
4349                          );
4350
4351             /* Wait until current path for debug screen is read and unset. */
4352             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4353                 yield();
4354
4355             tc_stat.curentry = NULL;
4356         }
4357
4358         curpath[len] = '\0';
4359     }
4360
4361     closedir(dir);
4362
4363     return success;
4364 }
4365
4366 void tagcache_screensync_event(void)
4367 {
4368     tc_stat.curentry = NULL;
4369 }
4370
4371 void tagcache_screensync_enable(bool state)
4372 {
4373     tc_stat.syncscreen = state;
4374 }
4375
4376 void tagcache_build(const char *path)
4377 {
4378     struct tagcache_header header;
4379     bool ret;
4380
4381     curpath[0] = '\0';
4382     data_size = 0;
4383     total_entry_count = 0;
4384     processed_dir_count = 0;
4385
4386 #ifdef HAVE_DIRCACHE
4387     while (dircache_is_initializing())
4388         sleep(1);
4389 #endif
4390
4391     logf("updating tagcache");
4392
4393     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4394     if (cachefd >= 0)
4395     {
4396         logf("skipping, cache already waiting for commit");
4397         close(cachefd);
4398         return ;
4399     }
4400
4401     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4402     if (cachefd < 0)
4403     {
4404         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4405         return ;
4406     }
4407
4408     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4409
4410     cpu_boost(true);
4411
4412     logf("Scanning files...");
4413     /* Scan for new files. */
4414     memset(&header, 0, sizeof(struct tagcache_header));
4415     write(cachefd, &header, sizeof(struct tagcache_header));
4416
4417     ret = true;
4418     roots_ll.path = path;
4419     roots_ll.next = NULL;
4420     struct search_roots_ll * this;
4421     /* check_dir might add new roots */
4422     for(this = &roots_ll; this; this = this->next)
4423     {
4424         strcpy(curpath, this->path);
4425         ret = ret && check_dir(this->path, true);
4426     }
4427     if (roots_ll.next)
4428         free_search_roots(roots_ll.next);
4429
4430     /* Write the header. */
4431     header.magic = TAGCACHE_MAGIC;
4432     header.datasize = data_size;
4433     header.entry_count = total_entry_count;
4434     lseek(cachefd, 0, SEEK_SET);
4435     write(cachefd, &header, sizeof(struct tagcache_header));
4436     close(cachefd);
4437
4438     if (filenametag_fd >= 0)
4439     {
4440         close(filenametag_fd);
4441         filenametag_fd = -1;
4442     }
4443
4444     if (!ret)
4445     {
4446         logf("Aborted.");
4447         cpu_boost(false);
4448         return ;
4449     }
4450
4451     /* Commit changes to the database. */
4452 #ifdef __PCTOOL__
4453     allocate_tempbuf();
4454 #endif
4455     if (commit())
4456     {
4457         logf("tagcache built!");
4458     }
4459 #ifdef __PCTOOL__
4460     free_tempbuf();
4461 #endif
4462
4463 #ifdef HAVE_TC_RAMCACHE
4464     if (hdr)
4465     {
4466         /* Import runtime statistics if we just initialized the db. */
4467         if (current_tcmh.serial == 0)
4468             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4469     }
4470 #endif
4471
4472     cpu_boost(false);
4473 }
4474
4475 #ifdef HAVE_TC_RAMCACHE
4476 static void load_ramcache(void)
4477 {
4478     if (!hdr)
4479         return ;
4480
4481     cpu_boost(true);
4482
4483     /* At first we should load the cache (if exists). */
4484     tc_stat.ramcache = load_tagcache();
4485
4486     if (!tc_stat.ramcache)
4487     {
4488         /* If loading failed, it must indicate some problem with the db
4489          * so disable it entirely to prevent further issues. */
4490         tc_stat.ready = false;
4491         hdr = NULL;
4492     }
4493
4494     cpu_boost(false);
4495 }
4496
4497 void tagcache_unload_ramcache(void)
4498 {
4499     tc_stat.ramcache = false;
4500     /* Just to make sure there is no statefile present. */
4501     // remove(TAGCACHE_STATEFILE);
4502 }
4503 #endif
4504
4505 #ifndef __PCTOOL__
4506 static void tagcache_thread(void)
4507 {
4508     struct queue_event ev;
4509     bool check_done = false;
4510
4511     /* If the previous cache build/update was interrupted, commit
4512      * the changes first in foreground. */
4513     cpu_boost(true);
4514     allocate_tempbuf();
4515     commit();
4516     free_tempbuf();
4517
4518 #ifdef HAVE_TC_RAMCACHE
4519 # ifdef HAVE_EEPROM_SETTINGS
4520     if (firmware_settings.initialized && firmware_settings.disk_clean
4521         && global_settings.tagcache_ram)
4522     {
4523         check_done = tagcache_dumpload();
4524     }
4525
4526     remove(TAGCACHE_STATEFILE);
4527 # endif
4528
4529     /* Allocate space for the tagcache if found on disk. */
4530     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4531         allocate_tagcache();
4532 #endif
4533
4534     cpu_boost(false);
4535     tc_stat.initialized = true;
4536
4537     /* Don't delay bootup with the header check but do it on background. */
4538     if (!tc_stat.ready)
4539     {
4540         sleep(HZ);
4541         tc_stat.ready = check_all_headers();
4542         tc_stat.readyvalid = true;
4543     }
4544
4545     while (1)
4546     {
4547         run_command_queue(false);
4548
4549         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4550
4551         switch (ev.id)
4552         {
4553             case Q_IMPORT_CHANGELOG:
4554                 tagcache_import_changelog();
4555                 break;
4556
4557             case Q_REBUILD:
4558                 remove_files();
4559                 remove(TAGCACHE_FILE_TEMP);
4560                 tagcache_build("/");
4561                 break;
4562
4563             case Q_UPDATE:
4564                 tagcache_build("/");
4565 #ifdef HAVE_TC_RAMCACHE
4566                 load_ramcache();
4567 #endif
4568                 check_deleted_files();
4569                 break ;
4570
4571             case Q_START_SCAN:
4572                 check_done = false;
4573             case SYS_TIMEOUT:
4574                 if (check_done || !tc_stat.ready)
4575                     break ;
4576
4577 #ifdef HAVE_TC_RAMCACHE
4578                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4579                 {
4580                     load_ramcache();
4581                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4582                         tagcache_build("/");
4583                 }
4584                 else
4585 #endif
4586                 if (global_settings.tagcache_autoupdate)
4587                 {
4588                     tagcache_build("/");
4589
4590                     /* This will be very slow unless dircache is enabled
4591                        or target is flash based, but do it anyway for
4592                        consistency. */
4593                     check_deleted_files();
4594                 }
4595
4596                 logf("tagcache check done");
4597
4598                 check_done = true;
4599                 break ;
4600
4601             case Q_STOP_SCAN:
4602                 break ;
4603
4604             case SYS_POWEROFF:
4605                 break ;
4606
4607 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4608             case SYS_USB_CONNECTED:
4609                 logf("USB: TagCache");
4610                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4611                 usb_wait_for_disconnect(&tagcache_queue);
4612                 break ;
4613 #endif
4614         }
4615     }
4616 }
4617
4618 bool tagcache_prepare_shutdown(void)
4619 {
4620     if (tagcache_get_commit_step() > 0)
4621         return false;
4622
4623     tagcache_stop_scan();
4624     while (read_lock || write_lock)
4625         sleep(1);
4626
4627     return true;
4628 }
4629
4630 void tagcache_shutdown(void)
4631 {
4632     /* Flush the command queue. */
4633     run_command_queue(true);
4634
4635 #ifdef HAVE_EEPROM_SETTINGS
4636     if (tc_stat.ramcache)
4637         tagcache_dumpsave();
4638 #endif
4639 }
4640
4641 static int get_progress(void)
4642 {
4643     int total_count = -1;
4644
4645 #ifdef HAVE_DIRCACHE
4646     if (dircache_is_enabled())
4647     {
4648         total_count = dircache_get_entry_count();
4649     }
4650     else
4651 #endif
4652 #ifdef HAVE_TC_RAMCACHE
4653     {
4654         if (hdr && tc_stat.ramcache)
4655             total_count = current_tcmh.tch.entry_count;
4656     }
4657 #endif
4658
4659     if (total_count < 0)
4660         return -1;
4661
4662     return processed_dir_count * 100 / total_count;
4663 }
4664
4665 struct tagcache_stat* tagcache_get_stat(void)
4666 {
4667     tc_stat.progress = get_progress();
4668     tc_stat.processed_entries = processed_dir_count;
4669
4670     return &tc_stat;
4671 }
4672
4673 void tagcache_start_scan(void)
4674 {
4675     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4676 }
4677
4678 bool tagcache_update(void)
4679 {
4680     if (!tc_stat.ready)
4681         return false;
4682
4683     queue_post(&tagcache_queue, Q_UPDATE, 0);
4684     return false;
4685 }
4686
4687 bool tagcache_rebuild()
4688 {
4689     queue_post(&tagcache_queue, Q_REBUILD, 0);
4690     return false;
4691 }
4692
4693 void tagcache_stop_scan(void)
4694 {
4695     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4696 }
4697
4698 #ifdef HAVE_TC_RAMCACHE
4699 bool tagcache_is_ramcache(void)
4700 {
4701     return tc_stat.ramcache;
4702 }
4703 #endif
4704
4705 #endif /* !__PCTOOL__ */
4706
4707
4708 void tagcache_init(void)
4709 {
4710     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4711     memset(&current_tcmh, 0, sizeof(struct master_header));
4712     filenametag_fd = -1;
4713     write_lock = read_lock = 0;
4714
4715 #ifndef __PCTOOL__
4716     mutex_init(&command_queue_mutex);
4717     queue_init(&tagcache_queue, true);
4718     create_thread(tagcache_thread, tagcache_stack,
4719                   sizeof(tagcache_stack), 0, tagcache_thread_name
4720                   IF_PRIO(, PRIORITY_BACKGROUND)
4721                   IF_COP(, CPU));
4722 #else
4723     tc_stat.initialized = true;
4724     allocate_tempbuf();
4725     commit();
4726     free_tempbuf();
4727     tc_stat.ready = check_all_headers();
4728 #endif
4729 }
4730
4731 #ifdef __PCTOOL__
4732 void tagcache_reverse_scan(void)
4733 {
4734     logf("Checking for deleted files");
4735     check_deleted_files();
4736 }
4737 #endif
4738
4739 bool tagcache_is_initialized(void)
4740 {
4741     return tc_stat.initialized;
4742 }
4743 bool tagcache_is_fully_initialized(void)
4744 {
4745     return tc_stat.readyvalid;
4746 }
4747 bool tagcache_is_usable(void)
4748 {
4749     return tc_stat.initialized && tc_stat.ready;
4750 }
4751 int tagcache_get_commit_step(void)
4752 {
4753     return tc_stat.commit_step;
4754 }
4755 int tagcache_get_max_commit_step(void)
4756 {
4757     return (int)(SORTED_TAGS_COUNT)+1;
4758 }
4759