apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #ifdef APPLICATION
  64 #include <unistd.h> /* readlink() */
  65 #endif
  66 #include "config.h"
  67 #include "ata_idle_notify.h"
  68 #include "thread.h"
  69 #include "kernel.h"
  70 #include "system.h"
  71 #include "logf.h"
  72 #include "string-extra.h"
  73 #include "usb.h"
  74 #include "metadata.h"
  75 #include "tagcache.h"
  76 #include "buffer.h"
  77 #include "crc32.h"
  78 #include "misc.h"
  79 #include "settings.h"
  80 #include "dir.h"
  81 #include "filefuncs.h"
  82 #include "structec.h"
  83 #include "debug.h"
  84
  85 #ifndef __PCTOOL__
  86 #include "lang.h"
  87 #include "eeprom_settings.h"
  88 #endif
  89
  90 #ifdef __PCTOOL__
  91 #define yield() do { } while(0)
  92 #define sim_sleep(timeout) do { } while(0)
  93 #define do_timed_yield() do { } while(0)
  94 #endif
  95
  96 #ifndef __PCTOOL__
  97 /* Tag Cache thread. */
  98 static struct event_queue tagcache_queue;
  99 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
 100 static const char tagcache_thread_name[] = "tagcache";
 101 #endif
 102
 103 /* Previous path when scanning directory tree recursively. */
 104 static char curpath[TAG_MAXLEN+32];
 105
 106 /* Used when removing duplicates. */
 107 static char *tempbuf;     /* Allocated when needed. */
 108 static long tempbufidx;   /* Current location in buffer. */
 109 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 110 static long tempbuf_left; /* Buffer space left. */
 111 static long tempbuf_pos;
 112
 113 #define SORTED_TAGS_COUNT 8
 114 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 115 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 116 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 117     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 118 /* Tags we want to get sorted (loaded to the tempbuf). */
 119 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 120     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 121     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 122
 123 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 124 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 125     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 126     (1LU << tag_albumartist) | (1LU << tag_grouping))
 127
 128 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 129 static const char *tags_str[] = { "artist", "album", "genre", "title",
 130     "filename", "composer", "comment", "albumartist", "grouping", "year",
 131     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 132     "playtime", "lastplayed", "commitid", "mtime" };
 133
 134 /* Status information of the tagcache. */
 135 static struct tagcache_stat tc_stat;
 136
 137 /* Queue commands. */
 138 enum tagcache_queue {
 139     Q_STOP_SCAN = 0,
 140     Q_START_SCAN,
 141     Q_IMPORT_CHANGELOG,
 142     Q_UPDATE,
 143     Q_REBUILD,
 144
 145     /* Internal tagcache command queue. */
 146     CMD_UPDATE_MASTER_HEADER,
 147     CMD_UPDATE_NUMERIC,
 148 };
 149
 150 struct tagcache_command_entry {
 151     int32_t command;
 152     int32_t idx_id;
 153     int32_t tag;
 154     int32_t data;
 155 };
 156
 157 #ifndef __PCTOOL__
 158 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 159 static volatile int command_queue_widx = 0;
 160 static volatile int command_queue_ridx = 0;
 161 static struct mutex command_queue_mutex;
 162 #endif
 163
 164 /* Tag database structures. */
 165
 166 /* Variable-length tag entry in tag files. */
 167 struct tagfile_entry {
 168     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 169     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 170     char tag_data[0];  /* Begin of the tag data */
 171 };
 172
 173 /* Fixed-size tag entry in master db index. */
 174 struct index_entry {
 175     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 176     int32_t flag;                /* Status flags */
 177 };
 178
 179 /* Header is the same in every file. */
 180 struct tagcache_header {
 181     int32_t magic;       /* Header version number */
 182     int32_t datasize;    /* Data size in bytes */
 183     int32_t entry_count; /* Number of entries in this file */
 184 };
 185
 186 struct master_header {
 187     struct tagcache_header tch;
 188     int32_t serial; /* Increasing counting number */
 189     int32_t commitid; /* Number of commits so far */
 190     int32_t dirty;
 191 };
 192
 193 /* For the endianess correction */
 194 static const char *tagfile_entry_ec   = "ll";
 195 /**
 196  Note: This should be (1 + TAG_COUNT) amount of l's.
 197  */
 198 static const char *index_entry_ec     = "llllllllllllllllllllll";
 199
 200 static const char *tagcache_header_ec = "lll";
 201 static const char *master_header_ec   = "llllll";
 202
 203 static struct master_header current_tcmh;
 204
 205 #ifdef HAVE_TC_RAMCACHE
 206 /* Header is created when loading database to ram. */
 207 struct ramcache_header {
 208     struct master_header h;      /* Header from the master index */
 209     struct index_entry *indices; /* Master index file content */
 210     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 211     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 212 };
 213
 214 # ifdef HAVE_EEPROM_SETTINGS
 215 struct statefile_header {
 216     struct ramcache_header *hdr;
 217     struct tagcache_stat tc_stat;
 218 };
 219 # endif
 220
 221 /* Pointer to allocated ramcache_header */
 222 static struct ramcache_header *hdr;
 223 #endif
 224
 225 /**
 226  * Full tag entries stored in a temporary file waiting
 227  * for commit to the cache. */
 228 struct temp_file_entry {
 229     long tag_offset[TAG_COUNT];
 230     short tag_length[TAG_COUNT];
 231     long flag;
 232
 233     long data_length;
 234 };
 235
 236 struct tempbuf_id_list {
 237     long id;
 238     struct tempbuf_id_list *next;
 239 };
 240
 241 struct tempbuf_searchidx {
 242     long idx_id;
 243     char *str;
 244     int seek;
 245     struct tempbuf_id_list idlist;
 246 };
 247
 248 /* Lookup buffer for fixing messed up index while after sorting. */
 249 static long commit_entry_count;
 250 static long lookup_buffer_depth;
 251 static struct tempbuf_searchidx **lookup;
 252
 253 /* Used when building the temporary file. */
 254 static int cachefd = -1, filenametag_fd;
 255 static int total_entry_count = 0;
 256 static int data_size = 0;
 257 static int processed_dir_count;
 258
 259 /* Thread safe locking */
 260 static volatile int write_lock;
 261 static volatile int read_lock;
 262
 263 static bool delete_entry(long idx_id);
 264
 265 const char* tagcache_tag_to_str(int tag)
 266 {
 267     return tags_str[tag];
 268 }
 269
 270 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 271 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 272 {
 273     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 274 }
 275
 276 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 277 {
 278     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 279 }
 280
 281 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 282 {
 283     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 284 }
 285
 286 #ifdef HAVE_DIRCACHE
 287 /**
 288  * Returns true if specified flag is still present, i.e., dircache
 289  * has not been reloaded.
 290  */
 291 static bool is_dircache_intact(void)
 292 {
 293     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 294 }
 295 #endif
 296
 297 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 298 {
 299     int fd;
 300     char buf[MAX_PATH];
 301     int rc;
 302
 303     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 304         return -1;
 305
 306     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 307
 308     fd = open(buf, write ? O_RDWR : O_RDONLY);
 309     if (fd < 0)
 310     {
 311         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 312         tc_stat.ready = false;
 313         return fd;
 314     }
 315
 316     /* Check the header. */
 317     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 318     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 319     {
 320         logf("header error");
 321         tc_stat.ready = false;
 322         close(fd);
 323         return -2;
 324     }
 325
 326     return fd;
 327 }
 328
 329 static int open_master_fd(struct master_header *hdr, bool write)
 330 {
 331     int fd;
 332     int rc;
 333
 334     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 335     if (fd < 0)
 336     {
 337         logf("master file open failed for R/W");
 338         tc_stat.ready = false;
 339         return fd;
 340     }
 341
 342     tc_stat.econ = false;
 343
 344     /* Check the header. */
 345     rc = read(fd, hdr, sizeof(struct master_header));
 346     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 347     {
 348         /* Success. */
 349         return fd;
 350     }
 351
 352     /* Trying to read again, this time with endianess correction enabled. */
 353     lseek(fd, 0, SEEK_SET);
 354
 355     rc = ecread(fd, hdr, 1, master_header_ec, true);
 356     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 357     {
 358         logf("header error");
 359         tc_stat.ready = false;
 360         close(fd);
 361         return -2;
 362     }
 363
 364     tc_stat.econ = true;
 365
 366     return fd;
 367 }
 368
 369 #ifndef __PCTOOL__
 370 static bool do_timed_yield(void)
 371 {
 372     /* Sorting can lock up for quite a while, so yield occasionally */
 373     static long wakeup_tick = 0;
 374     if (TIME_AFTER(current_tick, wakeup_tick))
 375     {
 376         wakeup_tick = current_tick + (HZ/4);
 377         yield();
 378         return true;
 379     }
 380     return false;
 381 }
 382 #endif
 383
 384 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 385 static long find_entry_ram(const char *filename,
 386                            const struct dircache_entry *dc)
 387 {
 388     static long last_pos = 0;
 389     int i;
 390
 391     /* Check if tagcache is loaded into ram. */
 392     if (!tc_stat.ramcache)
 393         return -1;
 394
 395     if (dc == NULL)
 396         dc = dircache_get_entry_ptr(filename);
 397
 398     if (dc == NULL)
 399     {
 400         logf("tagcache: file not found.");
 401         return -1;
 402     }
 403
 404     try_again:
 405
 406     if (last_pos > 0)
 407         i = last_pos;
 408     else
 409         i = 0;
 410
 411     for (; i < hdr->h.tch.entry_count; i++)
 412     {
 413         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 414         {
 415             last_pos = MAX(0, i - 3);
 416             return i;
 417         }
 418
 419         do_timed_yield();
 420     }
 421
 422     if (last_pos > 0)
 423     {
 424         last_pos = 0;
 425         goto try_again;
 426     }
 427
 428     return -1;
 429 }
 430 #endif
 431
 432 static long find_entry_disk(const char *filename, bool localfd)
 433 {
 434     struct tagcache_header tch;
 435     static long last_pos = -1;
 436     long pos_history[POS_HISTORY_COUNT];
 437     long pos_history_idx = 0;
 438     bool found = false;
 439     struct tagfile_entry tfe;
 440     int fd;
 441     char buf[TAG_MAXLEN+32];
 442     int i;
 443     int pos = -1;
 444
 445     if (!tc_stat.ready)
 446         return -2;
 447
 448     fd = filenametag_fd;
 449     if (fd < 0 || localfd)
 450     {
 451         last_pos = -1;
 452         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 453             return -1;
 454     }
 455
 456     check_again:
 457
 458     if (last_pos > 0)
 459         lseek(fd, last_pos, SEEK_SET);
 460     else
 461         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 462
 463     while (true)
 464     {
 465         pos = lseek(fd, 0, SEEK_CUR);
 466         for (i = pos_history_idx-1; i >= 0; i--)
 467             pos_history[i+1] = pos_history[i];
 468         pos_history[0] = pos;
 469
 470         if (ecread_tagfile_entry(fd, &tfe)
 471             != sizeof(struct tagfile_entry))
 472         {
 473             break ;
 474         }
 475
 476         if (tfe.tag_length >= (long)sizeof(buf))
 477         {
 478             logf("too long tag #1");
 479             close(fd);
 480             if (!localfd)
 481                 filenametag_fd = -1;
 482             last_pos = -1;
 483             return -2;
 484         }
 485
 486         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 487         {
 488             logf("read error #2");
 489             close(fd);
 490             if (!localfd)
 491                 filenametag_fd = -1;
 492             last_pos = -1;
 493             return -3;
 494         }
 495
 496         if (!strcasecmp(filename, buf))
 497         {
 498             last_pos = pos_history[pos_history_idx];
 499             found = true;
 500             break ;
 501         }
 502
 503         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 504             pos_history_idx++;
 505     }
 506
 507     /* Not found? */
 508     if (!found)
 509     {
 510         if (last_pos > 0)
 511         {
 512             last_pos = -1;
 513             logf("seek again");
 514             goto check_again;
 515         }
 516
 517         if (fd != filenametag_fd || localfd)
 518             close(fd);
 519         return -4;
 520     }
 521
 522     if (fd != filenametag_fd || localfd)
 523         close(fd);
 524
 525     return tfe.idx_id;
 526 }
 527
 528 static int find_index(const char *filename)
 529 {
 530     long idx_id = -1;
 531
 532 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 533     if (tc_stat.ramcache && is_dircache_intact())
 534         idx_id = find_entry_ram(filename, NULL);
 535 #endif
 536
 537     if (idx_id < 0)
 538         idx_id = find_entry_disk(filename, true);
 539
 540     return idx_id;
 541 }
 542
 543 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 544 {
 545     int idx_id;
 546
 547     if (!tc_stat.ready)
 548         return false;
 549
 550     idx_id = find_index(filename);
 551     if (idx_id < 0)
 552         return false;
 553
 554     if (!tagcache_search(tcs, tag_filename))
 555         return false;
 556
 557     tcs->entry_count = 0;
 558     tcs->idx_id = idx_id;
 559
 560     return true;
 561 }
 562
 563 static bool get_index(int masterfd, int idxid,
 564                       struct index_entry *idx, bool use_ram)
 565 {
 566     bool localfd = false;
 567
 568     if (idxid < 0)
 569     {
 570         logf("Incorrect idxid: %d", idxid);
 571         return false;
 572     }
 573
 574 #ifdef HAVE_TC_RAMCACHE
 575     if (tc_stat.ramcache && use_ram)
 576     {
 577         if (hdr->indices[idxid].flag & FLAG_DELETED)
 578             return false;
 579
 580 # ifdef HAVE_DIRCACHE
 581         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 582             || is_dircache_intact())
 583 #endif
 584         {
 585             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 586             return true;
 587         }
 588     }
 589 #else
 590     (void)use_ram;
 591 #endif
 592
 593     if (masterfd < 0)
 594     {
 595         struct master_header tcmh;
 596
 597         localfd = true;
 598         masterfd = open_master_fd(&tcmh, false);
 599         if (masterfd < 0)
 600             return false;
 601     }
 602
 603     lseek(masterfd, idxid * sizeof(struct index_entry)
 604           + sizeof(struct master_header), SEEK_SET);
 605     if (ecread_index_entry(masterfd, idx)
 606         != sizeof(struct index_entry))
 607     {
 608         logf("read error #3");
 609         if (localfd)
 610             close(masterfd);
 611
 612         return false;
 613     }
 614
 615     if (localfd)
 616         close(masterfd);
 617
 618     if (idx->flag & FLAG_DELETED)
 619         return false;
 620
 621     return true;
 622 }
 623
 624 #ifndef __PCTOOL__
 625
 626 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 627 {
 628     /* We need to exclude all memory only flags & tags when writing to disk. */
 629     if (idx->flag & FLAG_DIRCACHE)
 630     {
 631         logf("memory only flags!");
 632         return false;
 633     }
 634
 635 #ifdef HAVE_TC_RAMCACHE
 636     /* Only update numeric data. Writing the whole index to RAM by memcpy
 637      * destroys dircache pointers!
 638      */
 639     if (tc_stat.ramcache)
 640     {
 641         int tag;
 642         struct index_entry *idx_ram = &hdr->indices[idxid];
 643
 644         for (tag = 0; tag < TAG_COUNT; tag++)
 645         {
 646             if (TAGCACHE_IS_NUMERIC(tag))
 647             {
 648                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 649             }
 650         }
 651
 652         /* Don't touch the dircache flag or attributes. */
 653         idx_ram->flag = (idx->flag & 0x0000ffff)
 654             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 655     }
 656 #endif
 657
 658     lseek(masterfd, idxid * sizeof(struct index_entry)
 659           + sizeof(struct master_header), SEEK_SET);
 660     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 661     {
 662         logf("write error #3");
 663         logf("idxid: %d", idxid);
 664         return false;
 665     }
 666
 667     return true;
 668 }
 669
 670 #endif /* !__PCTOOL__ */
 671
 672 static bool open_files(struct tagcache_search *tcs, int tag)
 673 {
 674     if (tcs->idxfd[tag] < 0)
 675     {
 676         char fn[MAX_PATH];
 677
 678         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 679         tcs->idxfd[tag] = open(fn, O_RDONLY);
 680     }
 681
 682     if (tcs->idxfd[tag] < 0)
 683     {
 684         logf("File not open!");
 685         return false;
 686     }
 687
 688     return true;
 689 }
 690
 691 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 692                      int tag, char *buf, long size)
 693 {
 694     struct tagfile_entry tfe;
 695     long seek;
 696
 697     *buf = '\0';
 698
 699     if (TAGCACHE_IS_NUMERIC(tag))
 700         return false;
 701
 702     seek = idx->tag_seek[tag];
 703     if (seek < 0)
 704     {
 705         logf("Retrieve failed");
 706         return false;
 707     }
 708
 709 #ifdef HAVE_TC_RAMCACHE
 710     if (tcs->ramsearch)
 711     {
 712         struct tagfile_entry *ep;
 713
 714 # ifdef HAVE_DIRCACHE
 715         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 716             && is_dircache_intact())
 717         {
 718             dircache_copy_path((struct dircache_entry *)seek,
 719                                buf, size);
 720             return true;
 721         }
 722         else
 723 # endif
 724         if (tag != tag_filename)
 725         {
 726             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 727             strlcpy(buf, ep->tag_data, size);
 728
 729             return true;
 730         }
 731     }
 732 #endif
 733
 734     if (!open_files(tcs, tag))
 735         return false;
 736
 737     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 738     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 739         != sizeof(struct tagfile_entry))
 740     {
 741         logf("read error #5");
 742         return false;
 743     }
 744
 745     if (tfe.tag_length >= size)
 746     {
 747         logf("too small buffer");
 748         return false;
 749     }
 750
 751     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 752         tfe.tag_length)
 753     {
 754         logf("read error #6");
 755         return false;
 756     }
 757
 758     buf[tfe.tag_length] = '\0';
 759
 760     return true;
 761 }
 762
 763 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 764
 765 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 766 {
 767 #ifndef __PCTOOL__
 768     if (! COMMAND_QUEUE_IS_EMPTY)
 769     {
 770         /* Attempt to find tag data through store-to-load forwarding in
 771            command queue */
 772         long result = -1;
 773
 774         mutex_lock(&command_queue_mutex);
 775
 776         int ridx = command_queue_widx;
 777
 778         while (ridx != command_queue_ridx)
 779         {
 780             if (--ridx < 0)
 781                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 782
 783             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 784                 && command_queue[ridx].idx_id == idx_id
 785                 && command_queue[ridx].tag == tag)
 786             {
 787                 result = command_queue[ridx].data;
 788                 break;
 789             }
 790         }
 791
 792         mutex_unlock(&command_queue_mutex);
 793
 794         if (result >= 0)
 795         {
 796             logf("read_numeric_tag: "
 797                  "Recovered tag %d value %lX from write queue",
 798                  tag, result);
 799             return result;
 800         }
 801     }
 802 #endif
 803
 804     return idx->tag_seek[tag];
 805 }
 806
 807
 808 static long check_virtual_tags(int tag, int idx_id,
 809                                const struct index_entry *idx)
 810 {
 811     long data = 0;
 812
 813     switch (tag)
 814     {
 815         case tag_virt_length_sec:
 816             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 817             break;
 818
 819         case tag_virt_length_min:
 820             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 821             break;
 822
 823         case tag_virt_playtime_sec:
 824             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 825             break;
 826
 827         case tag_virt_playtime_min:
 828             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 829             break;
 830
 831         case tag_virt_autoscore:
 832             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 833                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 834             {
 835                 data = 0;
 836             }
 837             else
 838             {
 839                 /* A straight calculus gives:
 840                      autoscore = 100 * playtime / length / playcout (1)
 841                    Now, consider the euclidian division of playtime by length:
 842                      playtime = alpha * length + beta
 843                    With:
 844                      0 <= beta < length
 845                    Now, (1) becomes:
 846                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 847                    Both terms should be small enough to avoid any overflow
 848                 */
 849                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 850                               / read_numeric_tag(tag_length, idx_id, idx))
 851                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 852                                  % read_numeric_tag(tag_length, idx_id, idx)))
 853                          / read_numeric_tag(tag_length, idx_id, idx);
 854                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 855             }
 856             break;
 857
 858         /* How many commits before the file has been added to the DB. */
 859         case tag_virt_entryage:
 860             data = current_tcmh.commitid
 861                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 862             break;
 863
 864         default:
 865             data = read_numeric_tag(tag, idx_id, idx);
 866     }
 867
 868     return data;
 869 }
 870
 871 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 872 {
 873     struct index_entry idx;
 874
 875     if (!tc_stat.ready)
 876         return false;
 877
 878     if (!TAGCACHE_IS_NUMERIC(tag))
 879         return -1;
 880
 881     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 882         return -2;
 883
 884     return check_virtual_tags(tag, tcs->idx_id, &idx);
 885 }
 886
 887 inline static bool str_ends_with(const char *str1, const char *str2)
 888 {
 889     int str_len = strlen(str1);
 890     int clause_len = strlen(str2);
 891
 892     if (clause_len > str_len)
 893         return false;
 894
 895     return !strcasecmp(&str1[str_len - clause_len], str2);
 896 }
 897
 898 inline static bool str_oneof(const char *str, const char *list)
 899 {
 900     const char *sep;
 901     int l, len = strlen(str);
 902
 903     while (*list)
 904     {
 905         sep = strchr(list, '|');
 906         l = sep ? (long)sep - (long)list : (int)strlen(list);
 907         if ((l==len) && !strncasecmp(str, list, len))
 908             return true;
 909         list += sep ? l + 1 : l;
 910     }
 911
 912     return false;
 913 }
 914
 915 static bool check_against_clause(long numeric, const char *str,
 916                                  const struct tagcache_search_clause *clause)
 917 {
 918     if (clause->numeric)
 919     {
 920         switch (clause->type)
 921         {
 922             case clause_is:
 923                 return numeric == clause->numeric_data;
 924             case clause_is_not:
 925                 return numeric != clause->numeric_data;
 926             case clause_gt:
 927                 return numeric > clause->numeric_data;
 928             case clause_gteq:
 929                 return numeric >= clause->numeric_data;
 930             case clause_lt:
 931                 return numeric < clause->numeric_data;
 932             case clause_lteq:
 933                 return numeric <= clause->numeric_data;
 934             default:
 935                 logf("Incorrect numeric tag: %d", clause->type);
 936         }
 937     }
 938     else
 939     {
 940         switch (clause->type)
 941         {
 942             case clause_is:
 943                 return !strcasecmp(clause->str, str);
 944             case clause_is_not:
 945                 return strcasecmp(clause->str, str);
 946             case clause_gt:
 947                 return 0>strcasecmp(clause->str, str);
 948             case clause_gteq:
 949                 return 0>=strcasecmp(clause->str, str);
 950             case clause_lt:
 951                 return 0<strcasecmp(clause->str, str);
 952             case clause_lteq:
 953                 return 0<=strcasecmp(clause->str, str);
 954             case clause_contains:
 955                 return (strcasestr(str, clause->str) != NULL);
 956             case clause_not_contains:
 957                 return (strcasestr(str, clause->str) == NULL);
 958             case clause_begins_with:
 959                 return (strcasestr(str, clause->str) == str);
 960             case clause_not_begins_with:
 961                 return (strcasestr(str, clause->str) != str);
 962             case clause_ends_with:
 963                 return str_ends_with(str, clause->str);
 964             case clause_not_ends_with:
 965                 return !str_ends_with(str, clause->str);
 966             case clause_oneof:
 967                 return str_oneof(str, clause->str);
 968
 969             default:
 970                 logf("Incorrect tag: %d", clause->type);
 971         }
 972     }
 973
 974     return false;
 975 }
 976
 977 static bool check_clauses(struct tagcache_search *tcs,
 978                           struct index_entry *idx,
 979                           struct tagcache_search_clause **clause, int count)
 980 {
 981     int i;
 982
 983 #ifdef HAVE_TC_RAMCACHE
 984     if (tcs->ramsearch)
 985     {
 986         /* Go through all conditional clauses. */
 987         for (i = 0; i < count; i++)
 988         {
 989             struct tagfile_entry *tfe;
 990             int seek;
 991             char buf[256];
 992             char *str = NULL;
 993
 994             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
 995
 996             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 997             {
 998                 if (clause[i]->tag == tag_filename)
 999                 {
1000                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1001                     str = buf;
1002                 }
1003                 else
1004                 {
1005                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1006                     str = tfe->tag_data;
1007                 }
1008             }
1009
1010             if (!check_against_clause(seek, str, clause[i]))
1011                 return false;
1012         }
1013     }
1014     else
1015 #endif
1016     {
1017         /* Check for conditions. */
1018         for (i = 0; i < count; i++)
1019         {
1020             struct tagfile_entry tfe;
1021             int seek;
1022             char str[256];
1023
1024             seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1025
1026             memset(str, 0, sizeof str);
1027             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1028             {
1029                 int fd = tcs->idxfd[clause[i]->tag];
1030                 lseek(fd, seek, SEEK_SET);
1031                 ecread_tagfile_entry(fd, &tfe);
1032                 if (tfe.tag_length >= (int)sizeof(str))
1033                 {
1034                     logf("Too long tag read!");
1035                     break ;
1036                 }
1037
1038                 read(fd, str, tfe.tag_length);
1039
1040                 /* Check if entry has been deleted. */
1041                 if (str[0] == '\0')
1042                     break;
1043             }
1044
1045             if (!check_against_clause(seek, str, clause[i]))
1046                 return false;
1047         }
1048     }
1049
1050     return true;
1051 }
1052
1053 bool tagcache_check_clauses(struct tagcache_search *tcs,
1054                             struct tagcache_search_clause **clause, int count)
1055 {
1056     struct index_entry idx;
1057
1058     if (count == 0)
1059         return true;
1060
1061     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1062         return false;
1063
1064     return check_clauses(tcs, &idx, clause, count);
1065 }
1066
1067 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1068 {
1069     int i;
1070
1071     /* If uniq buffer is not defined we must return true for search to work. */
1072     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1073         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1074     {
1075         return true;
1076     }
1077
1078     for (i = 0; i < tcs->unique_list_count; i++)
1079     {
1080         /* Return false if entry is found. */
1081         if (tcs->unique_list[i] == id)
1082             return false;
1083     }
1084
1085     if (tcs->unique_list_count < tcs->unique_list_capacity)
1086     {
1087         tcs->unique_list[i] = id;
1088         tcs->unique_list_count++;
1089     }
1090
1091     return true;
1092 }
1093
1094 static bool build_lookup_list(struct tagcache_search *tcs)
1095 {
1096     struct index_entry entry;
1097     int i, j;
1098
1099     tcs->seek_list_count = 0;
1100
1101 #ifdef HAVE_TC_RAMCACHE
1102     if (tcs->ramsearch
1103 # ifdef HAVE_DIRCACHE
1104         && (tcs->type != tag_filename || is_dircache_intact())
1105 # endif
1106         )
1107     {
1108         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1109         {
1110             struct tagcache_seeklist_entry *seeklist;
1111             struct index_entry *idx = &hdr->indices[i];
1112             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1113                 break ;
1114
1115             /* Skip deleted files. */
1116             if (idx->flag & FLAG_DELETED)
1117                 continue;
1118
1119             /* Go through all filters.. */
1120             for (j = 0; j < tcs->filter_count; j++)
1121             {
1122                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1123                 {
1124                     break ;
1125                 }
1126             }
1127
1128             if (j < tcs->filter_count)
1129                 continue ;
1130
1131             /* Check for conditions. */
1132             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1133                 continue;
1134
1135             /* Add to the seek list if not already in uniq buffer. */
1136             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1137                 continue;
1138
1139             /* Lets add it. */
1140             seeklist = &tcs->seeklist[tcs->seek_list_count];
1141             seeklist->seek = idx->tag_seek[tcs->type];
1142             seeklist->flag = idx->flag;
1143             seeklist->idx_id = i;
1144             tcs->seek_list_count++;
1145         }
1146
1147         tcs->seek_pos = i;
1148
1149         return tcs->seek_list_count > 0;
1150     }
1151 #endif
1152
1153     if (tcs->masterfd < 0)
1154     {
1155         struct master_header tcmh;
1156         tcs->masterfd = open_master_fd(&tcmh, false);
1157     }
1158
1159     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1160             sizeof(struct master_header), SEEK_SET);
1161
1162     while (ecread_index_entry(tcs->masterfd, &entry)
1163            == sizeof(struct index_entry))
1164     {
1165         struct tagcache_seeklist_entry *seeklist;
1166
1167         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1168             break ;
1169
1170         i = tcs->seek_pos;
1171         tcs->seek_pos++;
1172
1173         /* Check if entry has been deleted. */
1174         if (entry.flag & FLAG_DELETED)
1175             continue;
1176
1177         /* Go through all filters.. */
1178         for (j = 0; j < tcs->filter_count; j++)
1179         {
1180             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1181                 break ;
1182         }
1183
1184         if (j < tcs->filter_count)
1185             continue ;
1186
1187         /* Check for conditions. */
1188         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1189             continue;
1190
1191         /* Add to the seek list if not already in uniq buffer. */
1192         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1193             continue;
1194
1195         /* Lets add it. */
1196         seeklist = &tcs->seeklist[tcs->seek_list_count];
1197         seeklist->seek = entry.tag_seek[tcs->type];
1198         seeklist->flag = entry.flag;
1199         seeklist->idx_id = i;
1200         tcs->seek_list_count++;
1201
1202         yield();
1203     }
1204
1205     return tcs->seek_list_count > 0;
1206 }
1207
1208
1209 static void remove_files(void)
1210 {
1211     int i;
1212     char buf[MAX_PATH];
1213
1214     tc_stat.ready = false;
1215     tc_stat.ramcache = false;
1216     tc_stat.econ = false;
1217     remove(TAGCACHE_FILE_MASTER);
1218     for (i = 0; i < TAG_COUNT; i++)
1219     {
1220         if (TAGCACHE_IS_NUMERIC(i))
1221             continue;
1222
1223         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1224         remove(buf);
1225     }
1226 }
1227
1228
1229 static bool check_all_headers(void)
1230 {
1231     struct master_header myhdr;
1232     struct tagcache_header tch;
1233     int tag;
1234     int fd;
1235
1236     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1237         return false;
1238
1239     close(fd);
1240     if (myhdr.dirty)
1241     {
1242         logf("tagcache is dirty!");
1243         return false;
1244     }
1245
1246     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1247
1248     for (tag = 0; tag < TAG_COUNT; tag++)
1249     {
1250         if (TAGCACHE_IS_NUMERIC(tag))
1251             continue;
1252
1253         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1254             return false;
1255
1256         close(fd);
1257     }
1258
1259     return true;
1260 }
1261
1262 bool tagcache_is_busy(void)
1263 {
1264     return read_lock || write_lock;
1265 }
1266
1267 bool tagcache_search(struct tagcache_search *tcs, int tag)
1268 {
1269     struct tagcache_header tag_hdr;
1270     struct master_header   master_hdr;
1271     int i;
1272
1273     while (read_lock)
1274         sleep(1);
1275
1276     memset(tcs, 0, sizeof(struct tagcache_search));
1277     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1278         return false;
1279
1280     tcs->position = sizeof(struct tagcache_header);
1281     tcs->type = tag;
1282     tcs->seek_pos = 0;
1283     tcs->list_position = 0;
1284     tcs->seek_list_count = 0;
1285     tcs->filter_count = 0;
1286     tcs->masterfd = -1;
1287
1288     for (i = 0; i < TAG_COUNT; i++)
1289         tcs->idxfd[i] = -1;
1290
1291 #ifndef HAVE_TC_RAMCACHE
1292     tcs->ramsearch = false;
1293 #else
1294     tcs->ramsearch = tc_stat.ramcache;
1295     if (tcs->ramsearch)
1296     {
1297         tcs->entry_count = hdr->entry_count[tcs->type];
1298     }
1299     else
1300 #endif
1301     {
1302         /* Always open as R/W so we can pass tcs to functions that modify data also
1303          * without failing. */
1304         tcs->masterfd = open_master_fd(&master_hdr, true);
1305         if (tcs->masterfd < 0)
1306             return false;
1307
1308         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1309         {
1310             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1311             if (tcs->idxfd[tcs->type] < 0)
1312                 return false;
1313
1314             tcs->entry_count = tag_hdr.entry_count;
1315         }
1316         else
1317         {
1318             tcs->entry_count = master_hdr.tch.entry_count;
1319         }
1320     }
1321
1322     tcs->valid = true;
1323     tcs->initialized = true;
1324     write_lock++;
1325
1326     return true;
1327 }
1328
1329 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1330                                  void *buffer, long length)
1331 {
1332     tcs->unique_list = (unsigned long *)buffer;
1333     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1334     tcs->unique_list_count = 0;
1335 }
1336
1337 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1338                                 int tag, int seek)
1339 {
1340     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1341         return false;
1342
1343     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1344         return false;
1345
1346     tcs->filter_tag[tcs->filter_count] = tag;
1347     tcs->filter_seek[tcs->filter_count] = seek;
1348     tcs->filter_count++;
1349
1350     return true;
1351 }
1352
1353 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1354                                 struct tagcache_search_clause *clause)
1355 {
1356     int i;
1357
1358     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1359     {
1360         logf("Too many clauses");
1361         return false;
1362     }
1363
1364     /* Check if there is already a similar filter in present (filters are
1365      * much faster than clauses).
1366      */
1367     for (i = 0; i < tcs->filter_count; i++)
1368     {
1369         if (tcs->filter_tag[i] == clause->tag)
1370             return true;
1371     }
1372
1373     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1374     {
1375         char buf[MAX_PATH];
1376
1377         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1378         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1379     }
1380
1381     tcs->clause[tcs->clause_count] = clause;
1382     tcs->clause_count++;
1383
1384     return true;
1385 }
1386
1387 static bool get_next(struct tagcache_search *tcs)
1388 {
1389     static char buf[TAG_MAXLEN+32];
1390     struct tagfile_entry entry;
1391     long flag = 0;
1392
1393     if (!tcs->valid || !tc_stat.ready)
1394         return false;
1395
1396     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1397 #ifdef HAVE_TC_RAMCACHE
1398         && !tcs->ramsearch
1399 #endif
1400         )
1401         return false;
1402
1403     /* Relative fetch. */
1404     if (tcs->filter_count > 0 || tcs->clause_count > 0
1405         || TAGCACHE_IS_NUMERIC(tcs->type)
1406 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1407         /* We need to retrieve flag status for dircache. */
1408         || (tcs->ramsearch && tcs->type == tag_filename)
1409 #endif
1410         )
1411     {
1412         struct tagcache_seeklist_entry *seeklist;
1413
1414         /* Check for end of list. */
1415         if (tcs->list_position == tcs->seek_list_count)
1416         {
1417             tcs->list_position = 0;
1418
1419             /* Try to fetch more. */
1420             if (!build_lookup_list(tcs))
1421             {
1422                 tcs->valid = false;
1423                 return false;
1424             }
1425         }
1426
1427         seeklist = &tcs->seeklist[tcs->list_position];
1428         flag = seeklist->flag;
1429         tcs->position = seeklist->seek;
1430         tcs->idx_id = seeklist->idx_id;
1431         tcs->list_position++;
1432     }
1433     else
1434     {
1435         if (tcs->entry_count == 0)
1436         {
1437             tcs->valid = false;
1438             return false;
1439         }
1440
1441         tcs->entry_count--;
1442     }
1443
1444     tcs->result_seek = tcs->position;
1445
1446     if (TAGCACHE_IS_NUMERIC(tcs->type))
1447     {
1448         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1449         tcs->result = buf;
1450         tcs->result_len = strlen(buf) + 1;
1451         return true;
1452     }
1453
1454     /* Direct fetch. */
1455 #ifdef HAVE_TC_RAMCACHE
1456     if (tcs->ramsearch)
1457     {
1458
1459 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1460         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1461             && is_dircache_intact())
1462         {
1463             dircache_copy_path((struct dircache_entry *)tcs->position,
1464                                buf, sizeof buf);
1465             tcs->result = buf;
1466             tcs->result_len = strlen(buf) + 1;
1467             tcs->ramresult = false;
1468
1469             return true;
1470         }
1471         else
1472 #endif
1473         if (tcs->type != tag_filename)
1474         {
1475             struct tagfile_entry *ep;
1476
1477             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1478             tcs->result = ep->tag_data;
1479             tcs->result_len = strlen(tcs->result) + 1;
1480             tcs->idx_id = ep->idx_id;
1481             tcs->ramresult = true;
1482
1483             /* Increase position for the next run. This may get overwritten. */
1484             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1485
1486             return true;
1487         }
1488     }
1489 #endif
1490
1491     if (!open_files(tcs, tcs->type))
1492     {
1493         tcs->valid = false;
1494         return false;
1495     }
1496
1497     /* Seek stream to the correct position and continue to direct fetch. */
1498     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1499
1500     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1501     {
1502         logf("read error #5");
1503         tcs->valid = false;
1504         return false;
1505     }
1506
1507     if (entry.tag_length > (long)sizeof(buf))
1508     {
1509         tcs->valid = false;
1510         logf("too long tag #2");
1511         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1512         return false;
1513     }
1514
1515     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1516     {
1517         tcs->valid = false;
1518         logf("read error #4");
1519         return false;
1520     }
1521
1522     /**
1523      Update the position for the next read (this may be overridden
1524      if filters or clauses are being used).
1525      */
1526     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1527     tcs->result = buf;
1528     tcs->result_len = strlen(tcs->result) + 1;
1529     tcs->idx_id = entry.idx_id;
1530     tcs->ramresult = false;
1531
1532     return true;
1533 }
1534
1535 bool tagcache_get_next(struct tagcache_search *tcs)
1536 {
1537     while (get_next(tcs))
1538     {
1539         if (tcs->result_len > 1)
1540             return true;
1541     }
1542
1543     return false;
1544 }
1545
1546 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1547                        int tag, char *buf, long size)
1548 {
1549     struct index_entry idx;
1550
1551     *buf = '\0';
1552     if (!get_index(tcs->masterfd, idxid, &idx, true))
1553         return false;
1554
1555     return retrieve(tcs, &idx, tag, buf, size);
1556 }
1557
1558 static bool update_master_header(void)
1559 {
1560     struct master_header myhdr;
1561     int fd;
1562
1563     if (!tc_stat.ready)
1564         return false;
1565
1566     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1567         return false;
1568
1569     myhdr.serial = current_tcmh.serial;
1570     myhdr.commitid = current_tcmh.commitid;
1571     myhdr.dirty = current_tcmh.dirty;
1572
1573     /* Write it back */
1574     lseek(fd, 0, SEEK_SET);
1575     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1576     close(fd);
1577
1578 #ifdef HAVE_TC_RAMCACHE
1579     if (hdr)
1580     {
1581         hdr->h.serial = current_tcmh.serial;
1582         hdr->h.commitid = current_tcmh.commitid;
1583         hdr->h.dirty = current_tcmh.dirty;
1584     }
1585 #endif
1586
1587     return true;
1588 }
1589
1590 #if 0
1591
1592 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1593 {
1594     struct tagentry *entry;
1595
1596     if (tcs->type != tag_title)
1597         return ;
1598
1599     /* We will need reserve buffer for this. */
1600     if (tcs->ramcache)
1601     {
1602         struct tagfile_entry *ep;
1603
1604         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1605         tcs->seek_list[tcs->seek_list_count];
1606     }
1607
1608     entry = find_entry_ram();
1609
1610 }
1611 #endif
1612
1613 void tagcache_search_finish(struct tagcache_search *tcs)
1614 {
1615     int i;
1616
1617     if (!tcs->initialized)
1618         return;
1619
1620     if (tcs->masterfd >= 0)
1621     {
1622         close(tcs->masterfd);
1623         tcs->masterfd = -1;
1624     }
1625
1626     for (i = 0; i < TAG_COUNT; i++)
1627     {
1628         if (tcs->idxfd[i] >= 0)
1629         {
1630             close(tcs->idxfd[i]);
1631             tcs->idxfd[i] = -1;
1632         }
1633     }
1634
1635     tcs->ramsearch = false;
1636     tcs->valid = false;
1637     tcs->initialized = 0;
1638     if (write_lock > 0)
1639         write_lock--;
1640 }
1641
1642 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1643 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1644 {
1645     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1646 }
1647
1648 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1649 {
1650     return check_virtual_tags(tag, idx_id, entry);
1651 }
1652
1653 static char* get_tag_string(const struct index_entry *entry, int tag)
1654 {
1655     char* s = get_tag(entry, tag)->tag_data;
1656     return strcmp(s, UNTAGGED) ? s : NULL;
1657 }
1658
1659 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1660 {
1661     struct index_entry *entry;
1662     int idx_id;
1663
1664     if (!tc_stat.ready || !tc_stat.ramcache)
1665         return false;
1666
1667     /* Find the corresponding entry in tagcache. */
1668     idx_id = find_entry_ram(filename, NULL);
1669     if (idx_id < 0)
1670         return false;
1671
1672     entry = &hdr->indices[idx_id];
1673
1674     memset(id3, 0, sizeof(struct mp3entry));
1675
1676     id3->title        = get_tag_string(entry, tag_title);
1677     id3->artist       = get_tag_string(entry, tag_artist);
1678     id3->album        = get_tag_string(entry, tag_album);
1679     id3->genre_string = get_tag_string(entry, tag_genre);
1680     id3->composer     = get_tag_string(entry, tag_composer);
1681     id3->comment      = get_tag_string(entry, tag_comment);
1682     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1683     id3->grouping     = get_tag_string(entry, tag_grouping);
1684
1685     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1686     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1687     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1688     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1689     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1690     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1691
1692     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1693     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1694     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1695     if (id3->bitrate == 0)
1696         id3->bitrate = 1;
1697
1698 #if CONFIG_CODEC == SWCODEC
1699     if (global_settings.autoresume_enable)
1700     {
1701         id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1702         logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1703              id3->title, id3->offset);
1704     }
1705 #endif
1706
1707     return true;
1708 }
1709 #endif
1710
1711 static inline void write_item(const char *item)
1712 {
1713     int len = strlen(item) + 1;
1714
1715     data_size += len;
1716     write(cachefd, item, len);
1717 }
1718
1719 static int check_if_empty(char **tag)
1720 {
1721     int length;
1722
1723     if (*tag == NULL || **tag == '\0')
1724     {
1725         *tag = UNTAGGED;
1726         return sizeof(UNTAGGED); /* Tag length */
1727     }
1728
1729     length = strlen(*tag);
1730     if (length > TAG_MAXLEN)
1731     {
1732         logf("over length tag: %s", *tag);
1733         length = TAG_MAXLEN;
1734         (*tag)[length] = '\0';
1735     }
1736
1737     return length + 1;
1738 }
1739
1740 #define ADD_TAG(entry,tag,data) \
1741     /* Adding tag */ \
1742     entry.tag_offset[tag] = offset; \
1743     entry.tag_length[tag] = check_if_empty(data); \
1744     offset += entry.tag_length[tag]
1745 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1746  * idea, as it uses lots of stack and is called from a recursive function
1747  * (check_dir).
1748  */
1749 static void __attribute__ ((noinline)) add_tagcache(char *path,
1750                                                     unsigned long mtime
1751 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1752                                                     ,const struct dircache_entry *dc
1753 #endif
1754                                                    )
1755 {
1756     struct mp3entry id3;
1757     struct temp_file_entry entry;
1758     bool ret;
1759     int fd;
1760     int idx_id = -1;
1761     char tracknumfix[3];
1762     int offset = 0;
1763     int path_length = strlen(path);
1764     bool has_albumartist;
1765     bool has_grouping;
1766
1767 #ifdef SIMULATOR
1768     /* Crude logging for the sim - to aid in debugging */
1769     int logfd = open(ROCKBOX_DIR "/database.log",
1770                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1771     if (logfd >= 0) {
1772         write(logfd, path, strlen(path));
1773         write(logfd, "\n", 1);
1774         close(logfd);
1775     }
1776 #endif
1777
1778     if (cachefd < 0)
1779         return ;
1780
1781     /* Check for overlength file path. */
1782     if (path_length > TAG_MAXLEN)
1783     {
1784         /* Path can't be shortened. */
1785         logf("Too long path: %s", path);
1786         return ;
1787     }
1788
1789     /* Check if the file is supported. */
1790     if (probe_file_format(path) == AFMT_UNKNOWN)
1791         return ;
1792
1793     /* Check if the file is already cached. */
1794 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1795     if (tc_stat.ramcache && is_dircache_intact())
1796     {
1797         idx_id = find_entry_ram(path, dc);
1798     }
1799 #endif
1800
1801     /* Be sure the entry doesn't exist. */
1802     if (filenametag_fd >= 0 && idx_id < 0)
1803         idx_id = find_entry_disk(path, false);
1804
1805     /* Check if file has been modified. */
1806     if (idx_id >= 0)
1807     {
1808         struct index_entry idx;
1809
1810         /* TODO: Mark that the index exists (for fast reverse scan) */
1811         //found_idx[idx_id/8] |= idx_id%8;
1812
1813         if (!get_index(-1, idx_id, &idx, true))
1814         {
1815             logf("failed to retrieve index entry");
1816             return ;
1817         }
1818
1819         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1820         {
1821             /* No changes to file. */
1822             return ;
1823         }
1824
1825         /* Metadata might have been changed. Delete the entry. */
1826         logf("Re-adding: %s", path);
1827         if (!delete_entry(idx_id))
1828         {
1829             logf("delete_entry failed: %d", idx_id);
1830             return ;
1831         }
1832     }
1833
1834     fd = open(path, O_RDONLY);
1835     if (fd < 0)
1836     {
1837         logf("open fail: %s", path);
1838         return ;
1839     }
1840
1841     memset(&id3, 0, sizeof(struct mp3entry));
1842     memset(&entry, 0, sizeof(struct temp_file_entry));
1843     memset(&tracknumfix, 0, sizeof(tracknumfix));
1844     ret = get_metadata(&id3, fd, path);
1845     close(fd);
1846
1847     if (!ret)
1848         return ;
1849
1850     logf("-> %s", path);
1851
1852     /* Generate track number if missing. */
1853     if (id3.tracknum <= 0)
1854     {
1855         const char *p = strrchr(path, '.');
1856
1857         if (p == NULL)
1858             p = &path[strlen(path)-1];
1859
1860         while (*p != '/')
1861         {
1862             if (isdigit(*p) && isdigit(*(p-1)))
1863             {
1864                 tracknumfix[1] = *p--;
1865                 tracknumfix[0] = *p;
1866                 break;
1867             }
1868             p--;
1869         }
1870
1871         if (tracknumfix[0] != '\0')
1872         {
1873             id3.tracknum = atoi(tracknumfix);
1874             /* Set a flag to indicate track number has been generated. */
1875             entry.flag |= FLAG_TRKNUMGEN;
1876         }
1877         else
1878         {
1879             /* Unable to generate track number. */
1880             id3.tracknum = -1;
1881         }
1882     }
1883
1884     /* Numeric tags */
1885     entry.tag_offset[tag_year] = id3.year;
1886     entry.tag_offset[tag_discnumber] = id3.discnum;
1887     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1888     entry.tag_offset[tag_length] = id3.length;
1889     entry.tag_offset[tag_bitrate] = id3.bitrate;
1890     entry.tag_offset[tag_mtime] = mtime;
1891
1892     /* String tags. */
1893     has_albumartist = id3.albumartist != NULL
1894         && strlen(id3.albumartist) > 0;
1895     has_grouping = id3.grouping != NULL
1896         && strlen(id3.grouping) > 0;
1897
1898     ADD_TAG(entry, tag_filename, &path);
1899     ADD_TAG(entry, tag_title, &id3.title);
1900     ADD_TAG(entry, tag_artist, &id3.artist);
1901     ADD_TAG(entry, tag_album, &id3.album);
1902     ADD_TAG(entry, tag_genre, &id3.genre_string);
1903     ADD_TAG(entry, tag_composer, &id3.composer);
1904     ADD_TAG(entry, tag_comment, &id3.comment);
1905     if (has_albumartist)
1906     {
1907         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1908     }
1909     else
1910     {
1911         ADD_TAG(entry, tag_albumartist, &id3.artist);
1912     }
1913     if (has_grouping)
1914     {
1915         ADD_TAG(entry, tag_grouping, &id3.grouping);
1916     }
1917     else
1918     {
1919         ADD_TAG(entry, tag_grouping, &id3.title);
1920     }
1921     entry.data_length = offset;
1922
1923     /* Write the header */
1924     write(cachefd, &entry, sizeof(struct temp_file_entry));
1925
1926     /* And tags also... Correct order is critical */
1927     write_item(path);
1928     write_item(id3.title);
1929     write_item(id3.artist);
1930     write_item(id3.album);
1931     write_item(id3.genre_string);
1932     write_item(id3.composer);
1933     write_item(id3.comment);
1934     if (has_albumartist)
1935     {
1936         write_item(id3.albumartist);
1937     }
1938     else
1939     {
1940         write_item(id3.artist);
1941     }
1942     if (has_grouping)
1943     {
1944         write_item(id3.grouping);
1945     }
1946     else
1947     {
1948         write_item(id3.title);
1949     }
1950     total_entry_count++;
1951 }
1952
1953 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1954 {
1955     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1956     int len = strlen(str)+1;
1957     int i;
1958     unsigned crc32;
1959     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1960     char buf[TAG_MAXLEN+32];
1961
1962     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1963         buf[i] = tolower(str[i]);
1964     buf[i] = '\0';
1965
1966     crc32 = crc_32(buf, i, 0xffffffff);
1967
1968     if (unique)
1969     {
1970         /* Check if the crc does not exist -> entry does not exist for sure. */
1971         for (i = 0; i < tempbufidx; i++)
1972         {
1973             if (crcbuf[-i] != crc32)
1974                 continue;
1975
1976             if (!strcasecmp(str, index[i].str))
1977             {
1978                 if (id < 0 || id >= lookup_buffer_depth)
1979                 {
1980                     logf("lookup buf overf.: %d", id);
1981                     return false;
1982                 }
1983
1984                 lookup[id] = &index[i];
1985                 return true;
1986             }
1987         }
1988     }
1989
1990     /* Insert to CRC buffer. */
1991     crcbuf[-tempbufidx] = crc32;
1992     tempbuf_left -= 4;
1993
1994     /* Insert it to the buffer. */
1995     tempbuf_left -= len;
1996     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1997         return false;
1998
1999     if (id >= lookup_buffer_depth)
2000     {
2001         logf("lookup buf overf. #2: %d", id);
2002         return false;
2003     }
2004
2005     if (id >= 0)
2006     {
2007         lookup[id] = &index[tempbufidx];
2008         index[tempbufidx].idlist.id = id;
2009     }
2010     else
2011         index[tempbufidx].idlist.id = -1;
2012
2013     index[tempbufidx].idlist.next = NULL;
2014     index[tempbufidx].idx_id = idx_id;
2015     index[tempbufidx].seek = -1;
2016     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2017     memcpy(index[tempbufidx].str, str, len);
2018     tempbuf_pos += len;
2019     tempbufidx++;
2020
2021     return true;
2022 }
2023
2024 static int compare(const void *p1, const void *p2)
2025 {
2026     do_timed_yield();
2027
2028     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2029     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2030
2031     if (strcmp(e1->str, UNTAGGED) == 0)
2032     {
2033         if (strcmp(e2->str, UNTAGGED) == 0)
2034             return 0;
2035         return -1;
2036     }
2037     else if (strcmp(e2->str, UNTAGGED) == 0)
2038         return 1;
2039
2040     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2041 }
2042
2043 static int tempbuf_sort(int fd)
2044 {
2045     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2046     struct tagfile_entry fe;
2047     int i;
2048     int length;
2049
2050     /* Generate reverse lookup entries. */
2051     for (i = 0; i < lookup_buffer_depth; i++)
2052     {
2053         struct tempbuf_id_list *idlist;
2054
2055         if (!lookup[i])
2056             continue;
2057
2058         if (lookup[i]->idlist.id == i)
2059             continue;
2060
2061         idlist = &lookup[i]->idlist;
2062         while (idlist->next != NULL)
2063             idlist = idlist->next;
2064
2065         tempbuf_left -= sizeof(struct tempbuf_id_list);
2066         if (tempbuf_left - 4 < 0)
2067             return -1;
2068
2069         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2070         if (tempbuf_pos & 0x03)
2071         {
2072             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2073             tempbuf_left -= 3;
2074             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2075         }
2076         tempbuf_pos += sizeof(struct tempbuf_id_list);
2077
2078         idlist = idlist->next;
2079         idlist->id = i;
2080         idlist->next = NULL;
2081
2082         do_timed_yield();
2083     }
2084
2085     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2086     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2087
2088     for (i = 0; i < tempbufidx; i++)
2089     {
2090         struct tempbuf_id_list *idlist = &index[i].idlist;
2091
2092         /* Fix the lookup list. */
2093         while (idlist != NULL)
2094         {
2095             if (idlist->id >= 0)
2096                 lookup[idlist->id] = &index[i];
2097             idlist = idlist->next;
2098         }
2099
2100         index[i].seek = lseek(fd, 0, SEEK_CUR);
2101         length = strlen(index[i].str) + 1;
2102         fe.tag_length = length;
2103         fe.idx_id = index[i].idx_id;
2104
2105         /* Check the chunk alignment. */
2106         if ((fe.tag_length + sizeof(struct tagfile_entry))
2107             % TAGFILE_ENTRY_CHUNK_LENGTH)
2108         {
2109             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2110                 ((fe.tag_length + sizeof(struct tagfile_entry))
2111                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2112         }
2113
2114 #ifdef TAGCACHE_STRICT_ALIGN
2115         /* Make sure the entry is long aligned. */
2116         if (index[i].seek & 0x03)
2117         {
2118             logf("tempbuf_sort: alignment error!");
2119             return -3;
2120         }
2121 #endif
2122
2123         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2124             sizeof(struct tagfile_entry))
2125         {
2126             logf("tempbuf_sort: write error #1");
2127             return -1;
2128         }
2129
2130         if (write(fd, index[i].str, length) != length)
2131         {
2132             logf("tempbuf_sort: write error #2");
2133             return -2;
2134         }
2135
2136         /* Write some padding. */
2137         if (fe.tag_length - length > 0)
2138             write(fd, "XXXXXXXX", fe.tag_length - length);
2139     }
2140
2141     return i;
2142 }
2143
2144 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2145 {
2146     if (id < 0 || id >= lookup_buffer_depth)
2147         return NULL;
2148
2149     return lookup[id];
2150 }
2151
2152
2153 inline static int tempbuf_find_location(int id)
2154 {
2155     struct tempbuf_searchidx *entry;
2156
2157     entry = tempbuf_locate(id);
2158     if (entry == NULL)
2159         return -1;
2160
2161     return entry->seek;
2162 }
2163
2164 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2165 {
2166     struct master_header tcmh;
2167     struct index_entry idx;
2168     int masterfd;
2169     int masterfd_pos;
2170     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2171     int max_entries;
2172     int entries_processed = 0;
2173     int i, j;
2174     char buf[TAG_MAXLEN];
2175
2176     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2177
2178     logf("Building numeric indices...");
2179     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2180
2181     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2182         return false;
2183
2184     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2185                          SEEK_CUR);
2186     if (masterfd_pos == filesize(masterfd))
2187     {
2188         logf("we can't append!");
2189         close(masterfd);
2190         return false;
2191     }
2192
2193     while (entries_processed < h->entry_count)
2194     {
2195         int count = MIN(h->entry_count - entries_processed, max_entries);
2196
2197         /* Read in as many entries as possible. */
2198         for (i = 0; i < count; i++)
2199         {
2200             struct temp_file_entry *tfe = &entrybuf[i];
2201             int datastart;
2202
2203             /* Read in numeric data. */
2204             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2205                 sizeof(struct temp_file_entry))
2206             {
2207                 logf("read fail #1");
2208                 close(masterfd);
2209                 return false;
2210             }
2211
2212             datastart = lseek(tmpfd, 0, SEEK_CUR);
2213
2214             /**
2215              * Read string data from the following tags:
2216              * - tag_filename
2217              * - tag_artist
2218              * - tag_album
2219              * - tag_title
2220              *
2221              * A crc32 hash is calculated from the read data
2222              * and stored back to the data offset field kept in memory.
2223              */
2224 #define tmpdb_read_string_tag(tag) \
2225     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2226     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2227     { \
2228         logf("read fail: buffer overflow"); \
2229         close(masterfd); \
2230         return false; \
2231     } \
2232     \
2233     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2234         tfe->tag_length[tag]) \
2235     { \
2236         logf("read fail #2"); \
2237         close(masterfd); \
2238         return false; \
2239     } \
2240     \
2241     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2242     lseek(tmpfd, datastart, SEEK_SET)
2243
2244             tmpdb_read_string_tag(tag_filename);
2245             tmpdb_read_string_tag(tag_artist);
2246             tmpdb_read_string_tag(tag_album);
2247             tmpdb_read_string_tag(tag_title);
2248
2249             /* Seek to the end of the string data. */
2250             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2251         }
2252
2253         /* Backup the master index position. */
2254         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2255         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2256
2257         /* Check if we can resurrect some deleted runtime statistics data. */
2258         for (i = 0; i < tcmh.tch.entry_count; i++)
2259         {
2260             /* Read the index entry. */
2261             if (ecread_index_entry(masterfd, &idx)
2262                 != sizeof(struct index_entry))
2263             {
2264                 logf("read fail #3");
2265                 close(masterfd);
2266                 return false;
2267             }
2268
2269             /**
2270              * Skip unless the entry is marked as being deleted
2271              * or the data has already been resurrected.
2272              */
2273             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2274                 continue;
2275
2276             /* Now try to match the entry. */
2277             /**
2278              * To succesfully match a song, the following conditions
2279              * must apply:
2280              *
2281              * For numeric fields: tag_length
2282              * - Full identical match is required
2283              *
2284              * If tag_filename matches, no further checking necessary.
2285              *
2286              * For string hashes: tag_artist, tag_album, tag_title
2287              * - Two of these must match
2288              */
2289             for (j = 0; j < count; j++)
2290             {
2291                 struct temp_file_entry *tfe = &entrybuf[j];
2292
2293                 /* Try to match numeric fields first. */
2294                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2295                     continue;
2296
2297                 /* Now it's time to do the hash matching. */
2298                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2299                 {
2300                     int match_count = 0;
2301
2302                     /* No filename match, check if we can match two other tags. */
2303 #define tmpdb_match(tag) \
2304     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2305         match_count++
2306
2307                     tmpdb_match(tag_artist);
2308                     tmpdb_match(tag_album);
2309                     tmpdb_match(tag_title);
2310
2311                     if (match_count < 2)
2312                     {
2313                         /* Still no match found, give up. */
2314                         continue;
2315                     }
2316                 }
2317
2318                 /* A match found, now copy & resurrect the statistical data. */
2319 #define tmpdb_copy_tag(tag) \
2320     tfe->tag_offset[tag] = idx.tag_seek[tag]
2321
2322                 tmpdb_copy_tag(tag_playcount);
2323                 tmpdb_copy_tag(tag_rating);
2324                 tmpdb_copy_tag(tag_playtime);
2325                 tmpdb_copy_tag(tag_lastplayed);
2326                 tmpdb_copy_tag(tag_commitid);
2327                 tmpdb_copy_tag(tag_lastoffset);
2328
2329                 /* Avoid processing this entry again. */
2330                 idx.flag |= FLAG_RESURRECTED;
2331
2332                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2333                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2334                 {
2335                     logf("masterfd writeback fail #1");
2336                     close(masterfd);
2337                     return false;
2338                 }
2339
2340                 logf("Entry resurrected");
2341             }
2342         }
2343
2344
2345         /* Restore the master index position. */
2346         lseek(masterfd, masterfd_pos, SEEK_SET);
2347
2348         /* Commit the data to the index. */
2349         for (i = 0; i < count; i++)
2350         {
2351             int loc = lseek(masterfd, 0, SEEK_CUR);
2352
2353             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2354             {
2355                 logf("read fail #3");
2356                 close(masterfd);
2357                 return false;
2358             }
2359
2360             for (j = 0; j < TAG_COUNT; j++)
2361             {
2362                 if (!TAGCACHE_IS_NUMERIC(j))
2363                     continue;
2364
2365                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2366             }
2367             idx.flag = entrybuf[i].flag;
2368
2369             if (idx.tag_seek[tag_commitid])
2370             {
2371                 /* Data has been resurrected. */
2372                 idx.flag |= FLAG_DIRTYNUM;
2373             }
2374             else if (tc_stat.ready && current_tcmh.commitid > 0)
2375             {
2376                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2377                 idx.flag |= FLAG_DIRTYNUM;
2378             }
2379
2380             /* Write back the updated index. */
2381             lseek(masterfd, loc, SEEK_SET);
2382             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2383             {
2384                 logf("write fail");
2385                 close(masterfd);
2386                 return false;
2387             }
2388         }
2389
2390         entries_processed += count;
2391         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2392     }
2393
2394     close(masterfd);
2395
2396     return true;
2397 }
2398
2399 /**
2400  * Return values:
2401  *     > 0   success
2402  *    == 0   temporary failure
2403  *     < 0   fatal error
2404  */
2405 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2406 {
2407     int i;
2408     struct tagcache_header tch;
2409     struct master_header   tcmh;
2410     struct index_entry idxbuf[IDX_BUF_DEPTH];
2411     int idxbuf_pos;
2412     char buf[TAG_MAXLEN+32];
2413     int fd = -1, masterfd;
2414     bool error = false;
2415     int init;
2416     int masterfd_pos;
2417
2418     logf("Building index: %d", index_type);
2419
2420     /* Check the number of entries we need to allocate ram for. */
2421     commit_entry_count = h->entry_count + 1;
2422
2423     masterfd = open_master_fd(&tcmh, false);
2424     if (masterfd >= 0)
2425     {
2426         commit_entry_count += tcmh.tch.entry_count;
2427         close(masterfd);
2428     }
2429     else
2430         remove_files(); /* Just to be sure we are clean. */
2431
2432     /* Open the index file, which contains the tag names. */
2433     fd = open_tag_fd(&tch, index_type, true);
2434     if (fd >= 0)
2435     {
2436         logf("tch.datasize=%ld", tch.datasize);
2437         lookup_buffer_depth = 1 +
2438         /* First part */ commit_entry_count +
2439         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2440     }
2441     else
2442     {
2443         lookup_buffer_depth = 1 +
2444         /* First part */ commit_entry_count +
2445         /* Second part */ 0;
2446     }
2447
2448     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2449     logf("commit_entry_count=%ld", commit_entry_count);
2450
2451     /* Allocate buffer for all index entries from both old and new
2452      * tag files. */
2453     tempbufidx = 0;
2454     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2455
2456     /* Allocate lookup buffer. The first portion of commit_entry_count
2457      * contains the new tags in the temporary file and the second
2458      * part for locating entries already in the db.
2459      *
2460      *  New tags  Old tags
2461      * +---------+---------------------------+
2462      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2463      * +---------+---------------------------+
2464      *
2465      * Old tags are inserted to a temporary buffer with position:
2466      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2467      * And new tags with index:
2468      *     tempbuf_insert(idx, ...);
2469      *
2470      * The buffer is sorted and written into tag file:
2471      *     tempbuf_sort(...);
2472      * leaving master index locations messed up.
2473      *
2474      * That is fixed using the lookup buffer for old tags:
2475      *     new_seek = tempbuf_find_location(old_seek, ...);
2476      * and for new tags:
2477      *     new_seek = tempbuf_find_location(idx);
2478      */
2479     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2480     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2481     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2482
2483     /* And calculate the remaining data space used mainly for storing
2484      * tag data (strings). */
2485     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2486     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2487     {
2488         logf("Buffer way too small!");
2489         return 0;
2490     }
2491
2492     if (fd >= 0)
2493     {
2494         /**
2495          * If tag file contains unique tags (sorted index), we will load
2496          * it entirely into memory so we can resort it later for use with
2497          * chunked browsing.
2498          */
2499         if (TAGCACHE_IS_SORTED(index_type))
2500         {
2501             logf("loading tags...");
2502             for (i = 0; i < tch.entry_count; i++)
2503             {
2504                 struct tagfile_entry entry;
2505                 int loc = lseek(fd, 0, SEEK_CUR);
2506                 bool ret;
2507
2508                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2509                 {
2510                     logf("read error #7");
2511                     close(fd);
2512                     return -2;
2513                 }
2514
2515                 if (entry.tag_length >= (int)sizeof(buf))
2516                 {
2517                     logf("too long tag #3");
2518                     close(fd);
2519                     return -2;
2520                 }
2521
2522                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2523                 {
2524                     logf("read error #8");
2525                     close(fd);
2526                     return -2;
2527                 }
2528
2529                 /* Skip deleted entries. */
2530                 if (buf[0] == '\0')
2531                     continue;
2532
2533                 /**
2534                  * Save the tag and tag id in the memory buffer. Tag id
2535                  * is saved so we can later reindex the master lookup
2536                  * table when the index gets resorted.
2537                  */
2538                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2539                                      + commit_entry_count, entry.idx_id,
2540                                      TAGCACHE_IS_UNIQUE(index_type));
2541                 if (!ret)
2542                 {
2543                     close(fd);
2544                     return -3;
2545                 }
2546                 do_timed_yield();
2547             }
2548             logf("done");
2549         }
2550         else
2551             tempbufidx = tch.entry_count;
2552     }
2553     else
2554     {
2555         /**
2556          * Creating new index file to store the tags. No need to preload
2557          * anything whether the index type is sorted or not.
2558          */
2559         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2560         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2561         if (fd < 0)
2562         {
2563             logf("%s open fail", buf);
2564             return -2;
2565         }
2566
2567         tch.magic = TAGCACHE_MAGIC;
2568         tch.entry_count = 0;
2569         tch.datasize = 0;
2570
2571         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2572             != sizeof(struct tagcache_header))
2573         {
2574             logf("header write failed");
2575             close(fd);
2576             return -2;
2577         }
2578     }
2579
2580     /* Loading the tag lookup file as "master file". */
2581     logf("Loading index file");
2582     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2583
2584     if (masterfd < 0)
2585     {
2586         logf("Creating new DB");
2587         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2588
2589         if (masterfd < 0)
2590         {
2591             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2592             close(fd);
2593             return -2;
2594         }
2595
2596         /* Write the header (write real values later). */
2597         memset(&tcmh, 0, sizeof(struct master_header));
2598         tcmh.tch = *h;
2599         tcmh.tch.entry_count = 0;
2600         tcmh.tch.datasize = 0;
2601         tcmh.dirty = true;
2602         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2603         init = true;
2604         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2605     }
2606     else
2607     {
2608         /**
2609          * Master file already exists so we need to process the current
2610          * file first.
2611          */
2612         init = false;
2613
2614         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2615             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2616         {
2617             logf("header error");
2618             close(fd);
2619             close(masterfd);
2620             return -2;
2621         }
2622
2623         /**
2624          * If we reach end of the master file, we need to expand it to
2625          * hold new tags. If the current index is not sorted, we can
2626          * simply append new data to end of the file.
2627          * However, if the index is sorted, we need to update all tag
2628          * pointers in the master file for the current index.
2629          */
2630         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2631             SEEK_CUR);
2632         if (masterfd_pos == filesize(masterfd))
2633         {
2634             logf("appending...");
2635             init = true;
2636         }
2637     }
2638
2639     /**
2640      * Load new unique tags in memory to be sorted later and added
2641      * to the master lookup file.
2642      */
2643     if (TAGCACHE_IS_SORTED(index_type))
2644     {
2645         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2646         /* h is the header of the temporary file containing new tags. */
2647         logf("inserting new tags...");
2648         for (i = 0; i < h->entry_count; i++)
2649         {
2650             struct temp_file_entry entry;
2651
2652             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2653                 sizeof(struct temp_file_entry))
2654             {
2655                 logf("read fail #3");
2656                 error = true;
2657                 goto error_exit;
2658             }
2659
2660             /* Read data. */
2661             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2662             {
2663                 logf("too long entry!");
2664                 error = true;
2665                 goto error_exit;
2666             }
2667
2668             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2669             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2670                 entry.tag_length[index_type])
2671             {
2672                 logf("read fail #4");
2673                 error = true;
2674                 goto error_exit;
2675             }
2676
2677             if (TAGCACHE_IS_UNIQUE(index_type))
2678                 error = !tempbuf_insert(buf, i, -1, true);
2679             else
2680                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2681
2682             if (error)
2683             {
2684                 logf("insert error");
2685                 goto error_exit;
2686             }
2687
2688             /* Skip to next. */
2689             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2690                     entry.tag_length[index_type], SEEK_CUR);
2691             do_timed_yield();
2692         }
2693         logf("done");
2694
2695         /* Sort the buffer data and write it to the index file. */
2696         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2697         /**
2698          * We need to truncate the index file now. There can be junk left
2699          * at the end of file (however, we _should_ always follow the
2700          * entry_count and don't crash with that).
2701          */
2702         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2703
2704         i = tempbuf_sort(fd);
2705         if (i < 0)
2706             goto error_exit;
2707         logf("sorted %d tags", i);
2708
2709         /**
2710          * Now update all indexes in the master lookup file.
2711          */
2712         logf("updating indices...");
2713         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2714         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2715         {
2716             int j;
2717             int loc = lseek(masterfd, 0, SEEK_CUR);
2718
2719             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2720
2721             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2722                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2723             {
2724                 logf("read fail #5");
2725                 error = true;
2726                 goto error_exit ;
2727             }
2728             lseek(masterfd, loc, SEEK_SET);
2729
2730             for (j = 0; j < idxbuf_pos; j++)
2731             {
2732                 if (idxbuf[j].flag & FLAG_DELETED)
2733                 {
2734                     /* We can just ignore deleted entries. */
2735                     // idxbuf[j].tag_seek[index_type] = 0;
2736                     continue;
2737                 }
2738
2739                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2740                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2741                     + commit_entry_count);
2742
2743                 if (idxbuf[j].tag_seek[index_type] < 0)
2744                 {
2745                     logf("update error: %ld/%d/%ld",
2746                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2747                     error = true;
2748                     goto error_exit;
2749                 }
2750
2751                 do_timed_yield();
2752             }
2753
2754             /* Write back the updated index. */
2755             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2756                         index_entry_ec, tc_stat.econ) !=
2757                 (int)sizeof(struct index_entry)*idxbuf_pos)
2758             {
2759                 logf("write fail");
2760                 error = true;
2761                 goto error_exit;
2762             }
2763         }
2764         logf("done");
2765     }
2766
2767     /**
2768      * Walk through the temporary file containing the new tags.
2769      */
2770     // build_normal_index(h, tmpfd, masterfd, idx);
2771     logf("updating new indices...");
2772     lseek(masterfd, masterfd_pos, SEEK_SET);
2773     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2774     lseek(fd, 0, SEEK_END);
2775     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2776     {
2777         int j;
2778
2779         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2780         if (init)
2781         {
2782             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2783         }
2784         else
2785         {
2786             int loc = lseek(masterfd, 0, SEEK_CUR);
2787
2788             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2789                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2790             {
2791                 logf("read fail #6");
2792                 error = true;
2793                 break ;
2794             }
2795             lseek(masterfd, loc, SEEK_SET);
2796         }
2797
2798         /* Read entry headers. */
2799         for (j = 0; j < idxbuf_pos; j++)
2800         {
2801             if (!TAGCACHE_IS_SORTED(index_type))
2802             {
2803                 struct temp_file_entry entry;
2804                 struct tagfile_entry fe;
2805
2806                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2807                     sizeof(struct temp_file_entry))
2808                 {
2809                     logf("read fail #7");
2810                     error = true;
2811                     break ;
2812                 }
2813
2814                 /* Read data. */
2815                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2816                 {
2817                     logf("too long entry!");
2818                     logf("length=%d", entry.tag_length[index_type]);
2819                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2820                     error = true;
2821                     break ;
2822                 }
2823
2824                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2825                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2826                     entry.tag_length[index_type])
2827                 {
2828                     logf("read fail #8");
2829                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2830                     logf("length=0x%02x", entry.tag_length[index_type]);
2831                     error = true;
2832                     break ;
2833                 }
2834
2835                 /* Write to index file. */
2836                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2837                 fe.tag_length = entry.tag_length[index_type];
2838                 fe.idx_id = tcmh.tch.entry_count + i + j;
2839                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2840                 write(fd, buf, fe.tag_length);
2841                 tempbufidx++;
2842
2843                 /* Skip to next. */
2844                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2845                       entry.tag_length[index_type], SEEK_CUR);
2846             }
2847             else
2848             {
2849                 /* Locate the correct entry from the sorted array. */
2850                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2851                 if (idxbuf[j].tag_seek[index_type] < 0)
2852                 {
2853                     logf("entry not found (%d)", j);
2854                     error = true;
2855                     break ;
2856                 }
2857             }
2858         }
2859
2860         /* Write index. */
2861         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2862                   index_entry_ec, tc_stat.econ) !=
2863             (int)sizeof(struct index_entry)*idxbuf_pos)
2864         {
2865             logf("tagcache: write fail #4");
2866             error = true;
2867             break ;
2868         }
2869
2870         do_timed_yield();
2871     }
2872     logf("done");
2873
2874     /* Finally write the header. */
2875     tch.magic = TAGCACHE_MAGIC;
2876     tch.entry_count = tempbufidx;
2877     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2878     lseek(fd, 0, SEEK_SET);
2879     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2880
2881     if (index_type != tag_filename)
2882         h->datasize += tch.datasize;
2883     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2884     error_exit:
2885
2886     close(fd);
2887     close(masterfd);
2888
2889     if (error)
2890         return -2;
2891
2892     return 1;
2893 }
2894
2895 static bool commit(void)
2896 {
2897     struct tagcache_header tch;
2898     struct master_header   tcmh;
2899     int i, len, rc;
2900     int tmpfd;
2901     int masterfd;
2902 #ifdef HAVE_DIRCACHE
2903     bool dircache_buffer_stolen = false;
2904 #endif
2905     bool local_allocation = false;
2906
2907     logf("committing tagcache");
2908
2909     while (write_lock)
2910         sleep(1);
2911
2912     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2913     if (tmpfd < 0)
2914     {
2915         logf("nothing to commit");
2916         return true;
2917     }
2918
2919
2920     /* Load the header. */
2921     len = sizeof(struct tagcache_header);
2922     rc = read(tmpfd, &tch, len);
2923
2924     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2925     {
2926         logf("incorrect tmpheader");
2927         close(tmpfd);
2928         remove(TAGCACHE_FILE_TEMP);
2929         return false;
2930     }
2931
2932     if (tch.entry_count == 0)
2933     {
2934         logf("nothing to commit");
2935         close(tmpfd);
2936         remove(TAGCACHE_FILE_TEMP);
2937         return true;
2938     }
2939
2940     /* Fully initialize existing headers (if any) before going further. */
2941     tc_stat.ready = check_all_headers();
2942
2943 #ifdef HAVE_EEPROM_SETTINGS
2944     remove(TAGCACHE_STATEFILE);
2945 #endif
2946
2947     /* At first be sure to unload the ramcache! */
2948 #ifdef HAVE_TC_RAMCACHE
2949     tc_stat.ramcache = false;
2950 #endif
2951
2952     read_lock++;
2953
2954     /* Try to steal every buffer we can :) */
2955     if (tempbuf_size == 0)
2956         local_allocation = true;
2957
2958 #ifdef HAVE_DIRCACHE
2959     if (tempbuf_size == 0)
2960     {
2961         /* Try to steal the dircache buffer. */
2962         tempbuf = dircache_steal_buffer(&tempbuf_size);
2963         tempbuf_size &= ~0x03;
2964
2965         if (tempbuf_size > 0)
2966         {
2967             dircache_buffer_stolen = true;
2968         }
2969     }
2970 #endif
2971
2972 #ifdef HAVE_TC_RAMCACHE
2973     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2974     {
2975         tempbuf = (char *)(hdr + 1);
2976         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2977         tempbuf_size &= ~0x03;
2978     }
2979 #endif
2980
2981     /* And finally fail if there are no buffers available. */
2982     if (tempbuf_size == 0)
2983     {
2984         logf("delaying commit until next boot");
2985         tc_stat.commit_delayed = true;
2986         close(tmpfd);
2987         read_lock--;
2988         return false;
2989     }
2990
2991     logf("commit %ld entries...", tch.entry_count);
2992
2993     /* Mark DB dirty so it will stay disabled if commit fails. */
2994     current_tcmh.dirty = true;
2995     update_master_header();
2996
2997     /* Now create the index files. */
2998     tc_stat.commit_step = 0;
2999     tch.datasize = 0;
3000     tc_stat.commit_delayed = false;
3001
3002     for (i = 0; i < TAG_COUNT; i++)
3003     {
3004         int ret;
3005
3006         if (TAGCACHE_IS_NUMERIC(i))
3007             continue;
3008
3009         tc_stat.commit_step++;
3010         ret = build_index(i, &tch, tmpfd);
3011         if (ret <= 0)
3012         {
3013             close(tmpfd);
3014             logf("tagcache failed init");
3015             if (ret == 0)
3016                 tc_stat.commit_delayed = true;
3017
3018             tc_stat.commit_step = 0;
3019             read_lock--;
3020             return false;
3021         }
3022     }
3023
3024     if (!build_numeric_indices(&tch, tmpfd))
3025     {
3026         logf("Failure to commit numeric indices");
3027         close(tmpfd);
3028         tc_stat.commit_step = 0;
3029         read_lock--;
3030         return false;
3031     }
3032
3033     close(tmpfd);
3034     remove(TAGCACHE_FILE_TEMP);
3035
3036     tc_stat.commit_step = 0;
3037
3038     /* Update the master index headers. */
3039     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3040     {
3041         read_lock--;
3042         return false;
3043     }
3044
3045     tcmh.tch.entry_count += tch.entry_count;
3046     tcmh.tch.datasize = sizeof(struct master_header)
3047         + sizeof(struct index_entry) * tcmh.tch.entry_count
3048         + tch.datasize;
3049     tcmh.dirty = false;
3050     tcmh.commitid++;
3051
3052     lseek(masterfd, 0, SEEK_SET);
3053     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3054     close(masterfd);
3055
3056     logf("tagcache committed");
3057     tc_stat.ready = check_all_headers();
3058     tc_stat.readyvalid = true;
3059
3060     if (local_allocation)
3061     {
3062         tempbuf = NULL;
3063         tempbuf_size = 0;
3064     }
3065
3066 #ifdef HAVE_DIRCACHE
3067     /* Rebuild the dircache, if we stole the buffer. */
3068     if (dircache_buffer_stolen)
3069         dircache_build(0);
3070 #endif
3071
3072 #ifdef HAVE_TC_RAMCACHE
3073     /* Reload tagcache. */
3074     if (tc_stat.ramcache_allocated > 0)
3075         tagcache_start_scan();
3076 #endif
3077
3078     read_lock--;
3079
3080     return true;
3081 }
3082
3083 static void allocate_tempbuf(void)
3084 {
3085     /* Yeah, malloc would be really nice now :) */
3086 #ifdef __PCTOOL__
3087     tempbuf_size = 32*1024*1024;
3088     tempbuf = malloc(tempbuf_size);
3089 #else
3090     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3091     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3092     audiobuf += tempbuf_size;
3093 #endif
3094 }
3095
3096 static void free_tempbuf(void)
3097 {
3098     if (tempbuf_size == 0)
3099         return ;
3100
3101 #ifdef __PCTOOL__
3102     free(tempbuf);
3103 #else
3104     audiobuf -= tempbuf_size;
3105 #endif
3106     tempbuf = NULL;
3107     tempbuf_size = 0;
3108 }
3109
3110 #ifndef __PCTOOL__
3111
3112 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3113 {
3114     struct index_entry idx;
3115
3116     if (!tc_stat.ready)
3117         return false;
3118
3119     if (!TAGCACHE_IS_NUMERIC(tag))
3120         return false;
3121
3122     if (!get_index(masterfd, idx_id, &idx, false))
3123         return false;
3124
3125     idx.tag_seek[tag] = data;
3126     idx.flag |= FLAG_DIRTYNUM;
3127
3128     return write_index(masterfd, idx_id, &idx);
3129 }
3130
3131 #if 0
3132 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3133                                    int tag, long data)
3134 {
3135     struct master_header myhdr;
3136
3137     if (tcs->masterfd < 0)
3138     {
3139         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3140             return false;
3141     }
3142
3143     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3144 }
3145 #endif
3146
3147 static bool command_queue_is_full(void)
3148 {
3149     int next;
3150
3151     next = command_queue_widx + 1;
3152     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3153         next = 0;
3154
3155     return (next == command_queue_ridx);
3156 }
3157
3158 static void command_queue_sync_callback(void *data)
3159 {
3160     (void)data;
3161     struct master_header myhdr;
3162     int masterfd;
3163
3164     mutex_lock(&command_queue_mutex);
3165
3166     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3167         return;
3168
3169     while (command_queue_ridx != command_queue_widx)
3170     {
3171         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3172
3173         switch (ce->command)
3174         {
3175             case CMD_UPDATE_MASTER_HEADER:
3176             {
3177                 close(masterfd);
3178                 update_master_header();
3179
3180                 /* Re-open the masterfd. */
3181                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3182                     return;
3183
3184                 break;
3185             }
3186             case CMD_UPDATE_NUMERIC:
3187             {
3188                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3189                 break;
3190             }
3191         }
3192
3193         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3194             command_queue_ridx = 0;
3195     }
3196
3197     close(masterfd);
3198
3199     tc_stat.queue_length = 0;
3200     mutex_unlock(&command_queue_mutex);
3201 }
3202
3203 static void run_command_queue(bool force)
3204 {
3205     if (COMMAND_QUEUE_IS_EMPTY)
3206         return;
3207
3208     if (force || command_queue_is_full())
3209         command_queue_sync_callback(NULL);
3210     else
3211         register_storage_idle_func(command_queue_sync_callback);
3212 }
3213
3214 static void queue_command(int cmd, long idx_id, int tag, long data)
3215 {
3216     while (1)
3217     {
3218         int next;
3219
3220         mutex_lock(&command_queue_mutex);
3221         next = command_queue_widx + 1;
3222         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3223             next = 0;
3224
3225         /* Make sure queue is not full. */
3226         if (next != command_queue_ridx)
3227         {
3228             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3229
3230             ce->command = cmd;
3231             ce->idx_id = idx_id;
3232             ce->tag = tag;
3233             ce->data = data;
3234
3235             command_queue_widx = next;
3236
3237             tc_stat.queue_length++;
3238
3239             mutex_unlock(&command_queue_mutex);
3240             break;
3241         }
3242
3243         /* Queue is full, try again later... */
3244         mutex_unlock(&command_queue_mutex);
3245         sleep(1);
3246     }
3247 }
3248
3249 long tagcache_increase_serial(void)
3250 {
3251     long old;
3252
3253     if (!tc_stat.ready)
3254         return -2;
3255
3256     while (read_lock)
3257         sleep(1);
3258
3259     old = current_tcmh.serial++;
3260     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3261
3262     return old;
3263 }
3264
3265 void tagcache_update_numeric(int idx_id, int tag, long data)
3266 {
3267     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3268 }
3269 #endif /* !__PCTOOL__ */
3270
3271 long tagcache_get_serial(void)
3272 {
3273     return current_tcmh.serial;
3274 }
3275
3276 long tagcache_get_commitid(void)
3277 {
3278     return current_tcmh.commitid;
3279 }
3280
3281 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3282 {
3283     char buf[512];
3284     int i;
3285
3286     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3287     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3288     {
3289         if (*datastr == '\0')
3290             break;
3291
3292         if (*datastr == '"' || *datastr == '\\')
3293             buf[i++] = '\\';
3294
3295         buf[i] = *(datastr++);
3296     }
3297
3298     strcpy(&buf[i], "\" ");
3299
3300     write(fd, buf, i + 2);
3301
3302     return true;
3303 }
3304
3305 #ifndef __PCTOOL__
3306
3307 static bool read_tag(char *dest, long size,
3308                      const char *src, const char *tagstr)
3309 {
3310     int pos;
3311     char current_tag[32];
3312
3313     while (*src != '\0')
3314     {
3315         /* Skip all whitespace */
3316         while (*src == ' ')
3317             src++;
3318
3319         if (*src == '\0')
3320             break;
3321
3322         pos = 0;
3323         /* Read in tag name */
3324         while (*src != '=' && *src != ' ')
3325         {
3326             current_tag[pos] = *src;
3327             src++;
3328             pos++;
3329
3330             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3331                 return false;
3332         }
3333         current_tag[pos] = '\0';
3334
3335         /* Read in tag data */
3336
3337         /* Find the start. */
3338         while (*src != '"' && *src != '\0')
3339             src++;
3340
3341         if (*src == '\0' || *(++src) == '\0')
3342             return false;
3343
3344         /* Read the data. */
3345         for (pos = 0; pos < size; pos++)
3346         {
3347             if (*src == '\0')
3348                 break;
3349
3350             if (*src == '\\')
3351             {
3352                 dest[pos] = *(src+1);
3353                 src += 2;
3354                 continue;
3355             }
3356
3357             dest[pos] = *src;
3358
3359             if (*src == '"')
3360             {
3361                 src++;
3362                 break;
3363             }
3364
3365             if (*src == '\0')
3366                 break;
3367
3368             src++;
3369         }
3370         dest[pos] = '\0';
3371
3372         if (!strcasecmp(tagstr, current_tag))
3373             return true;
3374     }
3375
3376     return false;
3377 }
3378
3379 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3380 {
3381     struct index_entry idx;
3382     char tag_data[TAG_MAXLEN+32];
3383     int idx_id;
3384     long masterfd = (long)parameters;
3385     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3386         tag_commitid };
3387     int i;
3388     (void)line_n;
3389
3390     if (*buf == '#')
3391         return 0;
3392
3393     logf("%d/%s", line_n, buf);
3394     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3395     {
3396         logf("filename missing");
3397         logf("-> %s", buf);
3398         return 0;
3399     }
3400
3401     idx_id = find_index(tag_data);
3402     if (idx_id < 0)
3403     {
3404         logf("entry not found");
3405         return 0;
3406     }
3407
3408     if (!get_index(masterfd, idx_id, &idx, false))
3409     {
3410         logf("failed to retrieve index entry");
3411         return 0;
3412     }
3413
3414     /* Stop if tag has already been modified. */
3415     if (idx.flag & FLAG_DIRTYNUM)
3416         return 0;
3417
3418     logf("import: %s", tag_data);
3419
3420     idx.flag |= FLAG_DIRTYNUM;
3421     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3422     {
3423         int data;
3424
3425         if (!read_tag(tag_data, sizeof tag_data, buf,
3426                       tagcache_tag_to_str(import_tags[i])))
3427         {
3428             continue;
3429         }
3430
3431         data = atoi(tag_data);
3432         if (data < 0)
3433             continue;
3434
3435         idx.tag_seek[import_tags[i]] = data;
3436
3437         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3438             current_tcmh.serial = data + 1;
3439         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3440             current_tcmh.commitid = data + 1;
3441     }
3442
3443     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3444 }
3445
3446 bool tagcache_import_changelog(void)
3447 {
3448     struct master_header myhdr;
3449     struct tagcache_header tch;
3450     int clfd;
3451     long masterfd;
3452     char buf[2048];
3453
3454     if (!tc_stat.ready)
3455         return false;
3456
3457     while (read_lock)
3458         sleep(1);
3459
3460     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3461     if (clfd < 0)
3462     {
3463         logf("failure to open changelog");
3464         return false;
3465     }
3466
3467     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3468     {
3469         close(clfd);
3470         return false;
3471     }
3472
3473     write_lock++;
3474
3475     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3476
3477     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3478                   parse_changelog_line);
3479
3480     close(clfd);
3481     close(masterfd);
3482
3483     if (filenametag_fd >= 0)
3484     {
3485         close(filenametag_fd);
3486         filenametag_fd = -1;
3487     }
3488
3489     write_lock--;
3490
3491     update_master_header();
3492
3493     return true;
3494 }
3495
3496 #endif /* !__PCTOOL__ */
3497
3498 bool tagcache_create_changelog(struct tagcache_search *tcs)
3499 {
3500     struct master_header myhdr;
3501     struct index_entry idx;
3502     char buf[TAG_MAXLEN+32];
3503     char temp[32];
3504     int clfd;
3505     int i, j;
3506
3507     if (!tc_stat.ready)
3508         return false;
3509
3510     if (!tagcache_search(tcs, tag_filename))
3511         return false;
3512
3513     /* Initialize the changelog */
3514     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3515     if (clfd < 0)
3516     {
3517         logf("failure to open changelog");
3518         return false;
3519     }
3520
3521     if (tcs->masterfd < 0)
3522     {
3523         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3524             return false;
3525     }
3526     else
3527     {
3528         lseek(tcs->masterfd, 0, SEEK_SET);
3529         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3530     }
3531
3532     write(clfd, "## Changelog version 1\n", 23);
3533
3534     for (i = 0; i < myhdr.tch.entry_count; i++)
3535     {
3536         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3537         {
3538             logf("read error #9");
3539             tagcache_search_finish(tcs);
3540             close(clfd);
3541             return false;
3542         }
3543
3544         /* Skip until the entry found has been modified. */
3545         if (! (idx.flag & FLAG_DIRTYNUM) )
3546             continue;
3547
3548         /* Skip deleted entries too. */
3549         if (idx.flag & FLAG_DELETED)
3550             continue;
3551
3552         /* Now retrieve all tags. */
3553         for (j = 0; j < TAG_COUNT; j++)
3554         {
3555             if (TAGCACHE_IS_NUMERIC(j))
3556             {
3557                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3558                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3559                 continue;
3560             }
3561
3562             tcs->type = j;
3563             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3564             write_tag(clfd, tagcache_tag_to_str(j), buf);
3565         }
3566
3567         write(clfd, "\n", 1);
3568         do_timed_yield();
3569     }
3570
3571     close(clfd);
3572
3573     tagcache_search_finish(tcs);
3574
3575     return true;
3576 }
3577
3578 static bool delete_entry(long idx_id)
3579 {
3580     int fd = -1;
3581     int masterfd = -1;
3582     int tag, i;
3583     struct index_entry idx, myidx;
3584     struct master_header myhdr;
3585     char buf[TAG_MAXLEN+32];
3586     int in_use[TAG_COUNT];
3587
3588     logf("delete_entry(): %ld", idx_id);
3589
3590 #ifdef HAVE_TC_RAMCACHE
3591     /* At first mark the entry removed from ram cache. */
3592     if (tc_stat.ramcache)
3593         hdr->indices[idx_id].flag |= FLAG_DELETED;
3594 #endif
3595
3596     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3597         return false;
3598
3599     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3600     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3601     {
3602         logf("delete_entry(): read error");
3603         goto cleanup;
3604     }
3605
3606     if (myidx.flag & FLAG_DELETED)
3607     {
3608         logf("delete_entry(): already deleted!");
3609         goto cleanup;
3610     }
3611
3612     myidx.flag |= FLAG_DELETED;
3613     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3614     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3615     {
3616         logf("delete_entry(): write_error #1");
3617         goto cleanup;
3618     }
3619
3620     /* Now check which tags are no longer in use (if any) */
3621     for (tag = 0; tag < TAG_COUNT; tag++)
3622         in_use[tag] = 0;
3623
3624     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3625     for (i = 0; i < myhdr.tch.entry_count; i++)
3626     {
3627         struct index_entry *idxp;
3628
3629 #ifdef HAVE_TC_RAMCACHE
3630         /* Use RAM DB if available for greater speed */
3631         if (tc_stat.ramcache)
3632             idxp = &hdr->indices[i];
3633         else
3634 #endif
3635         {
3636             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3637             {
3638                 logf("delete_entry(): read error #2");
3639                 goto cleanup;
3640             }
3641             idxp = &idx;
3642         }
3643
3644         if (idxp->flag & FLAG_DELETED)
3645             continue;
3646
3647         for (tag = 0; tag < TAG_COUNT; tag++)
3648         {
3649             if (TAGCACHE_IS_NUMERIC(tag))
3650                 continue;
3651
3652             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3653                 in_use[tag]++;
3654         }
3655     }
3656
3657     /* Now delete all tags no longer in use. */
3658     for (tag = 0; tag < TAG_COUNT; tag++)
3659     {
3660         struct tagcache_header tch;
3661         int oldseek = myidx.tag_seek[tag];
3662
3663         if (TAGCACHE_IS_NUMERIC(tag))
3664             continue;
3665
3666         /**
3667          * Replace tag seek with a hash value of the field string data.
3668          * That way runtime statistics of moved or altered files can be
3669          * resurrected.
3670          */
3671 #ifdef HAVE_TC_RAMCACHE
3672         if (tc_stat.ramcache && tag != tag_filename)
3673         {
3674             struct tagfile_entry *tfe;
3675             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3676
3677             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3678             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3679             myidx.tag_seek[tag] = *seek;
3680         }
3681         else
3682 #endif
3683         {
3684             struct tagfile_entry tfe;
3685
3686             /* Open the index file, which contains the tag names. */
3687             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3688                 goto cleanup;
3689
3690             /* Skip the header block */
3691             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3692             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3693             {
3694                 logf("delete_entry(): read error #3");
3695                 goto cleanup;
3696             }
3697
3698             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3699             {
3700                 logf("delete_entry(): read error #4");
3701                 goto cleanup;
3702             }
3703
3704             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3705         }
3706
3707         if (in_use[tag])
3708         {
3709             logf("in use: %d/%d", tag, in_use[tag]);
3710             if (fd >= 0)
3711             {
3712                 close(fd);
3713                 fd = -1;
3714             }
3715             continue;
3716         }
3717
3718 #ifdef HAVE_TC_RAMCACHE
3719         /* Delete from ram. */
3720         if (tc_stat.ramcache && tag != tag_filename)
3721         {
3722             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3723             tagentry->tag_data[0] = '\0';
3724         }
3725 #endif
3726
3727         /* Open the index file, which contains the tag names. */
3728         if (fd < 0)
3729         {
3730             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3731                 goto cleanup;
3732         }
3733
3734         /* Skip the header block */
3735         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3736
3737         /* Debug, print 10 first characters of the tag
3738         read(fd, buf, 10);
3739         buf[10]='\0';
3740         logf("TAG:%s", buf);
3741         lseek(fd, -10, SEEK_CUR);
3742         */
3743
3744         /* Write first data byte in tag as \0 */
3745         write(fd, "", 1);
3746
3747         /* Now tag data has been removed */
3748         close(fd);
3749         fd = -1;
3750     }
3751
3752     /* Write index entry back into master index. */
3753     lseek(masterfd, sizeof(struct master_header) +
3754           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3755     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3756     {
3757         logf("delete_entry(): write_error #2");
3758         goto cleanup;
3759     }
3760
3761     close(masterfd);
3762
3763     return true;
3764
3765     cleanup:
3766     if (fd >= 0)
3767         close(fd);
3768     if (masterfd >= 0)
3769         close(masterfd);
3770
3771     return false;
3772 }
3773
3774 #ifndef __PCTOOL__
3775 /**
3776  * Returns true if there is an event waiting in the queue
3777  * that requires the current operation to be aborted.
3778  */
3779 static bool check_event_queue(void)
3780 {
3781     struct queue_event ev;
3782
3783     if(!queue_peek(&tagcache_queue, &ev))
3784         return false;
3785
3786     switch (ev.id)
3787     {
3788         case Q_STOP_SCAN:
3789         case SYS_POWEROFF:
3790         case SYS_USB_CONNECTED:
3791             return true;
3792     }
3793
3794     return false;
3795 }
3796 #endif
3797
3798 #ifdef HAVE_TC_RAMCACHE
3799 static bool allocate_tagcache(void)
3800 {
3801     struct master_header tcmh;
3802     int fd;
3803
3804     /* Load the header. */
3805     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3806     {
3807         hdr = NULL;
3808         return false;
3809     }
3810
3811     close(fd);
3812
3813     /**
3814      * Now calculate the required cache size plus
3815      * some extra space for alignment fixes.
3816      */
3817     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3818         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3819     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3820     memset(hdr, 0, sizeof(struct ramcache_header));
3821     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3822     hdr->indices = (struct index_entry *)(hdr + 1);
3823     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3824
3825     return true;
3826 }
3827
3828 # ifdef HAVE_EEPROM_SETTINGS
3829 static bool tagcache_dumpload(void)
3830 {
3831     struct statefile_header shdr;
3832     int fd, rc;
3833     long offpos;
3834     int i;
3835
3836     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3837     if (fd < 0)
3838     {
3839         logf("no tagcache statedump");
3840         return false;
3841     }
3842
3843     /* Check the statefile memory placement */
3844     hdr = buffer_alloc(0);
3845     rc = read(fd, &shdr, sizeof(struct statefile_header));
3846     if (rc != sizeof(struct statefile_header)
3847         /* || (long)hdr != (long)shdr.hdr */)
3848     {
3849         logf("incorrect statefile");
3850         hdr = NULL;
3851         close(fd);
3852         return false;
3853     }
3854
3855     offpos = (long)hdr - (long)shdr.hdr;
3856
3857     /* Lets allocate real memory and load it */
3858     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3859     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3860     close(fd);
3861
3862     if (rc != shdr.tc_stat.ramcache_allocated)
3863     {
3864         logf("read failure!");
3865         hdr = NULL;
3866         return false;
3867     }
3868
3869     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3870
3871     /* Now fix the pointers */
3872     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3873     for (i = 0; i < TAG_COUNT; i++)
3874         hdr->tags[i] += offpos;
3875
3876     return true;
3877 }
3878
3879 static bool tagcache_dumpsave(void)
3880 {
3881     struct statefile_header shdr;
3882     int fd;
3883
3884     if (!tc_stat.ramcache)
3885         return false;
3886
3887     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3888     if (fd < 0)
3889     {
3890         logf("failed to create a statedump");
3891         return false;
3892     }
3893
3894     /* Create the header */
3895     shdr.hdr = hdr;
3896     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3897     write(fd, &shdr, sizeof(struct statefile_header));
3898
3899     /* And dump the data too */
3900     write(fd, hdr, tc_stat.ramcache_allocated);
3901     close(fd);
3902
3903     return true;
3904 }
3905 # endif
3906
3907 static bool load_tagcache(void)
3908 {
3909     struct tagcache_header *tch;
3910     long bytesleft = tc_stat.ramcache_allocated;
3911     struct index_entry *idx;
3912     int rc, fd;
3913     char *p;
3914     int i, tag;
3915
3916 # ifdef HAVE_DIRCACHE
3917     while (dircache_is_initializing())
3918         sleep(1);
3919
3920     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3921 # endif
3922
3923     logf("loading tagcache to ram...");
3924
3925     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3926     if (fd < 0)
3927     {
3928         logf("tagcache open failed");
3929         return false;
3930     }
3931
3932     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3933         != sizeof(struct master_header)
3934         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3935     {
3936         logf("incorrect header");
3937         return false;
3938     }
3939
3940     idx = hdr->indices;
3941
3942     /* Load the master index table. */
3943     for (i = 0; i < hdr->h.tch.entry_count; i++)
3944     {
3945         rc = ecread_index_entry(fd, idx);
3946         if (rc != sizeof(struct index_entry))
3947         {
3948             logf("read error #10");
3949             close(fd);
3950             return false;
3951         }
3952
3953         bytesleft -= sizeof(struct index_entry);
3954         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3955         {
3956             logf("too big tagcache.");
3957             close(fd);
3958             return false;
3959         }
3960
3961         idx++;
3962     }
3963
3964     close(fd);
3965
3966     /* Load the tags. */
3967     p = (char *)idx;
3968     for (tag = 0; tag < TAG_COUNT; tag++)
3969     {
3970         struct tagfile_entry *fe;
3971         char buf[TAG_MAXLEN+32];
3972
3973         if (TAGCACHE_IS_NUMERIC(tag))
3974             continue ;
3975
3976         //p = ((void *)p+1);
3977         p = (char *)((long)p & ~0x03) + 0x04;
3978         hdr->tags[tag] = p;
3979
3980         /* Check the header. */
3981         tch = (struct tagcache_header *)p;
3982         p += sizeof(struct tagcache_header);
3983
3984         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3985             return false;
3986
3987         for (hdr->entry_count[tag] = 0;
3988              hdr->entry_count[tag] < tch->entry_count;
3989              hdr->entry_count[tag]++)
3990         {
3991             long pos;
3992
3993             if (do_timed_yield())
3994             {
3995                 /* Abort if we got a critical event in queue */
3996                 if (check_event_queue())
3997                     return false;
3998             }
3999
4000             fe = (struct tagfile_entry *)p;
4001             pos = lseek(fd, 0, SEEK_CUR);
4002             rc = ecread_tagfile_entry(fd, fe);
4003             if (rc != sizeof(struct tagfile_entry))
4004             {
4005                 /* End of lookup table. */
4006                 logf("read error #11");
4007                 close(fd);
4008                 return false;
4009             }
4010
4011             /* We have a special handling for the filename tags. */
4012             if (tag == tag_filename)
4013             {
4014 # ifdef HAVE_DIRCACHE
4015                 const struct dircache_entry *dc;
4016 # endif
4017
4018                 // FIXME: This is wrong!
4019                 // idx = &hdr->indices[hdr->entry_count[i]];
4020                 idx = &hdr->indices[fe->idx_id];
4021
4022                 if (fe->tag_length >= (long)sizeof(buf)-1)
4023                 {
4024                     read(fd, buf, 10);
4025                     buf[10] = '\0';
4026                     logf("TAG:%s", buf);
4027                     logf("too long filename");
4028                     close(fd);
4029                     return false;
4030                 }
4031
4032                 rc = read(fd, buf, fe->tag_length);
4033                 if (rc != fe->tag_length)
4034                 {
4035                     logf("read error #12");
4036                     close(fd);
4037                     return false;
4038                 }
4039
4040                 /* Check if the entry has already been removed */
4041                 if (idx->flag & FLAG_DELETED)
4042                     continue;
4043
4044                 /* This flag must not be used yet. */
4045                 if (idx->flag & FLAG_DIRCACHE)
4046                 {
4047                     logf("internal error!");
4048                     close(fd);
4049                     return false;
4050                 }
4051
4052                 if (idx->tag_seek[tag] != pos)
4053                 {
4054                     logf("corrupt data structures!");
4055                     close(fd);
4056                     return false;
4057                 }
4058
4059 # ifdef HAVE_DIRCACHE
4060                 if (dircache_is_enabled())
4061                 {
4062                     dc = dircache_get_entry_ptr(buf);
4063                     if (dc == NULL)
4064                     {
4065                         logf("Entry no longer valid.");
4066                         logf("-> %s", buf);
4067                         if (global_settings.tagcache_autoupdate)
4068                             delete_entry(fe->idx_id);
4069                         continue ;
4070                     }
4071
4072                     idx->flag |= FLAG_DIRCACHE;
4073                     idx->tag_seek[tag_filename] = (long)dc;
4074                 }
4075                 else
4076 # endif
4077                 {
4078                     /* This will be very slow unless dircache is enabled
4079                        or target is flash based, but do it anyway for
4080                        consistency. */
4081                     /* Check if entry has been removed. */
4082                     if (global_settings.tagcache_autoupdate)
4083                     {
4084                         if (!file_exists(buf))
4085                         {
4086                             logf("Entry no longer valid.");
4087                             logf("-> %s", buf);
4088                             delete_entry(fe->idx_id);
4089                             continue;
4090                         }
4091                     }
4092                 }
4093
4094                 continue ;
4095             }
4096
4097             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4098             if (bytesleft < 0)
4099             {
4100                 logf("too big tagcache #2");
4101                 logf("tl: %ld", fe->tag_length);
4102                 logf("bl: %ld", bytesleft);
4103                 close(fd);
4104                 return false;
4105             }
4106
4107             p = fe->tag_data;
4108             rc = read(fd, fe->tag_data, fe->tag_length);
4109             p += rc;
4110
4111             if (rc != fe->tag_length)
4112             {
4113                 logf("read error #13");
4114                 logf("rc=0x%04x", rc); // 0x431
4115                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4116                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4117                 logf("tag=0x%02x", tag); // 0x00
4118                 close(fd);
4119                 return false;
4120             }
4121         }
4122         close(fd);
4123     }
4124
4125     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4126     logf("tagcache loaded into ram!");
4127
4128     return true;
4129 }
4130 #endif /* HAVE_TC_RAMCACHE */
4131
4132 static bool check_deleted_files(void)
4133 {
4134     int fd;
4135     char buf[TAG_MAXLEN+32];
4136     struct tagfile_entry tfe;
4137
4138     logf("reverse scan...");
4139     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4140     fd = open(buf, O_RDONLY);
4141
4142     if (fd < 0)
4143     {
4144         logf("%s open fail", buf);
4145         return false;
4146     }
4147
4148     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4149     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4150 #ifndef __PCTOOL__
4151            && !check_event_queue()
4152 #endif
4153            )
4154     {
4155         if (tfe.tag_length >= (long)sizeof(buf)-1)
4156         {
4157             logf("too long tag");
4158             close(fd);
4159             return false;
4160         }
4161
4162         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4163         {
4164             logf("read error #14");
4165             close(fd);
4166             return false;
4167         }
4168
4169         /* Check if the file has already deleted from the db. */
4170         if (*buf == '\0')
4171             continue;
4172
4173         /* Now check if the file exists. */
4174         if (!file_exists(buf))
4175         {
4176             logf("Entry no longer valid.");
4177             logf("-> %s / %ld", buf, tfe.tag_length);
4178             delete_entry(tfe.idx_id);
4179         }
4180     }
4181
4182     close(fd);
4183
4184     logf("done");
4185
4186     return true;
4187 }
4188
4189
4190 /* Note that this function must not be inlined, otherwise the whole point
4191  * of having the code in a separate function is lost.
4192  */
4193 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4194     int *ignore, int *unignore)
4195 {
4196     char newpath[MAX_PATH];
4197
4198     /* check for a database.ignore file */
4199     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4200     *ignore = file_exists(newpath);
4201     /* check for a database.unignore file */
4202     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4203     *unignore = file_exists(newpath);
4204 }
4205
4206 static struct search_roots_ll {
4207     const char *path;
4208     struct search_roots_ll * next;
4209 } roots_ll;
4210
4211 #ifdef APPLICATION
4212 /*
4213  * This adds a path to the search roots, possibly during traveling through
4214  * the filesystem. It only adds if the path is not inside an already existing
4215  * search root.
4216  *
4217  * Returns true if it added the path to the search roots
4218  *
4219  * Windows 2000 and greater supports symlinks, but they don't provide
4220  * realpath() or readlink(), and symlinks are rarely used on them so
4221  * ignore this for windows for now
4222  **/
4223 static bool add_search_root(const char *name)
4224 {
4225     (void)name;
4226 #ifndef WIN32
4227     struct search_roots_ll *this, *prev = NULL;
4228     char target[MAX_PATH];
4229     char _abs_target[MAX_PATH];
4230     char * abs_target;
4231     ssize_t len;
4232
4233     len = readlink(name, target, sizeof(target));
4234     if (len < 0)
4235         return false;
4236
4237     target[len] = '\0';
4238     /* realpath(target, NULL) doesn't work on android ... */
4239     abs_target = realpath(target, _abs_target);
4240     if (abs_target == NULL)
4241         return false;
4242
4243     for(this = &roots_ll; this; prev = this, this = this->next)
4244     {
4245         size_t root_len = strlen(this->path);
4246         /* check if the link target is inside of an existing search root
4247          * don't add if target is inside, we'll scan it later */
4248         if (!strncmp(this->path, abs_target, root_len))
4249             return false;
4250     }
4251
4252     if (prev)
4253     {
4254         size_t len = strlen(abs_target) + 1; /* count \0 */
4255         this = malloc(sizeof(struct search_roots_ll) + len );
4256         if (!this || len > MAX_PATH)
4257         {
4258             logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4259             free(this);
4260             prev->next = NULL;
4261         }
4262         this->path = ((char*)this) + sizeof(struct search_roots_ll);
4263         strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4264         this->next = NULL;
4265         prev->next = this;
4266         logf("Added %s to the search roots\n", abs_target);
4267         return true;
4268     }
4269 #endif
4270     return false;
4271 }
4272
4273 static int free_search_roots(struct search_roots_ll * start)
4274 {
4275     int ret = 0;
4276     if (start->next)
4277     {
4278         ret += free_search_roots(start->next);
4279         ret += sizeof(struct search_roots_ll);
4280         free(start->next);
4281     }
4282     return ret;
4283 }
4284 #else /* native, simulator */
4285 #define add_search_root(a) do {} while(0)
4286 #define free_search_roots(a) do {} while(0)
4287 #endif
4288
4289 static bool check_dir(const char *dirname, int add_files)
4290 {
4291     DIR *dir;
4292     int len;
4293     int success = false;
4294     int ignore, unignore;
4295
4296     dir = opendir(dirname);
4297     if (!dir)
4298     {
4299         logf("tagcache: opendir(%s) failed", dirname);
4300         return false;
4301     }
4302     /* check for a database.ignore and database.unignore */
4303     check_ignore(dirname, &ignore, &unignore);
4304
4305     /* don't do anything if both ignore and unignore are there */
4306     if (ignore != unignore)
4307         add_files = unignore;
4308
4309     /* Recursively scan the dir. */
4310 #ifdef __PCTOOL__
4311     while (1)
4312 #else
4313     while (!check_event_queue())
4314 #endif
4315     {
4316         struct dirent *entry = readdir(dir);
4317         if (entry == NULL)
4318         {
4319             success = true;
4320             break;
4321         }
4322
4323         if (!strcmp((char *)entry->d_name, ".") ||
4324             !strcmp((char *)entry->d_name, ".."))
4325             continue;
4326
4327         struct dirinfo info = dir_get_info(dir, entry);
4328
4329         yield();
4330
4331         len = strlen(curpath);
4332         /* don't add an extra / for curpath == / */
4333         if (len <= 1) len = 0;
4334         snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4335
4336         processed_dir_count++;
4337         if (info.attribute & ATTR_DIRECTORY)
4338         {   /* don't follow symlinks to dirs, but try to add it as a search root
4339              * this makes able to avoid looping in recursive symlinks */
4340             if (info.attribute & ATTR_LINK)
4341                 add_search_root(curpath);
4342             else
4343                 check_dir(curpath, add_files);
4344         }
4345         else if (add_files)
4346         {
4347             tc_stat.curentry = curpath;
4348
4349             /* Add a new entry to the temporary db file. */
4350             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4351 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4352                          , dir->internal_entry
4353 #endif
4354                          );
4355
4356             /* Wait until current path for debug screen is read and unset. */
4357             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4358                 yield();
4359
4360             tc_stat.curentry = NULL;
4361         }
4362
4363         curpath[len] = '\0';
4364     }
4365
4366     closedir(dir);
4367
4368     return success;
4369 }
4370
4371 void tagcache_screensync_event(void)
4372 {
4373     tc_stat.curentry = NULL;
4374 }
4375
4376 void tagcache_screensync_enable(bool state)
4377 {
4378     tc_stat.syncscreen = state;
4379 }
4380
4381 void tagcache_build(const char *path)
4382 {
4383     struct tagcache_header header;
4384     bool ret;
4385
4386     curpath[0] = '\0';
4387     data_size = 0;
4388     total_entry_count = 0;
4389     processed_dir_count = 0;
4390
4391 #ifdef HAVE_DIRCACHE
4392     while (dircache_is_initializing())
4393         sleep(1);
4394 #endif
4395
4396     logf("updating tagcache");
4397
4398     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4399     if (cachefd >= 0)
4400     {
4401         logf("skipping, cache already waiting for commit");
4402         close(cachefd);
4403         return ;
4404     }
4405
4406     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4407     if (cachefd < 0)
4408     {
4409         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4410         return ;
4411     }
4412
4413     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4414
4415     cpu_boost(true);
4416
4417     logf("Scanning files...");
4418     /* Scan for new files. */
4419     memset(&header, 0, sizeof(struct tagcache_header));
4420     write(cachefd, &header, sizeof(struct tagcache_header));
4421
4422     ret = true;
4423     roots_ll.path = path;
4424     roots_ll.next = NULL;
4425     struct search_roots_ll * this;
4426     /* check_dir might add new roots */
4427     for(this = &roots_ll; this; this = this->next)
4428     {
4429         strcpy(curpath, this->path);
4430         ret = ret && check_dir(this->path, true);
4431     }
4432     if (roots_ll.next)
4433         free_search_roots(roots_ll.next);
4434
4435     /* Write the header. */
4436     header.magic = TAGCACHE_MAGIC;
4437     header.datasize = data_size;
4438     header.entry_count = total_entry_count;
4439     lseek(cachefd, 0, SEEK_SET);
4440     write(cachefd, &header, sizeof(struct tagcache_header));
4441     close(cachefd);
4442
4443     if (filenametag_fd >= 0)
4444     {
4445         close(filenametag_fd);
4446         filenametag_fd = -1;
4447     }
4448
4449     if (!ret)
4450     {
4451         logf("Aborted.");
4452         cpu_boost(false);
4453         return ;
4454     }
4455
4456     /* Commit changes to the database. */
4457 #ifdef __PCTOOL__
4458     allocate_tempbuf();
4459 #endif
4460     if (commit())
4461     {
4462         remove(TAGCACHE_FILE_TEMP);
4463         logf("tagcache built!");
4464     }
4465 #ifdef __PCTOOL__
4466     free_tempbuf();
4467 #endif
4468
4469 #ifdef HAVE_TC_RAMCACHE
4470     if (hdr)
4471     {
4472         /* Import runtime statistics if we just initialized the db. */
4473         if (hdr->h.serial == 0)
4474             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4475     }
4476 #endif
4477
4478     cpu_boost(false);
4479 }
4480
4481 #ifdef HAVE_TC_RAMCACHE
4482 static void load_ramcache(void)
4483 {
4484     if (!hdr)
4485         return ;
4486
4487     cpu_boost(true);
4488
4489     /* At first we should load the cache (if exists). */
4490     tc_stat.ramcache = load_tagcache();
4491
4492     if (!tc_stat.ramcache)
4493     {
4494         /* If loading failed, it must indicate some problem with the db
4495          * so disable it entirely to prevent further issues. */
4496         tc_stat.ready = false;
4497         hdr = NULL;
4498     }
4499
4500     cpu_boost(false);
4501 }
4502
4503 void tagcache_unload_ramcache(void)
4504 {
4505     tc_stat.ramcache = false;
4506     /* Just to make sure there is no statefile present. */
4507     // remove(TAGCACHE_STATEFILE);
4508 }
4509 #endif
4510
4511 #ifndef __PCTOOL__
4512 static void tagcache_thread(void)
4513 {
4514     struct queue_event ev;
4515     bool check_done = false;
4516
4517     /* If the previous cache build/update was interrupted, commit
4518      * the changes first in foreground. */
4519     cpu_boost(true);
4520     allocate_tempbuf();
4521     commit();
4522     free_tempbuf();
4523
4524 #ifdef HAVE_TC_RAMCACHE
4525 # ifdef HAVE_EEPROM_SETTINGS
4526     if (firmware_settings.initialized && firmware_settings.disk_clean
4527         && global_settings.tagcache_ram)
4528     {
4529         check_done = tagcache_dumpload();
4530     }
4531
4532     remove(TAGCACHE_STATEFILE);
4533 # endif
4534
4535     /* Allocate space for the tagcache if found on disk. */
4536     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4537         allocate_tagcache();
4538 #endif
4539
4540     cpu_boost(false);
4541     tc_stat.initialized = true;
4542
4543     /* Don't delay bootup with the header check but do it on background. */
4544     if (!tc_stat.ready)
4545     {
4546         sleep(HZ);
4547         tc_stat.ready = check_all_headers();
4548         tc_stat.readyvalid = true;
4549     }
4550
4551     while (1)
4552     {
4553         run_command_queue(false);
4554
4555         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4556
4557         switch (ev.id)
4558         {
4559             case Q_IMPORT_CHANGELOG:
4560                 tagcache_import_changelog();
4561                 break;
4562
4563             case Q_REBUILD:
4564                 remove_files();
4565                 remove(TAGCACHE_FILE_TEMP);
4566                 tagcache_build("/");
4567                 break;
4568
4569             case Q_UPDATE:
4570                 tagcache_build("/");
4571 #ifdef HAVE_TC_RAMCACHE
4572                 load_ramcache();
4573 #endif
4574                 check_deleted_files();
4575                 break ;
4576
4577             case Q_START_SCAN:
4578                 check_done = false;
4579             case SYS_TIMEOUT:
4580                 if (check_done || !tc_stat.ready)
4581                     break ;
4582
4583 #ifdef HAVE_TC_RAMCACHE
4584                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4585                 {
4586                     load_ramcache();
4587                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4588                         tagcache_build("/");
4589                 }
4590                 else
4591 #endif
4592                 if (global_settings.tagcache_autoupdate)
4593                 {
4594                     tagcache_build("/");
4595
4596                     /* This will be very slow unless dircache is enabled
4597                        or target is flash based, but do it anyway for
4598                        consistency. */
4599                     check_deleted_files();
4600                 }
4601
4602                 logf("tagcache check done");
4603
4604                 check_done = true;
4605                 break ;
4606
4607             case Q_STOP_SCAN:
4608                 break ;
4609
4610             case SYS_POWEROFF:
4611                 break ;
4612
4613 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4614             case SYS_USB_CONNECTED:
4615                 logf("USB: TagCache");
4616                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4617                 usb_wait_for_disconnect(&tagcache_queue);
4618                 break ;
4619 #endif
4620         }
4621     }
4622 }
4623
4624 bool tagcache_prepare_shutdown(void)
4625 {
4626     if (tagcache_get_commit_step() > 0)
4627         return false;
4628
4629     tagcache_stop_scan();
4630     while (read_lock || write_lock)
4631         sleep(1);
4632
4633     return true;
4634 }
4635
4636 void tagcache_shutdown(void)
4637 {
4638     /* Flush the command queue. */
4639     run_command_queue(true);
4640
4641 #ifdef HAVE_EEPROM_SETTINGS
4642     if (tc_stat.ramcache)
4643         tagcache_dumpsave();
4644 #endif
4645 }
4646
4647 static int get_progress(void)
4648 {
4649     int total_count = -1;
4650
4651 #ifdef HAVE_DIRCACHE
4652     if (dircache_is_enabled())
4653     {
4654         total_count = dircache_get_entry_count();
4655     }
4656     else
4657 #endif
4658 #ifdef HAVE_TC_RAMCACHE
4659     {
4660         if (hdr && tc_stat.ramcache)
4661             total_count = hdr->h.tch.entry_count;
4662     }
4663 #endif
4664
4665     if (total_count < 0)
4666         return -1;
4667
4668     return processed_dir_count * 100 / total_count;
4669 }
4670
4671 struct tagcache_stat* tagcache_get_stat(void)
4672 {
4673     tc_stat.progress = get_progress();
4674     tc_stat.processed_entries = processed_dir_count;
4675
4676     return &tc_stat;
4677 }
4678
4679 void tagcache_start_scan(void)
4680 {
4681     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4682 }
4683
4684 bool tagcache_update(void)
4685 {
4686     if (!tc_stat.ready)
4687         return false;
4688
4689     queue_post(&tagcache_queue, Q_UPDATE, 0);
4690     return false;
4691 }
4692
4693 bool tagcache_rebuild()
4694 {
4695     queue_post(&tagcache_queue, Q_REBUILD, 0);
4696     return false;
4697 }
4698
4699 void tagcache_stop_scan(void)
4700 {
4701     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4702 }
4703
4704 #ifdef HAVE_TC_RAMCACHE
4705 bool tagcache_is_ramcache(void)
4706 {
4707     return tc_stat.ramcache;
4708 }
4709 #endif
4710
4711 #endif /* !__PCTOOL__ */
4712
4713
4714 void tagcache_init(void)
4715 {
4716     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4717     memset(&current_tcmh, 0, sizeof(struct master_header));
4718     filenametag_fd = -1;
4719     write_lock = read_lock = 0;
4720
4721 #ifndef __PCTOOL__
4722     mutex_init(&command_queue_mutex);
4723     queue_init(&tagcache_queue, true);
4724     create_thread(tagcache_thread, tagcache_stack,
4725                   sizeof(tagcache_stack), 0, tagcache_thread_name
4726                   IF_PRIO(, PRIORITY_BACKGROUND)
4727                   IF_COP(, CPU));
4728 #else
4729     tc_stat.initialized = true;
4730     allocate_tempbuf();
4731     commit();
4732     free_tempbuf();
4733     tc_stat.ready = check_all_headers();
4734 #endif
4735 }
4736
4737 #ifdef __PCTOOL__
4738 void tagcache_reverse_scan(void)
4739 {
4740     logf("Checking for deleted files");
4741     check_deleted_files();
4742 }
4743 #endif
4744
4745 bool tagcache_is_initialized(void)
4746 {
4747     return tc_stat.initialized;
4748 }
4749 bool tagcache_is_fully_initialized(void)
4750 {
4751     return tc_stat.readyvalid;
4752 }
4753 bool tagcache_is_usable(void)
4754 {
4755     return tc_stat.initialized && tc_stat.ready;
4756 }
4757 int tagcache_get_commit_step(void)
4758 {
4759     return tc_stat.commit_step;
4760 }
4761 int tagcache_get_max_commit_step(void)
4762 {
4763     return (int)(SORTED_TAGS_COUNT)+1;
4764 }
4765