apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #ifdef APPLICATION
  64 #include <unistd.h> /* readlink() */
  65 #include <limits.h> /* PATH_MAX */
  66 #endif
  67 #include "config.h"
  68 #include "ata_idle_notify.h"
  69 #include "thread.h"
  70 #include "kernel.h"
  71 #include "system.h"
  72 #include "logf.h"
  73 #include "string-extra.h"
  74 #include "usb.h"
  75 #include "metadata.h"
  76 #include "tagcache.h"
  77 #include "buffer.h"
  78 #include "crc32.h"
  79 #include "misc.h"
  80 #include "settings.h"
  81 #include "dir.h"
  82 #include "filefuncs.h"
  83 #include "structec.h"
  84 #include "debug.h"
  85
  86 #ifndef __PCTOOL__
  87 #include "lang.h"
  88 #include "eeprom_settings.h"
  89 #endif
  90
  91 #ifdef __PCTOOL__
  92 #define yield() do { } while(0)
  93 #define sim_sleep(timeout) do { } while(0)
  94 #define do_timed_yield() do { } while(0)
  95 #endif
  96
  97 #ifndef __PCTOOL__
  98 /* Tag Cache thread. */
  99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
 100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
 101 static const char tagcache_thread_name[] = "tagcache";
 102 #endif
 103
 104 /* Previous path when scanning directory tree recursively. */
 105 static char curpath[TAG_MAXLEN+32];
 106
 107 /* Used when removing duplicates. */
 108 static char *tempbuf;     /* Allocated when needed. */
 109 static long tempbufidx;   /* Current location in buffer. */
 110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 111 static long tempbuf_left; /* Buffer space left. */
 112 static long tempbuf_pos;
 113
 114 #define SORTED_TAGS_COUNT 8
 115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 118     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 119 /* Tags we want to get sorted (loaded to the tempbuf). */
 120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 121     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 122     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 123
 124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 126     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 127     (1LU << tag_albumartist) | (1LU << tag_grouping))
 128
 129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 130 static const char *tags_str[] = { "artist", "album", "genre", "title",
 131     "filename", "composer", "comment", "albumartist", "grouping", "year",
 132     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 133     "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
 134
 135 /* Status information of the tagcache. */
 136 static struct tagcache_stat tc_stat;
 137
 138 /* Queue commands. */
 139 enum tagcache_queue {
 140     Q_STOP_SCAN = 0,
 141     Q_START_SCAN,
 142     Q_IMPORT_CHANGELOG,
 143     Q_UPDATE,
 144     Q_REBUILD,
 145
 146     /* Internal tagcache command queue. */
 147     CMD_UPDATE_MASTER_HEADER,
 148     CMD_UPDATE_NUMERIC,
 149 };
 150
 151 struct tagcache_command_entry {
 152     int32_t command;
 153     int32_t idx_id;
 154     int32_t tag;
 155     int32_t data;
 156 };
 157
 158 #ifndef __PCTOOL__
 159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 160 static volatile int command_queue_widx = 0;
 161 static volatile int command_queue_ridx = 0;
 162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
 163 #endif
 164
 165 /* Tag database structures. */
 166
 167 /* Variable-length tag entry in tag files. */
 168 struct tagfile_entry {
 169     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 170     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 171     char tag_data[0];  /* Begin of the tag data */
 172 };
 173
 174 /* Fixed-size tag entry in master db index. */
 175 struct index_entry {
 176     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 177     int32_t flag;                /* Status flags */
 178 };
 179
 180 /* Header is the same in every file. */
 181 struct tagcache_header {
 182     int32_t magic;       /* Header version number */
 183     int32_t datasize;    /* Data size in bytes */
 184     int32_t entry_count; /* Number of entries in this file */
 185 };
 186
 187 struct master_header {
 188     struct tagcache_header tch;
 189     int32_t serial; /* Increasing counting number */
 190     int32_t commitid; /* Number of commits so far */
 191     int32_t dirty;
 192 };
 193
 194 /* For the endianess correction */
 195 static const char *tagfile_entry_ec   = "ll";
 196 /**
 197  Note: This should be (1 + TAG_COUNT) amount of l's.
 198  */
 199 static const char *index_entry_ec     = "llllllllllllllllllllll";
 200
 201 static const char *tagcache_header_ec = "lll";
 202 static const char *master_header_ec   = "llllll";
 203
 204 static struct master_header current_tcmh;
 205
 206 #ifdef HAVE_TC_RAMCACHE
 207 /* Header is created when loading database to ram. */
 208 struct ramcache_header {
 209     struct master_header h;      /* Header from the master index */
 210     struct index_entry *indices; /* Master index file content */
 211     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 212     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 213 };
 214
 215 # ifdef HAVE_EEPROM_SETTINGS
 216 struct statefile_header {
 217     struct ramcache_header *hdr;
 218     struct tagcache_stat tc_stat;
 219 };
 220 # endif
 221
 222 /* Pointer to allocated ramcache_header */
 223 static struct ramcache_header *hdr;
 224 #endif
 225
 226 /**
 227  * Full tag entries stored in a temporary file waiting
 228  * for commit to the cache. */
 229 struct temp_file_entry {
 230     long tag_offset[TAG_COUNT];
 231     short tag_length[TAG_COUNT];
 232     long flag;
 233
 234     long data_length;
 235 };
 236
 237 struct tempbuf_id_list {
 238     long id;
 239     struct tempbuf_id_list *next;
 240 };
 241
 242 struct tempbuf_searchidx {
 243     long idx_id;
 244     char *str;
 245     int seek;
 246     struct tempbuf_id_list idlist;
 247 };
 248
 249 /* Lookup buffer for fixing messed up index while after sorting. */
 250 static long commit_entry_count;
 251 static long lookup_buffer_depth;
 252 static struct tempbuf_searchidx **lookup;
 253
 254 /* Used when building the temporary file. */
 255 static int cachefd = -1, filenametag_fd;
 256 static int total_entry_count = 0;
 257 static int data_size = 0;
 258 static int processed_dir_count;
 259
 260 /* Thread safe locking */
 261 static volatile int write_lock;
 262 static volatile int read_lock;
 263
 264 static bool delete_entry(long idx_id);
 265
 266 const char* tagcache_tag_to_str(int tag)
 267 {
 268     return tags_str[tag];
 269 }
 270
 271 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 272 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 273 {
 274     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 275 }
 276
 277 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 278 {
 279     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 280 }
 281
 282 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 283 {
 284     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 285 }
 286
 287 #ifdef HAVE_DIRCACHE
 288 /**
 289  * Returns true if specified flag is still present, i.e., dircache
 290  * has not been reloaded.
 291  */
 292 static bool is_dircache_intact(void)
 293 {
 294     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 295 }
 296 #endif
 297
 298 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 299 {
 300     int fd;
 301     char buf[MAX_PATH];
 302     int rc;
 303
 304     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 305         return -1;
 306
 307     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 308
 309     fd = open(buf, write ? O_RDWR : O_RDONLY);
 310     if (fd < 0)
 311     {
 312         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 313         tc_stat.ready = false;
 314         return fd;
 315     }
 316
 317     /* Check the header. */
 318     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 319     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 320     {
 321         logf("header error");
 322         tc_stat.ready = false;
 323         close(fd);
 324         return -2;
 325     }
 326
 327     return fd;
 328 }
 329
 330 static int open_master_fd(struct master_header *hdr, bool write)
 331 {
 332     int fd;
 333     int rc;
 334
 335     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 336     if (fd < 0)
 337     {
 338         logf("master file open failed for R/W");
 339         tc_stat.ready = false;
 340         return fd;
 341     }
 342
 343     tc_stat.econ = false;
 344
 345     /* Check the header. */
 346     rc = read(fd, hdr, sizeof(struct master_header));
 347     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 348     {
 349         /* Success. */
 350         return fd;
 351     }
 352
 353     /* Trying to read again, this time with endianess correction enabled. */
 354     lseek(fd, 0, SEEK_SET);
 355
 356     rc = ecread(fd, hdr, 1, master_header_ec, true);
 357     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 358     {
 359         logf("header error");
 360         tc_stat.ready = false;
 361         close(fd);
 362         return -2;
 363     }
 364
 365     tc_stat.econ = true;
 366
 367     return fd;
 368 }
 369
 370 #ifndef __PCTOOL__
 371 static bool do_timed_yield(void)
 372 {
 373     /* Sorting can lock up for quite a while, so yield occasionally */
 374     static long wakeup_tick = 0;
 375     if (TIME_AFTER(current_tick, wakeup_tick))
 376     {
 377         wakeup_tick = current_tick + (HZ/4);
 378         yield();
 379         return true;
 380     }
 381     return false;
 382 }
 383 #endif
 384
 385 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 386 static long find_entry_ram(const char *filename,
 387                            const struct dircache_entry *dc)
 388 {
 389     static long last_pos = 0;
 390     int i;
 391
 392     /* Check if tagcache is loaded into ram. */
 393     if (!tc_stat.ramcache)
 394         return -1;
 395
 396     if (dc == NULL)
 397         dc = dircache_get_entry_ptr(filename);
 398
 399     if (dc == NULL)
 400     {
 401         logf("tagcache: file not found.");
 402         return -1;
 403     }
 404
 405     try_again:
 406
 407     if (last_pos > 0)
 408         i = last_pos;
 409     else
 410         i = 0;
 411
 412     for (; i < hdr->h.tch.entry_count; i++)
 413     {
 414         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 415         {
 416             last_pos = MAX(0, i - 3);
 417             return i;
 418         }
 419
 420         do_timed_yield();
 421     }
 422
 423     if (last_pos > 0)
 424     {
 425         last_pos = 0;
 426         goto try_again;
 427     }
 428
 429     return -1;
 430 }
 431 #endif
 432
 433 static long find_entry_disk(const char *filename_raw, bool localfd)
 434 {
 435     struct tagcache_header tch;
 436     static long last_pos = -1;
 437     long pos_history[POS_HISTORY_COUNT];
 438     long pos_history_idx = 0;
 439     bool found = false;
 440     struct tagfile_entry tfe;
 441     int fd;
 442     char buf[TAG_MAXLEN+32];
 443     int i;
 444     int pos = -1;
 445
 446     const char *filename = filename_raw;
 447 #ifdef APPLICATION
 448     char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
 449     if (realpath(filename, pathbuf) == pathbuf)
 450         filename = pathbuf;
 451 #endif
 452
 453     if (!tc_stat.ready)
 454         return -2;
 455
 456     fd = filenametag_fd;
 457     if (fd < 0 || localfd)
 458     {
 459         last_pos = -1;
 460         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 461             return -1;
 462     }
 463
 464     check_again:
 465
 466     if (last_pos > 0)
 467         lseek(fd, last_pos, SEEK_SET);
 468     else
 469         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 470
 471     while (true)
 472     {
 473         pos = lseek(fd, 0, SEEK_CUR);
 474         for (i = pos_history_idx-1; i >= 0; i--)
 475             pos_history[i+1] = pos_history[i];
 476         pos_history[0] = pos;
 477
 478         if (ecread_tagfile_entry(fd, &tfe)
 479             != sizeof(struct tagfile_entry))
 480         {
 481             break ;
 482         }
 483
 484         if (tfe.tag_length >= (long)sizeof(buf))
 485         {
 486             logf("too long tag #1");
 487             close(fd);
 488             if (!localfd)
 489                 filenametag_fd = -1;
 490             last_pos = -1;
 491             return -2;
 492         }
 493
 494         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 495         {
 496             logf("read error #2");
 497             close(fd);
 498             if (!localfd)
 499                 filenametag_fd = -1;
 500             last_pos = -1;
 501             return -3;
 502         }
 503
 504         if (!strcasecmp(filename, buf))
 505         {
 506             last_pos = pos_history[pos_history_idx];
 507             found = true;
 508             break ;
 509         }
 510
 511         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 512             pos_history_idx++;
 513     }
 514
 515     /* Not found? */
 516     if (!found)
 517     {
 518         if (last_pos > 0)
 519         {
 520             last_pos = -1;
 521             logf("seek again");
 522             goto check_again;
 523         }
 524
 525         if (fd != filenametag_fd || localfd)
 526             close(fd);
 527         return -4;
 528     }
 529
 530     if (fd != filenametag_fd || localfd)
 531         close(fd);
 532
 533     return tfe.idx_id;
 534 }
 535
 536 static int find_index(const char *filename)
 537 {
 538     long idx_id = -1;
 539
 540 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 541     if (tc_stat.ramcache && is_dircache_intact())
 542         idx_id = find_entry_ram(filename, NULL);
 543 #endif
 544
 545     if (idx_id < 0)
 546         idx_id = find_entry_disk(filename, true);
 547
 548     return idx_id;
 549 }
 550
 551 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 552 {
 553     int idx_id;
 554
 555     if (!tc_stat.ready)
 556         return false;
 557
 558     idx_id = find_index(filename);
 559     if (idx_id < 0)
 560         return false;
 561
 562     if (!tagcache_search(tcs, tag_filename))
 563         return false;
 564
 565     tcs->entry_count = 0;
 566     tcs->idx_id = idx_id;
 567
 568     return true;
 569 }
 570
 571 static bool get_index(int masterfd, int idxid,
 572                       struct index_entry *idx, bool use_ram)
 573 {
 574     bool localfd = false;
 575
 576     if (idxid < 0)
 577     {
 578         logf("Incorrect idxid: %d", idxid);
 579         return false;
 580     }
 581
 582 #ifdef HAVE_TC_RAMCACHE
 583     if (tc_stat.ramcache && use_ram)
 584     {
 585         if (hdr->indices[idxid].flag & FLAG_DELETED)
 586             return false;
 587
 588 # ifdef HAVE_DIRCACHE
 589         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 590             || is_dircache_intact())
 591 #endif
 592         {
 593             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 594             return true;
 595         }
 596     }
 597 #else
 598     (void)use_ram;
 599 #endif
 600
 601     if (masterfd < 0)
 602     {
 603         struct master_header tcmh;
 604
 605         localfd = true;
 606         masterfd = open_master_fd(&tcmh, false);
 607         if (masterfd < 0)
 608             return false;
 609     }
 610
 611     lseek(masterfd, idxid * sizeof(struct index_entry)
 612           + sizeof(struct master_header), SEEK_SET);
 613     if (ecread_index_entry(masterfd, idx)
 614         != sizeof(struct index_entry))
 615     {
 616         logf("read error #3");
 617         if (localfd)
 618             close(masterfd);
 619
 620         return false;
 621     }
 622
 623     if (localfd)
 624         close(masterfd);
 625
 626     if (idx->flag & FLAG_DELETED)
 627         return false;
 628
 629     return true;
 630 }
 631
 632 #ifndef __PCTOOL__
 633
 634 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 635 {
 636     /* We need to exclude all memory only flags & tags when writing to disk. */
 637     if (idx->flag & FLAG_DIRCACHE)
 638     {
 639         logf("memory only flags!");
 640         return false;
 641     }
 642
 643 #ifdef HAVE_TC_RAMCACHE
 644     /* Only update numeric data. Writing the whole index to RAM by memcpy
 645      * destroys dircache pointers!
 646      */
 647     if (tc_stat.ramcache)
 648     {
 649         int tag;
 650         struct index_entry *idx_ram = &hdr->indices[idxid];
 651
 652         for (tag = 0; tag < TAG_COUNT; tag++)
 653         {
 654             if (TAGCACHE_IS_NUMERIC(tag))
 655             {
 656                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 657             }
 658         }
 659
 660         /* Don't touch the dircache flag or attributes. */
 661         idx_ram->flag = (idx->flag & 0x0000ffff)
 662             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 663     }
 664 #endif
 665
 666     lseek(masterfd, idxid * sizeof(struct index_entry)
 667           + sizeof(struct master_header), SEEK_SET);
 668     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 669     {
 670         logf("write error #3");
 671         logf("idxid: %d", idxid);
 672         return false;
 673     }
 674
 675     return true;
 676 }
 677
 678 #endif /* !__PCTOOL__ */
 679
 680 static bool open_files(struct tagcache_search *tcs, int tag)
 681 {
 682     if (tcs->idxfd[tag] < 0)
 683     {
 684         char fn[MAX_PATH];
 685
 686         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 687         tcs->idxfd[tag] = open(fn, O_RDONLY);
 688     }
 689
 690     if (tcs->idxfd[tag] < 0)
 691     {
 692         logf("File not open!");
 693         return false;
 694     }
 695
 696     return true;
 697 }
 698
 699 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 700                      int tag, char *buf, long size)
 701 {
 702     struct tagfile_entry tfe;
 703     long seek;
 704
 705     *buf = '\0';
 706
 707     if (TAGCACHE_IS_NUMERIC(tag))
 708         return false;
 709
 710     seek = idx->tag_seek[tag];
 711     if (seek < 0)
 712     {
 713         logf("Retrieve failed");
 714         return false;
 715     }
 716
 717 #ifdef HAVE_TC_RAMCACHE
 718     if (tcs->ramsearch)
 719     {
 720         struct tagfile_entry *ep;
 721
 722 # ifdef HAVE_DIRCACHE
 723         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 724             && is_dircache_intact())
 725         {
 726             dircache_copy_path((struct dircache_entry *)seek,
 727                                buf, size);
 728             return true;
 729         }
 730         else
 731 # endif
 732         if (tag != tag_filename)
 733         {
 734             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 735             strlcpy(buf, ep->tag_data, size);
 736
 737             return true;
 738         }
 739     }
 740 #endif
 741
 742     if (!open_files(tcs, tag))
 743         return false;
 744
 745     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 746     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 747         != sizeof(struct tagfile_entry))
 748     {
 749         logf("read error #5");
 750         return false;
 751     }
 752
 753     if (tfe.tag_length >= size)
 754     {
 755         logf("too small buffer");
 756         return false;
 757     }
 758
 759     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 760         tfe.tag_length)
 761     {
 762         logf("read error #6");
 763         return false;
 764     }
 765
 766     buf[tfe.tag_length] = '\0';
 767
 768     return true;
 769 }
 770
 771 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
 772
 773 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
 774 {
 775 #ifndef __PCTOOL__
 776     if (! COMMAND_QUEUE_IS_EMPTY)
 777     {
 778         /* Attempt to find tag data through store-to-load forwarding in
 779            command queue */
 780         long result = -1;
 781
 782         mutex_lock(&command_queue_mutex);
 783
 784         int ridx = command_queue_widx;
 785
 786         while (ridx != command_queue_ridx)
 787         {
 788             if (--ridx < 0)
 789                 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
 790
 791             if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
 792                 && command_queue[ridx].idx_id == idx_id
 793                 && command_queue[ridx].tag == tag)
 794             {
 795                 result = command_queue[ridx].data;
 796                 break;
 797             }
 798         }
 799
 800         mutex_unlock(&command_queue_mutex);
 801
 802         if (result >= 0)
 803         {
 804             logf("read_numeric_tag: "
 805                  "Recovered tag %d value %lX from write queue",
 806                  tag, result);
 807             return result;
 808         }
 809     }
 810 #endif
 811
 812     return idx->tag_seek[tag];
 813 }
 814
 815
 816 static long check_virtual_tags(int tag, int idx_id,
 817                                const struct index_entry *idx)
 818 {
 819     long data = 0;
 820
 821     switch (tag)
 822     {
 823         case tag_virt_length_sec:
 824             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
 825             break;
 826
 827         case tag_virt_length_min:
 828             data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
 829             break;
 830
 831         case tag_virt_playtime_sec:
 832             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
 833             break;
 834
 835         case tag_virt_playtime_min:
 836             data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
 837             break;
 838
 839         case tag_virt_autoscore:
 840             if (read_numeric_tag(tag_length, idx_id, idx) == 0
 841                 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
 842             {
 843                 data = 0;
 844             }
 845             else
 846             {
 847                 /* A straight calculus gives:
 848                      autoscore = 100 * playtime / length / playcout (1)
 849                    Now, consider the euclidian division of playtime by length:
 850                      playtime = alpha * length + beta
 851                    With:
 852                      0 <= beta < length
 853                    Now, (1) becomes:
 854                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 855                    Both terms should be small enough to avoid any overflow
 856                 */
 857                 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 858                               / read_numeric_tag(tag_length, idx_id, idx))
 859                        + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
 860                                  % read_numeric_tag(tag_length, idx_id, idx)))
 861                          / read_numeric_tag(tag_length, idx_id, idx);
 862                 data /=  read_numeric_tag(tag_playcount, idx_id, idx);
 863             }
 864             break;
 865
 866         /* How many commits before the file has been added to the DB. */
 867         case tag_virt_entryage:
 868             data = current_tcmh.commitid
 869                    - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
 870             break;
 871
 872         default:
 873             data = read_numeric_tag(tag, idx_id, idx);
 874     }
 875
 876     return data;
 877 }
 878
 879 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 880 {
 881     struct index_entry idx;
 882
 883     if (!tc_stat.ready)
 884         return false;
 885
 886     if (!TAGCACHE_IS_NUMERIC(tag))
 887         return -1;
 888
 889     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 890         return -2;
 891
 892     return check_virtual_tags(tag, tcs->idx_id, &idx);
 893 }
 894
 895 inline static bool str_ends_with(const char *str1, const char *str2)
 896 {
 897     int str_len = strlen(str1);
 898     int clause_len = strlen(str2);
 899
 900     if (clause_len > str_len)
 901         return false;
 902
 903     return !strcasecmp(&str1[str_len - clause_len], str2);
 904 }
 905
 906 inline static bool str_oneof(const char *str, const char *list)
 907 {
 908     const char *sep;
 909     int l, len = strlen(str);
 910
 911     while (*list)
 912     {
 913         sep = strchr(list, '|');
 914         l = sep ? (long)sep - (long)list : (int)strlen(list);
 915         if ((l==len) && !strncasecmp(str, list, len))
 916             return true;
 917         list += sep ? l + 1 : l;
 918     }
 919
 920     return false;
 921 }
 922
 923 static bool check_against_clause(long numeric, const char *str,
 924                                  const struct tagcache_search_clause *clause)
 925 {
 926     if (clause->numeric)
 927     {
 928         switch (clause->type)
 929         {
 930             case clause_is:
 931                 return numeric == clause->numeric_data;
 932             case clause_is_not:
 933                 return numeric != clause->numeric_data;
 934             case clause_gt:
 935                 return numeric > clause->numeric_data;
 936             case clause_gteq:
 937                 return numeric >= clause->numeric_data;
 938             case clause_lt:
 939                 return numeric < clause->numeric_data;
 940             case clause_lteq:
 941                 return numeric <= clause->numeric_data;
 942             default:
 943                 logf("Incorrect numeric tag: %d", clause->type);
 944         }
 945     }
 946     else
 947     {
 948         switch (clause->type)
 949         {
 950             case clause_is:
 951                 return !strcasecmp(clause->str, str);
 952             case clause_is_not:
 953                 return strcasecmp(clause->str, str);
 954             case clause_gt:
 955                 return 0>strcasecmp(clause->str, str);
 956             case clause_gteq:
 957                 return 0>=strcasecmp(clause->str, str);
 958             case clause_lt:
 959                 return 0<strcasecmp(clause->str, str);
 960             case clause_lteq:
 961                 return 0<=strcasecmp(clause->str, str);
 962             case clause_contains:
 963                 return (strcasestr(str, clause->str) != NULL);
 964             case clause_not_contains:
 965                 return (strcasestr(str, clause->str) == NULL);
 966             case clause_begins_with:
 967                 return (strcasestr(str, clause->str) == str);
 968             case clause_not_begins_with:
 969                 return (strcasestr(str, clause->str) != str);
 970             case clause_ends_with:
 971                 return str_ends_with(str, clause->str);
 972             case clause_not_ends_with:
 973                 return !str_ends_with(str, clause->str);
 974             case clause_oneof:
 975                 return str_oneof(str, clause->str);
 976
 977             default:
 978                 logf("Incorrect tag: %d", clause->type);
 979         }
 980     }
 981
 982     return false;
 983 }
 984
 985 static bool check_clauses(struct tagcache_search *tcs,
 986                           struct index_entry *idx,
 987                           struct tagcache_search_clause **clauses, int count)
 988 {
 989     int i;
 990
 991     /* Go through all conditional clauses. */
 992     for (i = 0; i < count; i++)
 993     {
 994         int seek;
 995         char buf[256];
 996         char *str;
 997         struct tagcache_search_clause *clause = clauses[i];
 998
 999         if (clause->type == clause_logical_or)
1000             break; /* all conditions before logical-or satisfied --
1001                       stop processing clauses */
1002
1003 #ifdef HAVE_TC_RAMCACHE
1004         str = NULL;
1005
1006         if (tcs->ramsearch)
1007         {
1008             struct tagfile_entry *tfe;
1009
1010             seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1011
1012             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1013             {
1014                 if (clause->tag == tag_filename)
1015                 {
1016                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1017                     str = buf;
1018                 }
1019                 else
1020                 {
1021                     tfe = (struct tagfile_entry *)&hdr->tags[clause->tag][seek];
1022                     str = tfe->tag_data;
1023                 }
1024             }
1025         }
1026         else
1027 #endif
1028         {
1029             struct tagfile_entry tfe;
1030             str = buf;
1031
1032             seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1033
1034             memset(buf, 0, sizeof buf);
1035             if (!TAGCACHE_IS_NUMERIC(clause->tag))
1036             {
1037                 int fd = tcs->idxfd[clause->tag];
1038                 lseek(fd, seek, SEEK_SET);
1039                 ecread_tagfile_entry(fd, &tfe);
1040                 if (tfe.tag_length >= (int)sizeof(buf))
1041                 {
1042                     logf("Too long tag read!");
1043                     return false;
1044                 }
1045
1046                 read(fd, str, tfe.tag_length);
1047
1048                 /* Check if entry has been deleted. */
1049                 if (str[0] == '\0')
1050                     return false;
1051             }
1052         }
1053
1054         if (!check_against_clause(seek, str, clause))
1055         {
1056             /* Clause failed -- try finding a logical-or clause */
1057             while (++i < count)
1058             {
1059                 if (clauses[i]->type == clause_logical_or)
1060                     break;
1061             }
1062
1063             if (i < count)        /* Found logical-or? */
1064                 continue;         /* Check clauses after logical-or */
1065
1066             return false;
1067         }
1068     }
1069
1070     return true;
1071 }
1072
1073 bool tagcache_check_clauses(struct tagcache_search *tcs,
1074                             struct tagcache_search_clause **clause, int count)
1075 {
1076     struct index_entry idx;
1077
1078     if (count == 0)
1079         return true;
1080
1081     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1082         return false;
1083
1084     return check_clauses(tcs, &idx, clause, count);
1085 }
1086
1087 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1088 {
1089     int i;
1090
1091     /* If uniq buffer is not defined we must return true for search to work. */
1092     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1093         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1094     {
1095         return true;
1096     }
1097
1098     for (i = 0; i < tcs->unique_list_count; i++)
1099     {
1100         /* Return false if entry is found. */
1101         if (tcs->unique_list[i] == id)
1102             return false;
1103     }
1104
1105     if (tcs->unique_list_count < tcs->unique_list_capacity)
1106     {
1107         tcs->unique_list[i] = id;
1108         tcs->unique_list_count++;
1109     }
1110
1111     return true;
1112 }
1113
1114 static bool build_lookup_list(struct tagcache_search *tcs)
1115 {
1116     struct index_entry entry;
1117     int i, j;
1118
1119     tcs->seek_list_count = 0;
1120
1121 #ifdef HAVE_TC_RAMCACHE
1122     if (tcs->ramsearch
1123 # ifdef HAVE_DIRCACHE
1124         && (tcs->type != tag_filename || is_dircache_intact())
1125 # endif
1126         )
1127     {
1128         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1129         {
1130             struct tagcache_seeklist_entry *seeklist;
1131             struct index_entry *idx = &hdr->indices[i];
1132             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1133                 break ;
1134
1135             /* Skip deleted files. */
1136             if (idx->flag & FLAG_DELETED)
1137                 continue;
1138
1139             /* Go through all filters.. */
1140             for (j = 0; j < tcs->filter_count; j++)
1141             {
1142                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1143                 {
1144                     break ;
1145                 }
1146             }
1147
1148             if (j < tcs->filter_count)
1149                 continue ;
1150
1151             /* Check for conditions. */
1152             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1153                 continue;
1154
1155             /* Add to the seek list if not already in uniq buffer. */
1156             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1157                 continue;
1158
1159             /* Lets add it. */
1160             seeklist = &tcs->seeklist[tcs->seek_list_count];
1161             seeklist->seek = idx->tag_seek[tcs->type];
1162             seeklist->flag = idx->flag;
1163             seeklist->idx_id = i;
1164             tcs->seek_list_count++;
1165         }
1166
1167         tcs->seek_pos = i;
1168
1169         return tcs->seek_list_count > 0;
1170     }
1171 #endif
1172
1173     if (tcs->masterfd < 0)
1174     {
1175         struct master_header tcmh;
1176         tcs->masterfd = open_master_fd(&tcmh, false);
1177     }
1178
1179     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1180             sizeof(struct master_header), SEEK_SET);
1181
1182     while (ecread_index_entry(tcs->masterfd, &entry)
1183            == sizeof(struct index_entry))
1184     {
1185         struct tagcache_seeklist_entry *seeklist;
1186
1187         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1188             break ;
1189
1190         i = tcs->seek_pos;
1191         tcs->seek_pos++;
1192
1193         /* Check if entry has been deleted. */
1194         if (entry.flag & FLAG_DELETED)
1195             continue;
1196
1197         /* Go through all filters.. */
1198         for (j = 0; j < tcs->filter_count; j++)
1199         {
1200             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1201                 break ;
1202         }
1203
1204         if (j < tcs->filter_count)
1205             continue ;
1206
1207         /* Check for conditions. */
1208         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1209             continue;
1210
1211         /* Add to the seek list if not already in uniq buffer. */
1212         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1213             continue;
1214
1215         /* Lets add it. */
1216         seeklist = &tcs->seeklist[tcs->seek_list_count];
1217         seeklist->seek = entry.tag_seek[tcs->type];
1218         seeklist->flag = entry.flag;
1219         seeklist->idx_id = i;
1220         tcs->seek_list_count++;
1221
1222         yield();
1223     }
1224
1225     return tcs->seek_list_count > 0;
1226 }
1227
1228
1229 static void remove_files(void)
1230 {
1231     int i;
1232     char buf[MAX_PATH];
1233
1234     tc_stat.ready = false;
1235     tc_stat.ramcache = false;
1236     tc_stat.econ = false;
1237     remove(TAGCACHE_FILE_MASTER);
1238     for (i = 0; i < TAG_COUNT; i++)
1239     {
1240         if (TAGCACHE_IS_NUMERIC(i))
1241             continue;
1242
1243         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1244         remove(buf);
1245     }
1246 }
1247
1248
1249 static bool check_all_headers(void)
1250 {
1251     struct master_header myhdr;
1252     struct tagcache_header tch;
1253     int tag;
1254     int fd;
1255
1256     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1257         return false;
1258
1259     close(fd);
1260     if (myhdr.dirty)
1261     {
1262         logf("tagcache is dirty!");
1263         return false;
1264     }
1265
1266     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1267
1268     for (tag = 0; tag < TAG_COUNT; tag++)
1269     {
1270         if (TAGCACHE_IS_NUMERIC(tag))
1271             continue;
1272
1273         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1274             return false;
1275
1276         close(fd);
1277     }
1278
1279     return true;
1280 }
1281
1282 bool tagcache_is_busy(void)
1283 {
1284     return read_lock || write_lock;
1285 }
1286
1287 bool tagcache_search(struct tagcache_search *tcs, int tag)
1288 {
1289     struct tagcache_header tag_hdr;
1290     struct master_header   master_hdr;
1291     int i;
1292
1293     while (read_lock)
1294         sleep(1);
1295
1296     memset(tcs, 0, sizeof(struct tagcache_search));
1297     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1298         return false;
1299
1300     tcs->position = sizeof(struct tagcache_header);
1301     tcs->type = tag;
1302     tcs->seek_pos = 0;
1303     tcs->list_position = 0;
1304     tcs->seek_list_count = 0;
1305     tcs->filter_count = 0;
1306     tcs->masterfd = -1;
1307
1308     for (i = 0; i < TAG_COUNT; i++)
1309         tcs->idxfd[i] = -1;
1310
1311 #ifndef HAVE_TC_RAMCACHE
1312     tcs->ramsearch = false;
1313 #else
1314     tcs->ramsearch = tc_stat.ramcache;
1315     if (tcs->ramsearch)
1316     {
1317         tcs->entry_count = hdr->entry_count[tcs->type];
1318     }
1319     else
1320 #endif
1321     {
1322         /* Always open as R/W so we can pass tcs to functions that modify data also
1323          * without failing. */
1324         tcs->masterfd = open_master_fd(&master_hdr, true);
1325         if (tcs->masterfd < 0)
1326             return false;
1327
1328         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1329         {
1330             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1331             if (tcs->idxfd[tcs->type] < 0)
1332                 return false;
1333
1334             tcs->entry_count = tag_hdr.entry_count;
1335         }
1336         else
1337         {
1338             tcs->entry_count = master_hdr.tch.entry_count;
1339         }
1340     }
1341
1342     tcs->valid = true;
1343     tcs->initialized = true;
1344     write_lock++;
1345
1346     return true;
1347 }
1348
1349 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1350                                  void *buffer, long length)
1351 {
1352     tcs->unique_list = (unsigned long *)buffer;
1353     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1354     tcs->unique_list_count = 0;
1355 }
1356
1357 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1358                                 int tag, int seek)
1359 {
1360     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1361         return false;
1362
1363     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1364         return false;
1365
1366     tcs->filter_tag[tcs->filter_count] = tag;
1367     tcs->filter_seek[tcs->filter_count] = seek;
1368     tcs->filter_count++;
1369
1370     return true;
1371 }
1372
1373 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1374                                 struct tagcache_search_clause *clause)
1375 {
1376     int i;
1377
1378     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1379     {
1380         logf("Too many clauses");
1381         return false;
1382     }
1383
1384     if (clause->type != clause_logical_or)
1385     {
1386         /* Check if there is already a similar filter in present (filters are
1387          * much faster than clauses).
1388          */
1389         for (i = 0; i < tcs->filter_count; i++)
1390         {
1391             if (tcs->filter_tag[i] == clause->tag)
1392                 return true;
1393         }
1394
1395         if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1396         {
1397             char buf[MAX_PATH];
1398
1399             snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1400             tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1401         }
1402     }
1403
1404     tcs->clause[tcs->clause_count] = clause;
1405     tcs->clause_count++;
1406
1407     return true;
1408 }
1409
1410 static bool get_next(struct tagcache_search *tcs)
1411 {
1412     static char buf[TAG_MAXLEN+32];
1413     struct tagfile_entry entry;
1414 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1415     long flag = 0;
1416 #endif
1417
1418     if (!tcs->valid || !tc_stat.ready)
1419         return false;
1420
1421     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1422 #ifdef HAVE_TC_RAMCACHE
1423         && !tcs->ramsearch
1424 #endif
1425         )
1426         return false;
1427
1428     /* Relative fetch. */
1429     if (tcs->filter_count > 0 || tcs->clause_count > 0
1430         || TAGCACHE_IS_NUMERIC(tcs->type)
1431 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1432         /* We need to retrieve flag status for dircache. */
1433         || (tcs->ramsearch && tcs->type == tag_filename)
1434 #endif
1435         )
1436     {
1437         struct tagcache_seeklist_entry *seeklist;
1438
1439         /* Check for end of list. */
1440         if (tcs->list_position == tcs->seek_list_count)
1441         {
1442             tcs->list_position = 0;
1443
1444             /* Try to fetch more. */
1445             if (!build_lookup_list(tcs))
1446             {
1447                 tcs->valid = false;
1448                 return false;
1449             }
1450         }
1451
1452         seeklist = &tcs->seeklist[tcs->list_position];
1453 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1454         flag = seeklist->flag;
1455 #endif
1456         tcs->position = seeklist->seek;
1457         tcs->idx_id = seeklist->idx_id;
1458         tcs->list_position++;
1459     }
1460     else
1461     {
1462         if (tcs->entry_count == 0)
1463         {
1464             tcs->valid = false;
1465             return false;
1466         }
1467
1468         tcs->entry_count--;
1469     }
1470
1471     tcs->result_seek = tcs->position;
1472
1473     if (TAGCACHE_IS_NUMERIC(tcs->type))
1474     {
1475         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1476         tcs->result = buf;
1477         tcs->result_len = strlen(buf) + 1;
1478         return true;
1479     }
1480
1481     /* Direct fetch. */
1482 #ifdef HAVE_TC_RAMCACHE
1483     if (tcs->ramsearch)
1484     {
1485
1486 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1487         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1488             && is_dircache_intact())
1489         {
1490             dircache_copy_path((struct dircache_entry *)tcs->position,
1491                                buf, sizeof buf);
1492             tcs->result = buf;
1493             tcs->result_len = strlen(buf) + 1;
1494             tcs->ramresult = false;
1495
1496             return true;
1497         }
1498         else
1499 #endif
1500         if (tcs->type != tag_filename)
1501         {
1502             struct tagfile_entry *ep;
1503
1504             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1505             tcs->result = ep->tag_data;
1506             tcs->result_len = strlen(tcs->result) + 1;
1507             tcs->idx_id = ep->idx_id;
1508             tcs->ramresult = true;
1509
1510             /* Increase position for the next run. This may get overwritten. */
1511             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1512
1513             return true;
1514         }
1515     }
1516 #endif
1517
1518     if (!open_files(tcs, tcs->type))
1519     {
1520         tcs->valid = false;
1521         return false;
1522     }
1523
1524     /* Seek stream to the correct position and continue to direct fetch. */
1525     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1526
1527     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1528     {
1529         logf("read error #5");
1530         tcs->valid = false;
1531         return false;
1532     }
1533
1534     if (entry.tag_length > (long)sizeof(buf))
1535     {
1536         tcs->valid = false;
1537         logf("too long tag #2");
1538         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1539         return false;
1540     }
1541
1542     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1543     {
1544         tcs->valid = false;
1545         logf("read error #4");
1546         return false;
1547     }
1548
1549     /**
1550      Update the position for the next read (this may be overridden
1551      if filters or clauses are being used).
1552      */
1553     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1554     tcs->result = buf;
1555     tcs->result_len = strlen(tcs->result) + 1;
1556     tcs->idx_id = entry.idx_id;
1557     tcs->ramresult = false;
1558
1559     return true;
1560 }
1561
1562 bool tagcache_get_next(struct tagcache_search *tcs)
1563 {
1564     while (get_next(tcs))
1565     {
1566         if (tcs->result_len > 1)
1567             return true;
1568     }
1569
1570     return false;
1571 }
1572
1573 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1574                        int tag, char *buf, long size)
1575 {
1576     struct index_entry idx;
1577
1578     *buf = '\0';
1579     if (!get_index(tcs->masterfd, idxid, &idx, true))
1580         return false;
1581
1582     return retrieve(tcs, &idx, tag, buf, size);
1583 }
1584
1585 static bool update_master_header(void)
1586 {
1587     struct master_header myhdr;
1588     int fd;
1589
1590     if (!tc_stat.ready)
1591         return false;
1592
1593     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1594         return false;
1595
1596     myhdr.serial = current_tcmh.serial;
1597     myhdr.commitid = current_tcmh.commitid;
1598     myhdr.dirty = current_tcmh.dirty;
1599
1600     /* Write it back */
1601     lseek(fd, 0, SEEK_SET);
1602     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1603     close(fd);
1604
1605 #ifdef HAVE_TC_RAMCACHE
1606     if (hdr)
1607     {
1608         hdr->h.serial = current_tcmh.serial;
1609         hdr->h.commitid = current_tcmh.commitid;
1610         hdr->h.dirty = current_tcmh.dirty;
1611     }
1612 #endif
1613
1614     return true;
1615 }
1616
1617 #if 0
1618
1619 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1620 {
1621     struct tagentry *entry;
1622
1623     if (tcs->type != tag_title)
1624         return ;
1625
1626     /* We will need reserve buffer for this. */
1627     if (tcs->ramcache)
1628     {
1629         struct tagfile_entry *ep;
1630
1631         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1632         tcs->seek_list[tcs->seek_list_count];
1633     }
1634
1635     entry = find_entry_ram();
1636
1637 }
1638 #endif
1639
1640 void tagcache_search_finish(struct tagcache_search *tcs)
1641 {
1642     int i;
1643
1644     if (!tcs->initialized)
1645         return;
1646
1647     if (tcs->masterfd >= 0)
1648     {
1649         close(tcs->masterfd);
1650         tcs->masterfd = -1;
1651     }
1652
1653     for (i = 0; i < TAG_COUNT; i++)
1654     {
1655         if (tcs->idxfd[i] >= 0)
1656         {
1657             close(tcs->idxfd[i]);
1658             tcs->idxfd[i] = -1;
1659         }
1660     }
1661
1662     tcs->ramsearch = false;
1663     tcs->valid = false;
1664     tcs->initialized = 0;
1665     if (write_lock > 0)
1666         write_lock--;
1667 }
1668
1669 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1670 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1671 {
1672     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1673 }
1674
1675 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1676 {
1677     return check_virtual_tags(tag, idx_id, entry);
1678 }
1679
1680 static char* get_tag_string(const struct index_entry *entry, int tag)
1681 {
1682     char* s = get_tag(entry, tag)->tag_data;
1683     return strcmp(s, UNTAGGED) ? s : NULL;
1684 }
1685
1686 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1687 {
1688     struct index_entry *entry;
1689     int idx_id;
1690
1691     if (!tc_stat.ready || !tc_stat.ramcache)
1692         return false;
1693
1694     /* Find the corresponding entry in tagcache. */
1695     idx_id = find_entry_ram(filename, NULL);
1696     if (idx_id < 0)
1697         return false;
1698
1699     entry = &hdr->indices[idx_id];
1700
1701     memset(id3, 0, sizeof(struct mp3entry));
1702
1703     id3->title        = get_tag_string(entry, tag_title);
1704     id3->artist       = get_tag_string(entry, tag_artist);
1705     id3->album        = get_tag_string(entry, tag_album);
1706     id3->genre_string = get_tag_string(entry, tag_genre);
1707     id3->composer     = get_tag_string(entry, tag_composer);
1708     id3->comment      = get_tag_string(entry, tag_comment);
1709     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1710     id3->grouping     = get_tag_string(entry, tag_grouping);
1711
1712     id3->length     = get_tag_numeric(entry, tag_length, idx_id);
1713     id3->playcount  = get_tag_numeric(entry, tag_playcount, idx_id);
1714     id3->rating     = get_tag_numeric(entry, tag_rating, idx_id);
1715     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1716     id3->score      = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1717     id3->year       = get_tag_numeric(entry, tag_year, idx_id);
1718
1719     id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1720     id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1721     id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1722     if (id3->bitrate == 0)
1723         id3->bitrate = 1;
1724
1725 #if CONFIG_CODEC == SWCODEC
1726     if (global_settings.autoresume_enable)
1727     {
1728         id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1729         logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1730              id3->title, id3->offset);
1731     }
1732 #endif
1733
1734     return true;
1735 }
1736 #endif
1737
1738 static inline void write_item(const char *item)
1739 {
1740     int len = strlen(item) + 1;
1741
1742     data_size += len;
1743     write(cachefd, item, len);
1744 }
1745
1746 static int check_if_empty(char **tag)
1747 {
1748     int length;
1749
1750     if (*tag == NULL || **tag == '\0')
1751     {
1752         *tag = UNTAGGED;
1753         return sizeof(UNTAGGED); /* Tag length */
1754     }
1755
1756     length = strlen(*tag);
1757     if (length > TAG_MAXLEN)
1758     {
1759         logf("over length tag: %s", *tag);
1760         length = TAG_MAXLEN;
1761         (*tag)[length] = '\0';
1762     }
1763
1764     return length + 1;
1765 }
1766
1767 #define ADD_TAG(entry,tag,data) \
1768     /* Adding tag */ \
1769     entry.tag_offset[tag] = offset; \
1770     entry.tag_length[tag] = check_if_empty(data); \
1771     offset += entry.tag_length[tag]
1772 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1773  * idea, as it uses lots of stack and is called from a recursive function
1774  * (check_dir).
1775  */
1776 static void __attribute__ ((noinline)) add_tagcache(char *path,
1777                                                     unsigned long mtime
1778 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1779                                                     ,const struct dircache_entry *dc
1780 #endif
1781                                                    )
1782 {
1783     struct mp3entry id3;
1784     struct temp_file_entry entry;
1785     bool ret;
1786     int fd;
1787     int idx_id = -1;
1788     char tracknumfix[3];
1789     int offset = 0;
1790     int path_length = strlen(path);
1791     bool has_albumartist;
1792     bool has_grouping;
1793
1794 #ifdef SIMULATOR
1795     /* Crude logging for the sim - to aid in debugging */
1796     int logfd = open(ROCKBOX_DIR "/database.log",
1797                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1798     if (logfd >= 0) {
1799         write(logfd, path, strlen(path));
1800         write(logfd, "\n", 1);
1801         close(logfd);
1802     }
1803 #endif
1804
1805     if (cachefd < 0)
1806         return ;
1807
1808     /* Check for overlength file path. */
1809     if (path_length > TAG_MAXLEN)
1810     {
1811         /* Path can't be shortened. */
1812         logf("Too long path: %s", path);
1813         return ;
1814     }
1815
1816     /* Check if the file is supported. */
1817     if (probe_file_format(path) == AFMT_UNKNOWN)
1818         return ;
1819
1820     /* Check if the file is already cached. */
1821 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1822     if (tc_stat.ramcache && is_dircache_intact())
1823     {
1824         idx_id = find_entry_ram(path, dc);
1825     }
1826 #endif
1827
1828     /* Be sure the entry doesn't exist. */
1829     if (filenametag_fd >= 0 && idx_id < 0)
1830         idx_id = find_entry_disk(path, false);
1831
1832     /* Check if file has been modified. */
1833     if (idx_id >= 0)
1834     {
1835         struct index_entry idx;
1836
1837         /* TODO: Mark that the index exists (for fast reverse scan) */
1838         //found_idx[idx_id/8] |= idx_id%8;
1839
1840         if (!get_index(-1, idx_id, &idx, true))
1841         {
1842             logf("failed to retrieve index entry");
1843             return ;
1844         }
1845
1846         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1847         {
1848             /* No changes to file. */
1849             return ;
1850         }
1851
1852         /* Metadata might have been changed. Delete the entry. */
1853         logf("Re-adding: %s", path);
1854         if (!delete_entry(idx_id))
1855         {
1856             logf("delete_entry failed: %d", idx_id);
1857             return ;
1858         }
1859     }
1860
1861     fd = open(path, O_RDONLY);
1862     if (fd < 0)
1863     {
1864         logf("open fail: %s", path);
1865         return ;
1866     }
1867
1868     memset(&id3, 0, sizeof(struct mp3entry));
1869     memset(&entry, 0, sizeof(struct temp_file_entry));
1870     memset(&tracknumfix, 0, sizeof(tracknumfix));
1871     ret = get_metadata(&id3, fd, path);
1872     close(fd);
1873
1874     if (!ret)
1875         return ;
1876
1877     logf("-> %s", path);
1878
1879     /* Generate track number if missing. */
1880     if (id3.tracknum <= 0)
1881     {
1882         const char *p = strrchr(path, '.');
1883
1884         if (p == NULL)
1885             p = &path[strlen(path)-1];
1886
1887         while (*p != '/')
1888         {
1889             if (isdigit(*p) && isdigit(*(p-1)))
1890             {
1891                 tracknumfix[1] = *p--;
1892                 tracknumfix[0] = *p;
1893                 break;
1894             }
1895             p--;
1896         }
1897
1898         if (tracknumfix[0] != '\0')
1899         {
1900             id3.tracknum = atoi(tracknumfix);
1901             /* Set a flag to indicate track number has been generated. */
1902             entry.flag |= FLAG_TRKNUMGEN;
1903         }
1904         else
1905         {
1906             /* Unable to generate track number. */
1907             id3.tracknum = -1;
1908         }
1909     }
1910
1911     /* Numeric tags */
1912     entry.tag_offset[tag_year] = id3.year;
1913     entry.tag_offset[tag_discnumber] = id3.discnum;
1914     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1915     entry.tag_offset[tag_length] = id3.length;
1916     entry.tag_offset[tag_bitrate] = id3.bitrate;
1917     entry.tag_offset[tag_mtime] = mtime;
1918
1919     /* String tags. */
1920     has_albumartist = id3.albumartist != NULL
1921         && strlen(id3.albumartist) > 0;
1922     has_grouping = id3.grouping != NULL
1923         && strlen(id3.grouping) > 0;
1924
1925     ADD_TAG(entry, tag_filename, &path);
1926     ADD_TAG(entry, tag_title, &id3.title);
1927     ADD_TAG(entry, tag_artist, &id3.artist);
1928     ADD_TAG(entry, tag_album, &id3.album);
1929     ADD_TAG(entry, tag_genre, &id3.genre_string);
1930     ADD_TAG(entry, tag_composer, &id3.composer);
1931     ADD_TAG(entry, tag_comment, &id3.comment);
1932     if (has_albumartist)
1933     {
1934         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1935     }
1936     else
1937     {
1938         ADD_TAG(entry, tag_albumartist, &id3.artist);
1939     }
1940     if (has_grouping)
1941     {
1942         ADD_TAG(entry, tag_grouping, &id3.grouping);
1943     }
1944     else
1945     {
1946         ADD_TAG(entry, tag_grouping, &id3.title);
1947     }
1948     entry.data_length = offset;
1949
1950     /* Write the header */
1951     write(cachefd, &entry, sizeof(struct temp_file_entry));
1952
1953     /* And tags also... Correct order is critical */
1954     write_item(path);
1955     write_item(id3.title);
1956     write_item(id3.artist);
1957     write_item(id3.album);
1958     write_item(id3.genre_string);
1959     write_item(id3.composer);
1960     write_item(id3.comment);
1961     if (has_albumartist)
1962     {
1963         write_item(id3.albumartist);
1964     }
1965     else
1966     {
1967         write_item(id3.artist);
1968     }
1969     if (has_grouping)
1970     {
1971         write_item(id3.grouping);
1972     }
1973     else
1974     {
1975         write_item(id3.title);
1976     }
1977     total_entry_count++;
1978 }
1979
1980 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1981 {
1982     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1983     int len = strlen(str)+1;
1984     int i;
1985     unsigned crc32;
1986     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1987     char buf[TAG_MAXLEN+32];
1988
1989     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1990         buf[i] = tolower(str[i]);
1991     buf[i] = '\0';
1992
1993     crc32 = crc_32(buf, i, 0xffffffff);
1994
1995     if (unique)
1996     {
1997         /* Check if the crc does not exist -> entry does not exist for sure. */
1998         for (i = 0; i < tempbufidx; i++)
1999         {
2000             if (crcbuf[-i] != crc32)
2001                 continue;
2002
2003             if (!strcasecmp(str, index[i].str))
2004             {
2005                 if (id < 0 || id >= lookup_buffer_depth)
2006                 {
2007                     logf("lookup buf overf.: %d", id);
2008                     return false;
2009                 }
2010
2011                 lookup[id] = &index[i];
2012                 return true;
2013             }
2014         }
2015     }
2016
2017     /* Insert to CRC buffer. */
2018     crcbuf[-tempbufidx] = crc32;
2019     tempbuf_left -= 4;
2020
2021     /* Insert it to the buffer. */
2022     tempbuf_left -= len;
2023     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
2024         return false;
2025
2026     if (id >= lookup_buffer_depth)
2027     {
2028         logf("lookup buf overf. #2: %d", id);
2029         return false;
2030     }
2031
2032     if (id >= 0)
2033     {
2034         lookup[id] = &index[tempbufidx];
2035         index[tempbufidx].idlist.id = id;
2036     }
2037     else
2038         index[tempbufidx].idlist.id = -1;
2039
2040     index[tempbufidx].idlist.next = NULL;
2041     index[tempbufidx].idx_id = idx_id;
2042     index[tempbufidx].seek = -1;
2043     index[tempbufidx].str = &tempbuf[tempbuf_pos];
2044     memcpy(index[tempbufidx].str, str, len);
2045     tempbuf_pos += len;
2046     tempbufidx++;
2047
2048     return true;
2049 }
2050
2051 static int compare(const void *p1, const void *p2)
2052 {
2053     do_timed_yield();
2054
2055     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2056     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2057
2058     if (strcmp(e1->str, UNTAGGED) == 0)
2059     {
2060         if (strcmp(e2->str, UNTAGGED) == 0)
2061             return 0;
2062         return -1;
2063     }
2064     else if (strcmp(e2->str, UNTAGGED) == 0)
2065         return 1;
2066
2067     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2068 }
2069
2070 static int tempbuf_sort(int fd)
2071 {
2072     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2073     struct tagfile_entry fe;
2074     int i;
2075     int length;
2076
2077     /* Generate reverse lookup entries. */
2078     for (i = 0; i < lookup_buffer_depth; i++)
2079     {
2080         struct tempbuf_id_list *idlist;
2081
2082         if (!lookup[i])
2083             continue;
2084
2085         if (lookup[i]->idlist.id == i)
2086             continue;
2087
2088         idlist = &lookup[i]->idlist;
2089         while (idlist->next != NULL)
2090             idlist = idlist->next;
2091
2092         tempbuf_left -= sizeof(struct tempbuf_id_list);
2093         if (tempbuf_left - 4 < 0)
2094             return -1;
2095
2096         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2097         if (tempbuf_pos & 0x03)
2098         {
2099             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2100             tempbuf_left -= 3;
2101             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2102         }
2103         tempbuf_pos += sizeof(struct tempbuf_id_list);
2104
2105         idlist = idlist->next;
2106         idlist->id = i;
2107         idlist->next = NULL;
2108
2109         do_timed_yield();
2110     }
2111
2112     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2113     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2114
2115     for (i = 0; i < tempbufidx; i++)
2116     {
2117         struct tempbuf_id_list *idlist = &index[i].idlist;
2118
2119         /* Fix the lookup list. */
2120         while (idlist != NULL)
2121         {
2122             if (idlist->id >= 0)
2123                 lookup[idlist->id] = &index[i];
2124             idlist = idlist->next;
2125         }
2126
2127         index[i].seek = lseek(fd, 0, SEEK_CUR);
2128         length = strlen(index[i].str) + 1;
2129         fe.tag_length = length;
2130         fe.idx_id = index[i].idx_id;
2131
2132         /* Check the chunk alignment. */
2133         if ((fe.tag_length + sizeof(struct tagfile_entry))
2134             % TAGFILE_ENTRY_CHUNK_LENGTH)
2135         {
2136             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2137                 ((fe.tag_length + sizeof(struct tagfile_entry))
2138                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2139         }
2140
2141 #ifdef TAGCACHE_STRICT_ALIGN
2142         /* Make sure the entry is long aligned. */
2143         if (index[i].seek & 0x03)
2144         {
2145             logf("tempbuf_sort: alignment error!");
2146             return -3;
2147         }
2148 #endif
2149
2150         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2151             sizeof(struct tagfile_entry))
2152         {
2153             logf("tempbuf_sort: write error #1");
2154             return -1;
2155         }
2156
2157         if (write(fd, index[i].str, length) != length)
2158         {
2159             logf("tempbuf_sort: write error #2");
2160             return -2;
2161         }
2162
2163         /* Write some padding. */
2164         if (fe.tag_length - length > 0)
2165             write(fd, "XXXXXXXX", fe.tag_length - length);
2166     }
2167
2168     return i;
2169 }
2170
2171 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2172 {
2173     if (id < 0 || id >= lookup_buffer_depth)
2174         return NULL;
2175
2176     return lookup[id];
2177 }
2178
2179
2180 inline static int tempbuf_find_location(int id)
2181 {
2182     struct tempbuf_searchidx *entry;
2183
2184     entry = tempbuf_locate(id);
2185     if (entry == NULL)
2186         return -1;
2187
2188     return entry->seek;
2189 }
2190
2191 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2192 {
2193     struct master_header tcmh;
2194     struct index_entry idx;
2195     int masterfd;
2196     int masterfd_pos;
2197     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2198     int max_entries;
2199     int entries_processed = 0;
2200     int i, j;
2201     char buf[TAG_MAXLEN];
2202
2203     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2204
2205     logf("Building numeric indices...");
2206     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2207
2208     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2209         return false;
2210
2211     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2212                          SEEK_CUR);
2213     if (masterfd_pos == filesize(masterfd))
2214     {
2215         logf("we can't append!");
2216         close(masterfd);
2217         return false;
2218     }
2219
2220     while (entries_processed < h->entry_count)
2221     {
2222         int count = MIN(h->entry_count - entries_processed, max_entries);
2223
2224         /* Read in as many entries as possible. */
2225         for (i = 0; i < count; i++)
2226         {
2227             struct temp_file_entry *tfe = &entrybuf[i];
2228             int datastart;
2229
2230             /* Read in numeric data. */
2231             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2232                 sizeof(struct temp_file_entry))
2233             {
2234                 logf("read fail #1");
2235                 close(masterfd);
2236                 return false;
2237             }
2238
2239             datastart = lseek(tmpfd, 0, SEEK_CUR);
2240
2241             /**
2242              * Read string data from the following tags:
2243              * - tag_filename
2244              * - tag_artist
2245              * - tag_album
2246              * - tag_title
2247              *
2248              * A crc32 hash is calculated from the read data
2249              * and stored back to the data offset field kept in memory.
2250              */
2251 #define tmpdb_read_string_tag(tag) \
2252     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2253     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2254     { \
2255         logf("read fail: buffer overflow"); \
2256         close(masterfd); \
2257         return false; \
2258     } \
2259     \
2260     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2261         tfe->tag_length[tag]) \
2262     { \
2263         logf("read fail #2"); \
2264         close(masterfd); \
2265         return false; \
2266     } \
2267     \
2268     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2269     lseek(tmpfd, datastart, SEEK_SET)
2270
2271             tmpdb_read_string_tag(tag_filename);
2272             tmpdb_read_string_tag(tag_artist);
2273             tmpdb_read_string_tag(tag_album);
2274             tmpdb_read_string_tag(tag_title);
2275
2276             /* Seek to the end of the string data. */
2277             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2278         }
2279
2280         /* Backup the master index position. */
2281         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2282         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2283
2284         /* Check if we can resurrect some deleted runtime statistics data. */
2285         for (i = 0; i < tcmh.tch.entry_count; i++)
2286         {
2287             /* Read the index entry. */
2288             if (ecread_index_entry(masterfd, &idx)
2289                 != sizeof(struct index_entry))
2290             {
2291                 logf("read fail #3");
2292                 close(masterfd);
2293                 return false;
2294             }
2295
2296             /**
2297              * Skip unless the entry is marked as being deleted
2298              * or the data has already been resurrected.
2299              */
2300             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2301                 continue;
2302
2303             /* Now try to match the entry. */
2304             /**
2305              * To succesfully match a song, the following conditions
2306              * must apply:
2307              *
2308              * For numeric fields: tag_length
2309              * - Full identical match is required
2310              *
2311              * If tag_filename matches, no further checking necessary.
2312              *
2313              * For string hashes: tag_artist, tag_album, tag_title
2314              * - All three of these must match
2315              */
2316             for (j = 0; j < count; j++)
2317             {
2318                 struct temp_file_entry *tfe = &entrybuf[j];
2319
2320                 /* Try to match numeric fields first. */
2321                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2322                     continue;
2323
2324                 /* Now it's time to do the hash matching. */
2325                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2326                 {
2327                     int match_count = 0;
2328
2329                     /* No filename match, check if we can match two other tags. */
2330 #define tmpdb_match(tag) \
2331     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2332         match_count++
2333
2334                     tmpdb_match(tag_artist);
2335                     tmpdb_match(tag_album);
2336                     tmpdb_match(tag_title);
2337
2338                     if (match_count < 3)
2339                     {
2340                         /* Still no match found, give up. */
2341                         continue;
2342                     }
2343                 }
2344
2345                 /* A match found, now copy & resurrect the statistical data. */
2346 #define tmpdb_copy_tag(tag) \
2347     tfe->tag_offset[tag] = idx.tag_seek[tag]
2348
2349                 tmpdb_copy_tag(tag_playcount);
2350                 tmpdb_copy_tag(tag_rating);
2351                 tmpdb_copy_tag(tag_playtime);
2352                 tmpdb_copy_tag(tag_lastplayed);
2353                 tmpdb_copy_tag(tag_commitid);
2354                 tmpdb_copy_tag(tag_lastoffset);
2355
2356                 /* Avoid processing this entry again. */
2357                 idx.flag |= FLAG_RESURRECTED;
2358
2359                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2360                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2361                 {
2362                     logf("masterfd writeback fail #1");
2363                     close(masterfd);
2364                     return false;
2365                 }
2366
2367                 logf("Entry resurrected");
2368             }
2369         }
2370
2371
2372         /* Restore the master index position. */
2373         lseek(masterfd, masterfd_pos, SEEK_SET);
2374
2375         /* Commit the data to the index. */
2376         for (i = 0; i < count; i++)
2377         {
2378             int loc = lseek(masterfd, 0, SEEK_CUR);
2379
2380             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2381             {
2382                 logf("read fail #3");
2383                 close(masterfd);
2384                 return false;
2385             }
2386
2387             for (j = 0; j < TAG_COUNT; j++)
2388             {
2389                 if (!TAGCACHE_IS_NUMERIC(j))
2390                     continue;
2391
2392                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2393             }
2394             idx.flag = entrybuf[i].flag;
2395
2396             if (idx.tag_seek[tag_commitid])
2397             {
2398                 /* Data has been resurrected. */
2399                 idx.flag |= FLAG_DIRTYNUM;
2400             }
2401             else if (tc_stat.ready && current_tcmh.commitid > 0)
2402             {
2403                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2404                 idx.flag |= FLAG_DIRTYNUM;
2405             }
2406
2407             /* Write back the updated index. */
2408             lseek(masterfd, loc, SEEK_SET);
2409             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2410             {
2411                 logf("write fail");
2412                 close(masterfd);
2413                 return false;
2414             }
2415         }
2416
2417         entries_processed += count;
2418         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2419     }
2420
2421     close(masterfd);
2422
2423     return true;
2424 }
2425
2426 /**
2427  * Return values:
2428  *     > 0   success
2429  *    == 0   temporary failure
2430  *     < 0   fatal error
2431  */
2432 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2433 {
2434     int i;
2435     struct tagcache_header tch;
2436     struct master_header   tcmh;
2437     struct index_entry idxbuf[IDX_BUF_DEPTH];
2438     int idxbuf_pos;
2439     char buf[TAG_MAXLEN+32];
2440     int fd = -1, masterfd;
2441     bool error = false;
2442     int init;
2443     int masterfd_pos;
2444
2445     logf("Building index: %d", index_type);
2446
2447     /* Check the number of entries we need to allocate ram for. */
2448     commit_entry_count = h->entry_count + 1;
2449
2450     masterfd = open_master_fd(&tcmh, false);
2451     if (masterfd >= 0)
2452     {
2453         commit_entry_count += tcmh.tch.entry_count;
2454         close(masterfd);
2455     }
2456     else
2457         remove_files(); /* Just to be sure we are clean. */
2458
2459     /* Open the index file, which contains the tag names. */
2460     fd = open_tag_fd(&tch, index_type, true);
2461     if (fd >= 0)
2462     {
2463         logf("tch.datasize=%ld", tch.datasize);
2464         lookup_buffer_depth = 1 +
2465         /* First part */ commit_entry_count +
2466         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2467     }
2468     else
2469     {
2470         lookup_buffer_depth = 1 +
2471         /* First part */ commit_entry_count +
2472         /* Second part */ 0;
2473     }
2474
2475     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2476     logf("commit_entry_count=%ld", commit_entry_count);
2477
2478     /* Allocate buffer for all index entries from both old and new
2479      * tag files. */
2480     tempbufidx = 0;
2481     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2482
2483     /* Allocate lookup buffer. The first portion of commit_entry_count
2484      * contains the new tags in the temporary file and the second
2485      * part for locating entries already in the db.
2486      *
2487      *  New tags  Old tags
2488      * +---------+---------------------------+
2489      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2490      * +---------+---------------------------+
2491      *
2492      * Old tags are inserted to a temporary buffer with position:
2493      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2494      * And new tags with index:
2495      *     tempbuf_insert(idx, ...);
2496      *
2497      * The buffer is sorted and written into tag file:
2498      *     tempbuf_sort(...);
2499      * leaving master index locations messed up.
2500      *
2501      * That is fixed using the lookup buffer for old tags:
2502      *     new_seek = tempbuf_find_location(old_seek, ...);
2503      * and for new tags:
2504      *     new_seek = tempbuf_find_location(idx);
2505      */
2506     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2507     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2508     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2509
2510     /* And calculate the remaining data space used mainly for storing
2511      * tag data (strings). */
2512     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2513     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2514     {
2515         logf("Buffer way too small!");
2516         return 0;
2517     }
2518
2519     if (fd >= 0)
2520     {
2521         /**
2522          * If tag file contains unique tags (sorted index), we will load
2523          * it entirely into memory so we can resort it later for use with
2524          * chunked browsing.
2525          */
2526         if (TAGCACHE_IS_SORTED(index_type))
2527         {
2528             logf("loading tags...");
2529             for (i = 0; i < tch.entry_count; i++)
2530             {
2531                 struct tagfile_entry entry;
2532                 int loc = lseek(fd, 0, SEEK_CUR);
2533                 bool ret;
2534
2535                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2536                 {
2537                     logf("read error #7");
2538                     close(fd);
2539                     return -2;
2540                 }
2541
2542                 if (entry.tag_length >= (int)sizeof(buf))
2543                 {
2544                     logf("too long tag #3");
2545                     close(fd);
2546                     return -2;
2547                 }
2548
2549                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2550                 {
2551                     logf("read error #8");
2552                     close(fd);
2553                     return -2;
2554                 }
2555
2556                 /* Skip deleted entries. */
2557                 if (buf[0] == '\0')
2558                     continue;
2559
2560                 /**
2561                  * Save the tag and tag id in the memory buffer. Tag id
2562                  * is saved so we can later reindex the master lookup
2563                  * table when the index gets resorted.
2564                  */
2565                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2566                                      + commit_entry_count, entry.idx_id,
2567                                      TAGCACHE_IS_UNIQUE(index_type));
2568                 if (!ret)
2569                 {
2570                     close(fd);
2571                     return -3;
2572                 }
2573                 do_timed_yield();
2574             }
2575             logf("done");
2576         }
2577         else
2578             tempbufidx = tch.entry_count;
2579     }
2580     else
2581     {
2582         /**
2583          * Creating new index file to store the tags. No need to preload
2584          * anything whether the index type is sorted or not.
2585          */
2586         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2587         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2588         if (fd < 0)
2589         {
2590             logf("%s open fail", buf);
2591             return -2;
2592         }
2593
2594         tch.magic = TAGCACHE_MAGIC;
2595         tch.entry_count = 0;
2596         tch.datasize = 0;
2597
2598         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2599             != sizeof(struct tagcache_header))
2600         {
2601             logf("header write failed");
2602             close(fd);
2603             return -2;
2604         }
2605     }
2606
2607     /* Loading the tag lookup file as "master file". */
2608     logf("Loading index file");
2609     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2610
2611     if (masterfd < 0)
2612     {
2613         logf("Creating new DB");
2614         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2615
2616         if (masterfd < 0)
2617         {
2618             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2619             close(fd);
2620             return -2;
2621         }
2622
2623         /* Write the header (write real values later). */
2624         memset(&tcmh, 0, sizeof(struct master_header));
2625         tcmh.tch = *h;
2626         tcmh.tch.entry_count = 0;
2627         tcmh.tch.datasize = 0;
2628         tcmh.dirty = true;
2629         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2630         init = true;
2631         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2632     }
2633     else
2634     {
2635         /**
2636          * Master file already exists so we need to process the current
2637          * file first.
2638          */
2639         init = false;
2640
2641         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2642             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2643         {
2644             logf("header error");
2645             close(fd);
2646             close(masterfd);
2647             return -2;
2648         }
2649
2650         /**
2651          * If we reach end of the master file, we need to expand it to
2652          * hold new tags. If the current index is not sorted, we can
2653          * simply append new data to end of the file.
2654          * However, if the index is sorted, we need to update all tag
2655          * pointers in the master file for the current index.
2656          */
2657         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2658             SEEK_CUR);
2659         if (masterfd_pos == filesize(masterfd))
2660         {
2661             logf("appending...");
2662             init = true;
2663         }
2664     }
2665
2666     /**
2667      * Load new unique tags in memory to be sorted later and added
2668      * to the master lookup file.
2669      */
2670     if (TAGCACHE_IS_SORTED(index_type))
2671     {
2672         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2673         /* h is the header of the temporary file containing new tags. */
2674         logf("inserting new tags...");
2675         for (i = 0; i < h->entry_count; i++)
2676         {
2677             struct temp_file_entry entry;
2678
2679             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2680                 sizeof(struct temp_file_entry))
2681             {
2682                 logf("read fail #3");
2683                 error = true;
2684                 goto error_exit;
2685             }
2686
2687             /* Read data. */
2688             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2689             {
2690                 logf("too long entry!");
2691                 error = true;
2692                 goto error_exit;
2693             }
2694
2695             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2696             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2697                 entry.tag_length[index_type])
2698             {
2699                 logf("read fail #4");
2700                 error = true;
2701                 goto error_exit;
2702             }
2703
2704             if (TAGCACHE_IS_UNIQUE(index_type))
2705                 error = !tempbuf_insert(buf, i, -1, true);
2706             else
2707                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2708
2709             if (error)
2710             {
2711                 logf("insert error");
2712                 goto error_exit;
2713             }
2714
2715             /* Skip to next. */
2716             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2717                     entry.tag_length[index_type], SEEK_CUR);
2718             do_timed_yield();
2719         }
2720         logf("done");
2721
2722         /* Sort the buffer data and write it to the index file. */
2723         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2724         /**
2725          * We need to truncate the index file now. There can be junk left
2726          * at the end of file (however, we _should_ always follow the
2727          * entry_count and don't crash with that).
2728          */
2729         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2730
2731         i = tempbuf_sort(fd);
2732         if (i < 0)
2733             goto error_exit;
2734         logf("sorted %d tags", i);
2735
2736         /**
2737          * Now update all indexes in the master lookup file.
2738          */
2739         logf("updating indices...");
2740         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2741         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2742         {
2743             int j;
2744             int loc = lseek(masterfd, 0, SEEK_CUR);
2745
2746             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2747
2748             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2749                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2750             {
2751                 logf("read fail #5");
2752                 error = true;
2753                 goto error_exit ;
2754             }
2755             lseek(masterfd, loc, SEEK_SET);
2756
2757             for (j = 0; j < idxbuf_pos; j++)
2758             {
2759                 if (idxbuf[j].flag & FLAG_DELETED)
2760                 {
2761                     /* We can just ignore deleted entries. */
2762                     // idxbuf[j].tag_seek[index_type] = 0;
2763                     continue;
2764                 }
2765
2766                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2767                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2768                     + commit_entry_count);
2769
2770                 if (idxbuf[j].tag_seek[index_type] < 0)
2771                 {
2772                     logf("update error: %ld/%d/%ld",
2773                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2774                     error = true;
2775                     goto error_exit;
2776                 }
2777
2778                 do_timed_yield();
2779             }
2780
2781             /* Write back the updated index. */
2782             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2783                         index_entry_ec, tc_stat.econ) !=
2784                 (int)sizeof(struct index_entry)*idxbuf_pos)
2785             {
2786                 logf("write fail");
2787                 error = true;
2788                 goto error_exit;
2789             }
2790         }
2791         logf("done");
2792     }
2793
2794     /**
2795      * Walk through the temporary file containing the new tags.
2796      */
2797     // build_normal_index(h, tmpfd, masterfd, idx);
2798     logf("updating new indices...");
2799     lseek(masterfd, masterfd_pos, SEEK_SET);
2800     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2801     lseek(fd, 0, SEEK_END);
2802     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2803     {
2804         int j;
2805
2806         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2807         if (init)
2808         {
2809             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2810         }
2811         else
2812         {
2813             int loc = lseek(masterfd, 0, SEEK_CUR);
2814
2815             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2816                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2817             {
2818                 logf("read fail #6");
2819                 error = true;
2820                 break ;
2821             }
2822             lseek(masterfd, loc, SEEK_SET);
2823         }
2824
2825         /* Read entry headers. */
2826         for (j = 0; j < idxbuf_pos; j++)
2827         {
2828             if (!TAGCACHE_IS_SORTED(index_type))
2829             {
2830                 struct temp_file_entry entry;
2831                 struct tagfile_entry fe;
2832
2833                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2834                     sizeof(struct temp_file_entry))
2835                 {
2836                     logf("read fail #7");
2837                     error = true;
2838                     break ;
2839                 }
2840
2841                 /* Read data. */
2842                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2843                 {
2844                     logf("too long entry!");
2845                     logf("length=%d", entry.tag_length[index_type]);
2846                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2847                     error = true;
2848                     break ;
2849                 }
2850
2851                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2852                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2853                     entry.tag_length[index_type])
2854                 {
2855                     logf("read fail #8");
2856                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2857                     logf("length=0x%02x", entry.tag_length[index_type]);
2858                     error = true;
2859                     break ;
2860                 }
2861
2862                 /* Write to index file. */
2863                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2864                 fe.tag_length = entry.tag_length[index_type];
2865                 fe.idx_id = tcmh.tch.entry_count + i + j;
2866                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2867                 write(fd, buf, fe.tag_length);
2868                 tempbufidx++;
2869
2870                 /* Skip to next. */
2871                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2872                       entry.tag_length[index_type], SEEK_CUR);
2873             }
2874             else
2875             {
2876                 /* Locate the correct entry from the sorted array. */
2877                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2878                 if (idxbuf[j].tag_seek[index_type] < 0)
2879                 {
2880                     logf("entry not found (%d)", j);
2881                     error = true;
2882                     break ;
2883                 }
2884             }
2885         }
2886
2887         /* Write index. */
2888         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2889                   index_entry_ec, tc_stat.econ) !=
2890             (int)sizeof(struct index_entry)*idxbuf_pos)
2891         {
2892             logf("tagcache: write fail #4");
2893             error = true;
2894             break ;
2895         }
2896
2897         do_timed_yield();
2898     }
2899     logf("done");
2900
2901     /* Finally write the header. */
2902     tch.magic = TAGCACHE_MAGIC;
2903     tch.entry_count = tempbufidx;
2904     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2905     lseek(fd, 0, SEEK_SET);
2906     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2907
2908     if (index_type != tag_filename)
2909         h->datasize += tch.datasize;
2910     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2911     error_exit:
2912
2913     close(fd);
2914     close(masterfd);
2915
2916     if (error)
2917         return -2;
2918
2919     return 1;
2920 }
2921
2922 static bool commit(void)
2923 {
2924     struct tagcache_header tch;
2925     struct master_header   tcmh;
2926     int i, len, rc;
2927     int tmpfd;
2928     int masterfd;
2929 #ifdef HAVE_DIRCACHE
2930     bool dircache_buffer_stolen = false;
2931 #endif
2932     bool local_allocation = false;
2933
2934     logf("committing tagcache");
2935
2936     while (write_lock)
2937         sleep(1);
2938
2939     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2940     if (tmpfd < 0)
2941     {
2942         logf("nothing to commit");
2943         return true;
2944     }
2945
2946
2947     /* Load the header. */
2948     len = sizeof(struct tagcache_header);
2949     rc = read(tmpfd, &tch, len);
2950
2951     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2952     {
2953         logf("incorrect tmpheader");
2954         close(tmpfd);
2955         remove(TAGCACHE_FILE_TEMP);
2956         return false;
2957     }
2958
2959     if (tch.entry_count == 0)
2960     {
2961         logf("nothing to commit");
2962         close(tmpfd);
2963         remove(TAGCACHE_FILE_TEMP);
2964         return true;
2965     }
2966
2967     /* Fully initialize existing headers (if any) before going further. */
2968     tc_stat.ready = check_all_headers();
2969
2970 #ifdef HAVE_EEPROM_SETTINGS
2971     remove(TAGCACHE_STATEFILE);
2972 #endif
2973
2974     /* At first be sure to unload the ramcache! */
2975 #ifdef HAVE_TC_RAMCACHE
2976     tc_stat.ramcache = false;
2977 #endif
2978
2979     read_lock++;
2980
2981     /* Try to steal every buffer we can :) */
2982     if (tempbuf_size == 0)
2983         local_allocation = true;
2984
2985 #ifdef HAVE_DIRCACHE
2986     if (tempbuf_size == 0)
2987     {
2988         /* Try to steal the dircache buffer. */
2989         tempbuf = dircache_steal_buffer(&tempbuf_size);
2990         tempbuf_size &= ~0x03;
2991
2992         if (tempbuf_size > 0)
2993         {
2994             dircache_buffer_stolen = true;
2995         }
2996     }
2997 #endif
2998
2999 #ifdef HAVE_TC_RAMCACHE
3000     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
3001     {
3002         tempbuf = (char *)(hdr + 1);
3003         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
3004         tempbuf_size &= ~0x03;
3005     }
3006 #endif
3007
3008     /* And finally fail if there are no buffers available. */
3009     if (tempbuf_size == 0)
3010     {
3011         logf("delaying commit until next boot");
3012         tc_stat.commit_delayed = true;
3013         close(tmpfd);
3014         read_lock--;
3015         return false;
3016     }
3017
3018     logf("commit %ld entries...", tch.entry_count);
3019
3020     /* Mark DB dirty so it will stay disabled if commit fails. */
3021     current_tcmh.dirty = true;
3022     update_master_header();
3023
3024     /* Now create the index files. */
3025     tc_stat.commit_step = 0;
3026     tch.datasize = 0;
3027     tc_stat.commit_delayed = false;
3028
3029     for (i = 0; i < TAG_COUNT; i++)
3030     {
3031         int ret;
3032
3033         if (TAGCACHE_IS_NUMERIC(i))
3034             continue;
3035
3036         tc_stat.commit_step++;
3037         ret = build_index(i, &tch, tmpfd);
3038         if (ret <= 0)
3039         {
3040             close(tmpfd);
3041             logf("tagcache failed init");
3042             if (ret == 0)
3043                 tc_stat.commit_delayed = true;
3044
3045             tc_stat.commit_step = 0;
3046             read_lock--;
3047             return false;
3048         }
3049     }
3050
3051     if (!build_numeric_indices(&tch, tmpfd))
3052     {
3053         logf("Failure to commit numeric indices");
3054         close(tmpfd);
3055         tc_stat.commit_step = 0;
3056         read_lock--;
3057         return false;
3058     }
3059
3060     close(tmpfd);
3061
3062     tc_stat.commit_step = 0;
3063
3064     /* Update the master index headers. */
3065     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3066     {
3067         read_lock--;
3068         return false;
3069     }
3070
3071     remove(TAGCACHE_FILE_TEMP);
3072
3073     tcmh.tch.entry_count += tch.entry_count;
3074     tcmh.tch.datasize = sizeof(struct master_header)
3075         + sizeof(struct index_entry) * tcmh.tch.entry_count
3076         + tch.datasize;
3077     tcmh.dirty = false;
3078     tcmh.commitid++;
3079
3080     lseek(masterfd, 0, SEEK_SET);
3081     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3082     close(masterfd);
3083
3084     logf("tagcache committed");
3085     tc_stat.ready = check_all_headers();
3086     tc_stat.readyvalid = true;
3087
3088     if (local_allocation)
3089     {
3090         tempbuf = NULL;
3091         tempbuf_size = 0;
3092     }
3093
3094 #ifdef HAVE_DIRCACHE
3095     /* Rebuild the dircache, if we stole the buffer. */
3096     if (dircache_buffer_stolen)
3097         dircache_build(0);
3098 #endif
3099
3100 #ifdef HAVE_TC_RAMCACHE
3101     /* Reload tagcache. */
3102     if (tc_stat.ramcache_allocated > 0)
3103         tagcache_start_scan();
3104 #endif
3105
3106     read_lock--;
3107
3108     return true;
3109 }
3110
3111 static void allocate_tempbuf(void)
3112 {
3113     /* Yeah, malloc would be really nice now :) */
3114 #ifdef __PCTOOL__
3115     tempbuf_size = 32*1024*1024;
3116     tempbuf = malloc(tempbuf_size);
3117 #else
3118     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3119     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3120     audiobuf += tempbuf_size;
3121 #endif
3122 }
3123
3124 static void free_tempbuf(void)
3125 {
3126     if (tempbuf_size == 0)
3127         return ;
3128
3129 #ifdef __PCTOOL__
3130     free(tempbuf);
3131 #else
3132     audiobuf -= tempbuf_size;
3133 #endif
3134     tempbuf = NULL;
3135     tempbuf_size = 0;
3136 }
3137
3138 #ifndef __PCTOOL__
3139
3140 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3141 {
3142     struct index_entry idx;
3143
3144     if (!tc_stat.ready)
3145         return false;
3146
3147     if (!TAGCACHE_IS_NUMERIC(tag))
3148         return false;
3149
3150     if (!get_index(masterfd, idx_id, &idx, false))
3151         return false;
3152
3153     idx.tag_seek[tag] = data;
3154     idx.flag |= FLAG_DIRTYNUM;
3155
3156     return write_index(masterfd, idx_id, &idx);
3157 }
3158
3159 #if 0
3160 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3161                                    int tag, long data)
3162 {
3163     struct master_header myhdr;
3164
3165     if (tcs->masterfd < 0)
3166     {
3167         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3168             return false;
3169     }
3170
3171     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3172 }
3173 #endif
3174
3175 static bool command_queue_is_full(void)
3176 {
3177     int next;
3178
3179     next = command_queue_widx + 1;
3180     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3181         next = 0;
3182
3183     return (next == command_queue_ridx);
3184 }
3185
3186 static void command_queue_sync_callback(void *data)
3187 {
3188     (void)data;
3189     struct master_header myhdr;
3190     int masterfd;
3191
3192     mutex_lock(&command_queue_mutex);
3193
3194     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3195         return;
3196
3197     while (command_queue_ridx != command_queue_widx)
3198     {
3199         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3200
3201         switch (ce->command)
3202         {
3203             case CMD_UPDATE_MASTER_HEADER:
3204             {
3205                 close(masterfd);
3206                 update_master_header();
3207
3208                 /* Re-open the masterfd. */
3209                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3210                     return;
3211
3212                 break;
3213             }
3214             case CMD_UPDATE_NUMERIC:
3215             {
3216                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3217                 break;
3218             }
3219         }
3220
3221         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3222             command_queue_ridx = 0;
3223     }
3224
3225     close(masterfd);
3226
3227     tc_stat.queue_length = 0;
3228     mutex_unlock(&command_queue_mutex);
3229 }
3230
3231 static void run_command_queue(bool force)
3232 {
3233     if (COMMAND_QUEUE_IS_EMPTY)
3234         return;
3235
3236     if (force || command_queue_is_full())
3237         command_queue_sync_callback(NULL);
3238     else
3239         register_storage_idle_func(command_queue_sync_callback);
3240 }
3241
3242 static void queue_command(int cmd, long idx_id, int tag, long data)
3243 {
3244     while (1)
3245     {
3246         int next;
3247
3248         mutex_lock(&command_queue_mutex);
3249         next = command_queue_widx + 1;
3250         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3251             next = 0;
3252
3253         /* Make sure queue is not full. */
3254         if (next != command_queue_ridx)
3255         {
3256             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3257
3258             ce->command = cmd;
3259             ce->idx_id = idx_id;
3260             ce->tag = tag;
3261             ce->data = data;
3262
3263             command_queue_widx = next;
3264
3265             tc_stat.queue_length++;
3266
3267             mutex_unlock(&command_queue_mutex);
3268             break;
3269         }
3270
3271         /* Queue is full, try again later... */
3272         mutex_unlock(&command_queue_mutex);
3273         sleep(1);
3274     }
3275 }
3276
3277 long tagcache_increase_serial(void)
3278 {
3279     long old;
3280
3281     if (!tc_stat.ready)
3282         return -2;
3283
3284     while (read_lock)
3285         sleep(1);
3286
3287     old = current_tcmh.serial++;
3288     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3289
3290     return old;
3291 }
3292
3293 void tagcache_update_numeric(int idx_id, int tag, long data)
3294 {
3295     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3296 }
3297 #endif /* !__PCTOOL__ */
3298
3299 long tagcache_get_serial(void)
3300 {
3301     return current_tcmh.serial;
3302 }
3303
3304 long tagcache_get_commitid(void)
3305 {
3306     return current_tcmh.commitid;
3307 }
3308
3309 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3310 {
3311     char buf[512];
3312     int i;
3313
3314     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3315     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3316     {
3317         if (*datastr == '\0')
3318             break;
3319
3320         if (*datastr == '"' || *datastr == '\\')
3321             buf[i++] = '\\';
3322
3323         buf[i] = *(datastr++);
3324     }
3325
3326     strcpy(&buf[i], "\" ");
3327
3328     write(fd, buf, i + 2);
3329
3330     return true;
3331 }
3332
3333 #ifndef __PCTOOL__
3334
3335 static bool read_tag(char *dest, long size,
3336                      const char *src, const char *tagstr)
3337 {
3338     int pos;
3339     char current_tag[32];
3340
3341     while (*src != '\0')
3342     {
3343         /* Skip all whitespace */
3344         while (*src == ' ')
3345             src++;
3346
3347         if (*src == '\0')
3348             break;
3349
3350         pos = 0;
3351         /* Read in tag name */
3352         while (*src != '=' && *src != ' ')
3353         {
3354             current_tag[pos] = *src;
3355             src++;
3356             pos++;
3357
3358             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3359                 return false;
3360         }
3361         current_tag[pos] = '\0';
3362
3363         /* Read in tag data */
3364
3365         /* Find the start. */
3366         while (*src != '"' && *src != '\0')
3367             src++;
3368
3369         if (*src == '\0' || *(++src) == '\0')
3370             return false;
3371
3372         /* Read the data. */
3373         for (pos = 0; pos < size; pos++)
3374         {
3375             if (*src == '\0')
3376                 break;
3377
3378             if (*src == '\\')
3379             {
3380                 dest[pos] = *(src+1);
3381                 src += 2;
3382                 continue;
3383             }
3384
3385             dest[pos] = *src;
3386
3387             if (*src == '"')
3388             {
3389                 src++;
3390                 break;
3391             }
3392
3393             if (*src == '\0')
3394                 break;
3395
3396             src++;
3397         }
3398         dest[pos] = '\0';
3399
3400         if (!strcasecmp(tagstr, current_tag))
3401             return true;
3402     }
3403
3404     return false;
3405 }
3406
3407 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3408 {
3409     struct index_entry idx;
3410     char tag_data[TAG_MAXLEN+32];
3411     int idx_id;
3412     long masterfd = (long)parameters;
3413     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3414                                 tag_lastplayed, tag_commitid, tag_lastoffset };
3415     int i;
3416     (void)line_n;
3417
3418     if (*buf == '#')
3419         return 0;
3420
3421     logf("%d/%s", line_n, buf);
3422     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3423     {
3424         logf("filename missing");
3425         logf("-> %s", buf);
3426         return 0;
3427     }
3428
3429     idx_id = find_index(tag_data);
3430     if (idx_id < 0)
3431     {
3432         logf("entry not found");
3433         return 0;
3434     }
3435
3436     if (!get_index(masterfd, idx_id, &idx, false))
3437     {
3438         logf("failed to retrieve index entry");
3439         return 0;
3440     }
3441
3442     /* Stop if tag has already been modified. */
3443     if (idx.flag & FLAG_DIRTYNUM)
3444         return 0;
3445
3446     logf("import: %s", tag_data);
3447
3448     idx.flag |= FLAG_DIRTYNUM;
3449     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3450     {
3451         int data;
3452
3453         if (!read_tag(tag_data, sizeof tag_data, buf,
3454                       tagcache_tag_to_str(import_tags[i])))
3455         {
3456             continue;
3457         }
3458
3459         data = atoi(tag_data);
3460         if (data < 0)
3461             continue;
3462
3463         idx.tag_seek[import_tags[i]] = data;
3464
3465         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3466             current_tcmh.serial = data + 1;
3467         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3468             current_tcmh.commitid = data + 1;
3469     }
3470
3471     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3472 }
3473
3474 bool tagcache_import_changelog(void)
3475 {
3476     struct master_header myhdr;
3477     struct tagcache_header tch;
3478     int clfd;
3479     long masterfd;
3480     char buf[2048];
3481
3482     if (!tc_stat.ready)
3483         return false;
3484
3485     while (read_lock)
3486         sleep(1);
3487
3488     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3489     if (clfd < 0)
3490     {
3491         logf("failure to open changelog");
3492         return false;
3493     }
3494
3495     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3496     {
3497         close(clfd);
3498         return false;
3499     }
3500
3501     write_lock++;
3502
3503     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3504
3505     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3506                   parse_changelog_line);
3507
3508     close(clfd);
3509     close(masterfd);
3510
3511     if (filenametag_fd >= 0)
3512     {
3513         close(filenametag_fd);
3514         filenametag_fd = -1;
3515     }
3516
3517     write_lock--;
3518
3519     update_master_header();
3520
3521     return true;
3522 }
3523
3524 #endif /* !__PCTOOL__ */
3525
3526 bool tagcache_create_changelog(struct tagcache_search *tcs)
3527 {
3528     struct master_header myhdr;
3529     struct index_entry idx;
3530     char buf[TAG_MAXLEN+32];
3531     char temp[32];
3532     int clfd;
3533     int i, j;
3534
3535     if (!tc_stat.ready)
3536         return false;
3537
3538     if (!tagcache_search(tcs, tag_filename))
3539         return false;
3540
3541     /* Initialize the changelog */
3542     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3543     if (clfd < 0)
3544     {
3545         logf("failure to open changelog");
3546         return false;
3547     }
3548
3549     if (tcs->masterfd < 0)
3550     {
3551         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3552             return false;
3553     }
3554     else
3555     {
3556         lseek(tcs->masterfd, 0, SEEK_SET);
3557         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3558     }
3559
3560     write(clfd, "## Changelog version 1\n", 23);
3561
3562     for (i = 0; i < myhdr.tch.entry_count; i++)
3563     {
3564         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3565         {
3566             logf("read error #9");
3567             tagcache_search_finish(tcs);
3568             close(clfd);
3569             return false;
3570         }
3571
3572         /* Skip until the entry found has been modified. */
3573         if (! (idx.flag & FLAG_DIRTYNUM) )
3574             continue;
3575
3576         /* Skip deleted entries too. */
3577         if (idx.flag & FLAG_DELETED)
3578             continue;
3579
3580         /* Now retrieve all tags. */
3581         for (j = 0; j < TAG_COUNT; j++)
3582         {
3583             if (TAGCACHE_IS_NUMERIC(j))
3584             {
3585                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3586                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3587                 continue;
3588             }
3589
3590             tcs->type = j;
3591             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3592             write_tag(clfd, tagcache_tag_to_str(j), buf);
3593         }
3594
3595         write(clfd, "\n", 1);
3596         do_timed_yield();
3597     }
3598
3599     close(clfd);
3600
3601     tagcache_search_finish(tcs);
3602
3603     return true;
3604 }
3605
3606 static bool delete_entry(long idx_id)
3607 {
3608     int fd = -1;
3609     int masterfd = -1;
3610     int tag, i;
3611     struct index_entry idx, myidx;
3612     struct master_header myhdr;
3613     char buf[TAG_MAXLEN+32];
3614     int in_use[TAG_COUNT];
3615
3616     logf("delete_entry(): %ld", idx_id);
3617
3618 #ifdef HAVE_TC_RAMCACHE
3619     /* At first mark the entry removed from ram cache. */
3620     if (tc_stat.ramcache)
3621         hdr->indices[idx_id].flag |= FLAG_DELETED;
3622 #endif
3623
3624     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3625         return false;
3626
3627     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3628     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3629     {
3630         logf("delete_entry(): read error");
3631         goto cleanup;
3632     }
3633
3634     if (myidx.flag & FLAG_DELETED)
3635     {
3636         logf("delete_entry(): already deleted!");
3637         goto cleanup;
3638     }
3639
3640     myidx.flag |= FLAG_DELETED;
3641     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3642     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3643     {
3644         logf("delete_entry(): write_error #1");
3645         goto cleanup;
3646     }
3647
3648     /* Now check which tags are no longer in use (if any) */
3649     for (tag = 0; tag < TAG_COUNT; tag++)
3650         in_use[tag] = 0;
3651
3652     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3653     for (i = 0; i < myhdr.tch.entry_count; i++)
3654     {
3655         struct index_entry *idxp;
3656
3657 #ifdef HAVE_TC_RAMCACHE
3658         /* Use RAM DB if available for greater speed */
3659         if (tc_stat.ramcache)
3660             idxp = &hdr->indices[i];
3661         else
3662 #endif
3663         {
3664             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3665             {
3666                 logf("delete_entry(): read error #2");
3667                 goto cleanup;
3668             }
3669             idxp = &idx;
3670         }
3671
3672         if (idxp->flag & FLAG_DELETED)
3673             continue;
3674
3675         for (tag = 0; tag < TAG_COUNT; tag++)
3676         {
3677             if (TAGCACHE_IS_NUMERIC(tag))
3678                 continue;
3679
3680             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3681                 in_use[tag]++;
3682         }
3683     }
3684
3685     /* Now delete all tags no longer in use. */
3686     for (tag = 0; tag < TAG_COUNT; tag++)
3687     {
3688         struct tagcache_header tch;
3689         int oldseek = myidx.tag_seek[tag];
3690
3691         if (TAGCACHE_IS_NUMERIC(tag))
3692             continue;
3693
3694         /**
3695          * Replace tag seek with a hash value of the field string data.
3696          * That way runtime statistics of moved or altered files can be
3697          * resurrected.
3698          */
3699 #ifdef HAVE_TC_RAMCACHE
3700         if (tc_stat.ramcache && tag != tag_filename)
3701         {
3702             struct tagfile_entry *tfe;
3703             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3704
3705             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3706             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3707             myidx.tag_seek[tag] = *seek;
3708         }
3709         else
3710 #endif
3711         {
3712             struct tagfile_entry tfe;
3713
3714             /* Open the index file, which contains the tag names. */
3715             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3716                 goto cleanup;
3717
3718             /* Skip the header block */
3719             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3720             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3721             {
3722                 logf("delete_entry(): read error #3");
3723                 goto cleanup;
3724             }
3725
3726             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3727             {
3728                 logf("delete_entry(): read error #4");
3729                 goto cleanup;
3730             }
3731
3732             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3733         }
3734
3735         if (in_use[tag])
3736         {
3737             logf("in use: %d/%d", tag, in_use[tag]);
3738             if (fd >= 0)
3739             {
3740                 close(fd);
3741                 fd = -1;
3742             }
3743             continue;
3744         }
3745
3746 #ifdef HAVE_TC_RAMCACHE
3747         /* Delete from ram. */
3748         if (tc_stat.ramcache && tag != tag_filename)
3749         {
3750             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3751             tagentry->tag_data[0] = '\0';
3752         }
3753 #endif
3754
3755         /* Open the index file, which contains the tag names. */
3756         if (fd < 0)
3757         {
3758             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3759                 goto cleanup;
3760         }
3761
3762         /* Skip the header block */
3763         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3764
3765         /* Debug, print 10 first characters of the tag
3766         read(fd, buf, 10);
3767         buf[10]='\0';
3768         logf("TAG:%s", buf);
3769         lseek(fd, -10, SEEK_CUR);
3770         */
3771
3772         /* Write first data byte in tag as \0 */
3773         write(fd, "", 1);
3774
3775         /* Now tag data has been removed */
3776         close(fd);
3777         fd = -1;
3778     }
3779
3780     /* Write index entry back into master index. */
3781     lseek(masterfd, sizeof(struct master_header) +
3782           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3783     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3784     {
3785         logf("delete_entry(): write_error #2");
3786         goto cleanup;
3787     }
3788
3789     close(masterfd);
3790
3791     return true;
3792
3793     cleanup:
3794     if (fd >= 0)
3795         close(fd);
3796     if (masterfd >= 0)
3797         close(masterfd);
3798
3799     return false;
3800 }
3801
3802 #ifndef __PCTOOL__
3803 /**
3804  * Returns true if there is an event waiting in the queue
3805  * that requires the current operation to be aborted.
3806  */
3807 static bool check_event_queue(void)
3808 {
3809     struct queue_event ev;
3810
3811     if(!queue_peek(&tagcache_queue, &ev))
3812         return false;
3813
3814     switch (ev.id)
3815     {
3816         case Q_STOP_SCAN:
3817         case SYS_POWEROFF:
3818         case SYS_USB_CONNECTED:
3819             return true;
3820     }
3821
3822     return false;
3823 }
3824 #endif
3825
3826 #ifdef HAVE_TC_RAMCACHE
3827 static bool allocate_tagcache(void)
3828 {
3829     struct master_header tcmh;
3830     int fd;
3831
3832     /* Load the header. */
3833     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3834     {
3835         hdr = NULL;
3836         return false;
3837     }
3838
3839     close(fd);
3840
3841     /**
3842      * Now calculate the required cache size plus
3843      * some extra space for alignment fixes.
3844      */
3845     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3846         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3847     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3848     memset(hdr, 0, sizeof(struct ramcache_header));
3849     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3850     hdr->indices = (struct index_entry *)(hdr + 1);
3851     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3852
3853     return true;
3854 }
3855
3856 # ifdef HAVE_EEPROM_SETTINGS
3857 static bool tagcache_dumpload(void)
3858 {
3859     struct statefile_header shdr;
3860     int fd, rc;
3861     long offpos;
3862     int i;
3863
3864     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3865     if (fd < 0)
3866     {
3867         logf("no tagcache statedump");
3868         return false;
3869     }
3870
3871     /* Check the statefile memory placement */
3872     hdr = buffer_alloc(0);
3873     rc = read(fd, &shdr, sizeof(struct statefile_header));
3874     if (rc != sizeof(struct statefile_header)
3875         /* || (long)hdr != (long)shdr.hdr */)
3876     {
3877         logf("incorrect statefile");
3878         hdr = NULL;
3879         close(fd);
3880         return false;
3881     }
3882
3883     offpos = (long)hdr - (long)shdr.hdr;
3884
3885     /* Lets allocate real memory and load it */
3886     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3887     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3888     close(fd);
3889
3890     if (rc != shdr.tc_stat.ramcache_allocated)
3891     {
3892         logf("read failure!");
3893         hdr = NULL;
3894         return false;
3895     }
3896
3897     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3898
3899     /* Now fix the pointers */
3900     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3901     for (i = 0; i < TAG_COUNT; i++)
3902         hdr->tags[i] += offpos;
3903
3904     return true;
3905 }
3906
3907 static bool tagcache_dumpsave(void)
3908 {
3909     struct statefile_header shdr;
3910     int fd;
3911
3912     if (!tc_stat.ramcache)
3913         return false;
3914
3915     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3916     if (fd < 0)
3917     {
3918         logf("failed to create a statedump");
3919         return false;
3920     }
3921
3922     /* Create the header */
3923     shdr.hdr = hdr;
3924     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3925     write(fd, &shdr, sizeof(struct statefile_header));
3926
3927     /* And dump the data too */
3928     write(fd, hdr, tc_stat.ramcache_allocated);
3929     close(fd);
3930
3931     return true;
3932 }
3933 # endif
3934
3935 static bool load_tagcache(void)
3936 {
3937     struct tagcache_header *tch;
3938     long bytesleft = tc_stat.ramcache_allocated;
3939     struct index_entry *idx;
3940     int rc, fd;
3941     char *p;
3942     int i, tag;
3943
3944 # ifdef HAVE_DIRCACHE
3945     while (dircache_is_initializing())
3946         sleep(1);
3947
3948     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3949 # endif
3950
3951     logf("loading tagcache to ram...");
3952
3953     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3954     if (fd < 0)
3955     {
3956         logf("tagcache open failed");
3957         return false;
3958     }
3959
3960     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3961         != sizeof(struct master_header)
3962         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3963     {
3964         logf("incorrect header");
3965         return false;
3966     }
3967
3968     idx = hdr->indices;
3969
3970     /* Load the master index table. */
3971     for (i = 0; i < hdr->h.tch.entry_count; i++)
3972     {
3973         rc = ecread_index_entry(fd, idx);
3974         if (rc != sizeof(struct index_entry))
3975         {
3976             logf("read error #10");
3977             close(fd);
3978             return false;
3979         }
3980
3981         bytesleft -= sizeof(struct index_entry);
3982         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3983         {
3984             logf("too big tagcache.");
3985             close(fd);
3986             return false;
3987         }
3988
3989         idx++;
3990     }
3991
3992     close(fd);
3993
3994     /* Load the tags. */
3995     p = (char *)idx;
3996     for (tag = 0; tag < TAG_COUNT; tag++)
3997     {
3998         struct tagfile_entry *fe;
3999         char buf[TAG_MAXLEN+32];
4000
4001         if (TAGCACHE_IS_NUMERIC(tag))
4002             continue ;
4003
4004         //p = ((void *)p+1);
4005         p = (char *)((long)p & ~0x03) + 0x04;
4006         hdr->tags[tag] = p;
4007
4008         /* Check the header. */
4009         tch = (struct tagcache_header *)p;
4010         p += sizeof(struct tagcache_header);
4011
4012         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
4013             return false;
4014
4015         for (hdr->entry_count[tag] = 0;
4016              hdr->entry_count[tag] < tch->entry_count;
4017              hdr->entry_count[tag]++)
4018         {
4019             long pos;
4020
4021             if (do_timed_yield())
4022             {
4023                 /* Abort if we got a critical event in queue */
4024                 if (check_event_queue())
4025                     return false;
4026             }
4027
4028             fe = (struct tagfile_entry *)p;
4029             pos = lseek(fd, 0, SEEK_CUR);
4030             rc = ecread_tagfile_entry(fd, fe);
4031             if (rc != sizeof(struct tagfile_entry))
4032             {
4033                 /* End of lookup table. */
4034                 logf("read error #11");
4035                 close(fd);
4036                 return false;
4037             }
4038
4039             /* We have a special handling for the filename tags. */
4040             if (tag == tag_filename)
4041             {
4042 # ifdef HAVE_DIRCACHE
4043                 const struct dircache_entry *dc;
4044 # endif
4045
4046                 // FIXME: This is wrong!
4047                 // idx = &hdr->indices[hdr->entry_count[i]];
4048                 idx = &hdr->indices[fe->idx_id];
4049
4050                 if (fe->tag_length >= (long)sizeof(buf)-1)
4051                 {
4052                     read(fd, buf, 10);
4053                     buf[10] = '\0';
4054                     logf("TAG:%s", buf);
4055                     logf("too long filename");
4056                     close(fd);
4057                     return false;
4058                 }
4059
4060                 rc = read(fd, buf, fe->tag_length);
4061                 if (rc != fe->tag_length)
4062                 {
4063                     logf("read error #12");
4064                     close(fd);
4065                     return false;
4066                 }
4067
4068                 /* Check if the entry has already been removed */
4069                 if (idx->flag & FLAG_DELETED)
4070                     continue;
4071
4072                 /* This flag must not be used yet. */
4073                 if (idx->flag & FLAG_DIRCACHE)
4074                 {
4075                     logf("internal error!");
4076                     close(fd);
4077                     return false;
4078                 }
4079
4080                 if (idx->tag_seek[tag] != pos)
4081                 {
4082                     logf("corrupt data structures!");
4083                     close(fd);
4084                     return false;
4085                 }
4086
4087 # ifdef HAVE_DIRCACHE
4088                 if (dircache_is_enabled())
4089                 {
4090                     dc = dircache_get_entry_ptr(buf);
4091                     if (dc == NULL)
4092                     {
4093                         logf("Entry no longer valid.");
4094                         logf("-> %s", buf);
4095                         if (global_settings.tagcache_autoupdate)
4096                             delete_entry(fe->idx_id);
4097                         continue ;
4098                     }
4099
4100                     idx->flag |= FLAG_DIRCACHE;
4101                     idx->tag_seek[tag_filename] = (long)dc;
4102                 }
4103                 else
4104 # endif
4105                 {
4106                     /* This will be very slow unless dircache is enabled
4107                        or target is flash based, but do it anyway for
4108                        consistency. */
4109                     /* Check if entry has been removed. */
4110                     if (global_settings.tagcache_autoupdate)
4111                     {
4112                         if (!file_exists(buf))
4113                         {
4114                             logf("Entry no longer valid.");
4115                             logf("-> %s", buf);
4116                             delete_entry(fe->idx_id);
4117                             continue;
4118                         }
4119                     }
4120                 }
4121
4122                 continue ;
4123             }
4124
4125             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4126             if (bytesleft < 0)
4127             {
4128                 logf("too big tagcache #2");
4129                 logf("tl: %ld", fe->tag_length);
4130                 logf("bl: %ld", bytesleft);
4131                 close(fd);
4132                 return false;
4133             }
4134
4135             p = fe->tag_data;
4136             rc = read(fd, fe->tag_data, fe->tag_length);
4137             p += rc;
4138
4139             if (rc != fe->tag_length)
4140             {
4141                 logf("read error #13");
4142                 logf("rc=0x%04x", rc); // 0x431
4143                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4144                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4145                 logf("tag=0x%02x", tag); // 0x00
4146                 close(fd);
4147                 return false;
4148             }
4149         }
4150         close(fd);
4151     }
4152
4153     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4154     logf("tagcache loaded into ram!");
4155
4156     return true;
4157 }
4158 #endif /* HAVE_TC_RAMCACHE */
4159
4160 static bool check_deleted_files(void)
4161 {
4162     int fd;
4163     char buf[TAG_MAXLEN+32];
4164     struct tagfile_entry tfe;
4165
4166     logf("reverse scan...");
4167     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4168     fd = open(buf, O_RDONLY);
4169
4170     if (fd < 0)
4171     {
4172         logf("%s open fail", buf);
4173         return false;
4174     }
4175
4176     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4177     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4178 #ifndef __PCTOOL__
4179            && !check_event_queue()
4180 #endif
4181            )
4182     {
4183         if (tfe.tag_length >= (long)sizeof(buf)-1)
4184         {
4185             logf("too long tag");
4186             close(fd);
4187             return false;
4188         }
4189
4190         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4191         {
4192             logf("read error #14");
4193             close(fd);
4194             return false;
4195         }
4196
4197         /* Check if the file has already deleted from the db. */
4198         if (*buf == '\0')
4199             continue;
4200
4201         /* Now check if the file exists. */
4202         if (!file_exists(buf))
4203         {
4204             logf("Entry no longer valid.");
4205             logf("-> %s / %ld", buf, tfe.tag_length);
4206             delete_entry(tfe.idx_id);
4207         }
4208     }
4209
4210     close(fd);
4211
4212     logf("done");
4213
4214     return true;
4215 }
4216
4217
4218 /* Note that this function must not be inlined, otherwise the whole point
4219  * of having the code in a separate function is lost.
4220  */
4221 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4222     int *ignore, int *unignore)
4223 {
4224     char newpath[MAX_PATH];
4225
4226     /* check for a database.ignore file */
4227     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4228     *ignore = file_exists(newpath);
4229     /* check for a database.unignore file */
4230     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4231     *unignore = file_exists(newpath);
4232 }
4233
4234 static struct search_roots_ll {
4235     const char *path;
4236     struct search_roots_ll * next;
4237 } roots_ll;
4238
4239 #ifdef APPLICATION
4240 /*
4241  * This adds a path to the search roots, possibly during traveling through
4242  * the filesystem. It only adds if the path is not inside an already existing
4243  * search root.
4244  *
4245  * Returns true if it added the path to the search roots
4246  *
4247  * Windows 2000 and greater supports symlinks, but they don't provide
4248  * realpath() or readlink(), and symlinks are rarely used on them so
4249  * ignore this for windows for now
4250  **/
4251 static bool add_search_root(const char *name)
4252 {
4253     (void)name;
4254 #ifndef WIN32
4255     struct search_roots_ll *this, *prev = NULL;
4256     char target[MAX_PATH];
4257     /* Okay, realpath() is almost completely broken on android
4258      *
4259      * It doesn't accept NULL for resolved_name to dynamically allocate
4260      * the resulting path; and it assumes resolved_name to be PATH_MAX
4261      * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4262      * and blindly writes to the end if it
4263      *
4264      * therefore use sufficiently large static storage here
4265      * Note that PATH_MAX != MAX_PATH
4266      **/
4267     static char abs_target[PATH_MAX];
4268     ssize_t len;
4269
4270     len = readlink(name, target, sizeof(target));
4271     if (len < 0)
4272         return false;
4273
4274     target[len] = '\0';
4275     if (realpath(target, abs_target) == NULL)
4276         return false;
4277
4278     for(this = &roots_ll; this; prev = this, this = this->next)
4279     {
4280         size_t root_len = strlen(this->path);
4281         /* check if the link target is inside of an existing search root
4282          * don't add if target is inside, we'll scan it later */
4283         if (!strncmp(this->path, abs_target, root_len))
4284             return false;
4285     }
4286
4287     if (prev)
4288     {
4289         size_t len = strlen(abs_target) + 1; /* count \0 */
4290         this = malloc(sizeof(struct search_roots_ll) + len );
4291         if (!this || len > MAX_PATH)
4292         {
4293             logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4294             free(this);
4295             prev->next = NULL;
4296             return false;
4297         }
4298         this->path = ((char*)this) + sizeof(struct search_roots_ll);
4299         strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4300         this->next = NULL;
4301         prev->next = this;
4302         logf("Added %s to the search roots\n", abs_target);
4303         return true;
4304     }
4305 #endif
4306     return false;
4307 }
4308
4309 static int free_search_roots(struct search_roots_ll * start)
4310 {
4311     int ret = 0;
4312     if (start->next)
4313     {
4314         ret += free_search_roots(start->next);
4315         ret += sizeof(struct search_roots_ll);
4316         free(start->next);
4317     }
4318     return ret;
4319 }
4320 #else /* native, simulator */
4321 #define add_search_root(a) do {} while(0)
4322 #define free_search_roots(a) do {} while(0)
4323 #endif
4324
4325 static bool check_dir(const char *dirname, int add_files)
4326 {
4327     DIR *dir;
4328     int len;
4329     int success = false;
4330     int ignore, unignore;
4331
4332     dir = opendir(dirname);
4333     if (!dir)
4334     {
4335         logf("tagcache: opendir(%s) failed", dirname);
4336         return false;
4337     }
4338     /* check for a database.ignore and database.unignore */
4339     check_ignore(dirname, &ignore, &unignore);
4340
4341     /* don't do anything if both ignore and unignore are there */
4342     if (ignore != unignore)
4343         add_files = unignore;
4344
4345     /* Recursively scan the dir. */
4346 #ifdef __PCTOOL__
4347     while (1)
4348 #else
4349     while (!check_event_queue())
4350 #endif
4351     {
4352         struct dirent *entry = readdir(dir);
4353         if (entry == NULL)
4354         {
4355             success = true;
4356             break;
4357         }
4358
4359         if (!strcmp((char *)entry->d_name, ".") ||
4360             !strcmp((char *)entry->d_name, ".."))
4361             continue;
4362
4363         struct dirinfo info = dir_get_info(dir, entry);
4364
4365         yield();
4366
4367         len = strlen(curpath);
4368         /* don't add an extra / for curpath == / */
4369         if (len <= 1) len = 0;
4370         snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4371
4372         processed_dir_count++;
4373         if (info.attribute & ATTR_DIRECTORY)
4374 #ifndef SIMULATOR
4375         {   /* don't follow symlinks to dirs, but try to add it as a search root
4376              * this makes able to avoid looping in recursive symlinks */
4377             if (info.attribute & ATTR_LINK)
4378                 add_search_root(curpath);
4379             else
4380                 check_dir(curpath, add_files);
4381         }
4382 #else
4383             check_dir(curpath, add_files);
4384 #endif
4385         else if (add_files)
4386         {
4387             tc_stat.curentry = curpath;
4388
4389             /* Add a new entry to the temporary db file. */
4390             add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4391 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4392                          , dir->internal_entry
4393 #endif
4394                          );
4395
4396             /* Wait until current path for debug screen is read and unset. */
4397             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4398                 yield();
4399
4400             tc_stat.curentry = NULL;
4401         }
4402
4403         curpath[len] = '\0';
4404     }
4405
4406     closedir(dir);
4407
4408     return success;
4409 }
4410
4411 void tagcache_screensync_event(void)
4412 {
4413     tc_stat.curentry = NULL;
4414 }
4415
4416 void tagcache_screensync_enable(bool state)
4417 {
4418     tc_stat.syncscreen = state;
4419 }
4420
4421 void tagcache_build(const char *path)
4422 {
4423     struct tagcache_header header;
4424     bool ret;
4425
4426     curpath[0] = '\0';
4427     data_size = 0;
4428     total_entry_count = 0;
4429     processed_dir_count = 0;
4430
4431 #ifdef HAVE_DIRCACHE
4432     while (dircache_is_initializing())
4433         sleep(1);
4434 #endif
4435
4436     logf("updating tagcache");
4437
4438     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4439     if (cachefd >= 0)
4440     {
4441         logf("skipping, cache already waiting for commit");
4442         close(cachefd);
4443         return ;
4444     }
4445
4446     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4447     if (cachefd < 0)
4448     {
4449         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4450         return ;
4451     }
4452
4453     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4454
4455     cpu_boost(true);
4456
4457     logf("Scanning files...");
4458     /* Scan for new files. */
4459     memset(&header, 0, sizeof(struct tagcache_header));
4460     write(cachefd, &header, sizeof(struct tagcache_header));
4461
4462     ret = true;
4463     roots_ll.path = path;
4464     roots_ll.next = NULL;
4465     struct search_roots_ll * this;
4466     /* check_dir might add new roots */
4467     for(this = &roots_ll; this; this = this->next)
4468     {
4469         strcpy(curpath, this->path);
4470         ret = ret && check_dir(this->path, true);
4471     }
4472     if (roots_ll.next)
4473         free_search_roots(roots_ll.next);
4474
4475     /* Write the header. */
4476     header.magic = TAGCACHE_MAGIC;
4477     header.datasize = data_size;
4478     header.entry_count = total_entry_count;
4479     lseek(cachefd, 0, SEEK_SET);
4480     write(cachefd, &header, sizeof(struct tagcache_header));
4481     close(cachefd);
4482
4483     if (filenametag_fd >= 0)
4484     {
4485         close(filenametag_fd);
4486         filenametag_fd = -1;
4487     }
4488
4489     if (!ret)
4490     {
4491         logf("Aborted.");
4492         cpu_boost(false);
4493         return ;
4494     }
4495
4496     /* Commit changes to the database. */
4497 #ifdef __PCTOOL__
4498     allocate_tempbuf();
4499 #endif
4500     if (commit())
4501     {
4502         logf("tagcache built!");
4503     }
4504 #ifdef __PCTOOL__
4505     free_tempbuf();
4506 #endif
4507
4508 #ifdef HAVE_TC_RAMCACHE
4509     if (hdr)
4510     {
4511         /* Import runtime statistics if we just initialized the db. */
4512         if (hdr->h.serial == 0)
4513             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4514     }
4515 #endif
4516
4517     cpu_boost(false);
4518 }
4519
4520 #ifdef HAVE_TC_RAMCACHE
4521 static void load_ramcache(void)
4522 {
4523     if (!hdr)
4524         return ;
4525
4526     cpu_boost(true);
4527
4528     /* At first we should load the cache (if exists). */
4529     tc_stat.ramcache = load_tagcache();
4530
4531     if (!tc_stat.ramcache)
4532     {
4533         /* If loading failed, it must indicate some problem with the db
4534          * so disable it entirely to prevent further issues. */
4535         tc_stat.ready = false;
4536         hdr = NULL;
4537     }
4538
4539     cpu_boost(false);
4540 }
4541
4542 void tagcache_unload_ramcache(void)
4543 {
4544     tc_stat.ramcache = false;
4545     /* Just to make sure there is no statefile present. */
4546     // remove(TAGCACHE_STATEFILE);
4547 }
4548 #endif
4549
4550 #ifndef __PCTOOL__
4551 static void tagcache_thread(void)
4552 {
4553     struct queue_event ev;
4554     bool check_done = false;
4555
4556     /* If the previous cache build/update was interrupted, commit
4557      * the changes first in foreground. */
4558     cpu_boost(true);
4559     allocate_tempbuf();
4560     commit();
4561     free_tempbuf();
4562
4563 #ifdef HAVE_TC_RAMCACHE
4564 # ifdef HAVE_EEPROM_SETTINGS
4565     if (firmware_settings.initialized && firmware_settings.disk_clean
4566         && global_settings.tagcache_ram)
4567     {
4568         check_done = tagcache_dumpload();
4569     }
4570
4571     remove(TAGCACHE_STATEFILE);
4572 # endif
4573
4574     /* Allocate space for the tagcache if found on disk. */
4575     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4576         allocate_tagcache();
4577 #endif
4578
4579     cpu_boost(false);
4580     tc_stat.initialized = true;
4581
4582     /* Don't delay bootup with the header check but do it on background. */
4583     if (!tc_stat.ready)
4584     {
4585         sleep(HZ);
4586         tc_stat.ready = check_all_headers();
4587         tc_stat.readyvalid = true;
4588     }
4589
4590     while (1)
4591     {
4592         run_command_queue(false);
4593
4594         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4595
4596         switch (ev.id)
4597         {
4598             case Q_IMPORT_CHANGELOG:
4599                 tagcache_import_changelog();
4600                 break;
4601
4602             case Q_REBUILD:
4603                 remove_files();
4604                 remove(TAGCACHE_FILE_TEMP);
4605                 tagcache_build("/");
4606                 break;
4607
4608             case Q_UPDATE:
4609                 tagcache_build("/");
4610 #ifdef HAVE_TC_RAMCACHE
4611                 load_ramcache();
4612 #endif
4613                 check_deleted_files();
4614                 break ;
4615
4616             case Q_START_SCAN:
4617                 check_done = false;
4618             case SYS_TIMEOUT:
4619                 if (check_done || !tc_stat.ready)
4620                     break ;
4621
4622 #ifdef HAVE_TC_RAMCACHE
4623                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4624                 {
4625                     load_ramcache();
4626                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4627                         tagcache_build("/");
4628                 }
4629                 else
4630 #endif
4631                 if (global_settings.tagcache_autoupdate)
4632                 {
4633                     tagcache_build("/");
4634
4635                     /* This will be very slow unless dircache is enabled
4636                        or target is flash based, but do it anyway for
4637                        consistency. */
4638                     check_deleted_files();
4639                 }
4640
4641                 logf("tagcache check done");
4642
4643                 check_done = true;
4644                 break ;
4645
4646             case Q_STOP_SCAN:
4647                 break ;
4648
4649             case SYS_POWEROFF:
4650                 break ;
4651
4652 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4653             case SYS_USB_CONNECTED:
4654                 logf("USB: TagCache");
4655                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4656                 usb_wait_for_disconnect(&tagcache_queue);
4657                 break ;
4658 #endif
4659         }
4660     }
4661 }
4662
4663 bool tagcache_prepare_shutdown(void)
4664 {
4665     if (tagcache_get_commit_step() > 0)
4666         return false;
4667
4668     tagcache_stop_scan();
4669     while (read_lock || write_lock)
4670         sleep(1);
4671
4672     return true;
4673 }
4674
4675 void tagcache_shutdown(void)
4676 {
4677     /* Flush the command queue. */
4678     run_command_queue(true);
4679
4680 #ifdef HAVE_EEPROM_SETTINGS
4681     if (tc_stat.ramcache)
4682         tagcache_dumpsave();
4683 #endif
4684 }
4685
4686 static int get_progress(void)
4687 {
4688     int total_count = -1;
4689
4690 #ifdef HAVE_DIRCACHE
4691     if (dircache_is_enabled())
4692     {
4693         total_count = dircache_get_entry_count();
4694     }
4695     else
4696 #endif
4697 #ifdef HAVE_TC_RAMCACHE
4698     {
4699         if (hdr && tc_stat.ramcache)
4700             total_count = hdr->h.tch.entry_count;
4701     }
4702 #endif
4703
4704     if (total_count < 0)
4705         return -1;
4706
4707     return processed_dir_count * 100 / total_count;
4708 }
4709
4710 struct tagcache_stat* tagcache_get_stat(void)
4711 {
4712     tc_stat.progress = get_progress();
4713     tc_stat.processed_entries = processed_dir_count;
4714
4715     return &tc_stat;
4716 }
4717
4718 void tagcache_start_scan(void)
4719 {
4720     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4721 }
4722
4723 bool tagcache_update(void)
4724 {
4725     if (!tc_stat.ready)
4726         return false;
4727
4728     queue_post(&tagcache_queue, Q_UPDATE, 0);
4729     return false;
4730 }
4731
4732 bool tagcache_rebuild()
4733 {
4734     queue_post(&tagcache_queue, Q_REBUILD, 0);
4735     return false;
4736 }
4737
4738 void tagcache_stop_scan(void)
4739 {
4740     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4741 }
4742
4743 #ifdef HAVE_TC_RAMCACHE
4744 bool tagcache_is_ramcache(void)
4745 {
4746     return tc_stat.ramcache;
4747 }
4748 #endif
4749
4750 #endif /* !__PCTOOL__ */
4751
4752
4753 void tagcache_init(void)
4754 {
4755     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4756     memset(&current_tcmh, 0, sizeof(struct master_header));
4757     filenametag_fd = -1;
4758     write_lock = read_lock = 0;
4759
4760 #ifndef __PCTOOL__
4761     mutex_init(&command_queue_mutex);
4762     queue_init(&tagcache_queue, true);
4763     create_thread(tagcache_thread, tagcache_stack,
4764                   sizeof(tagcache_stack), 0, tagcache_thread_name
4765                   IF_PRIO(, PRIORITY_BACKGROUND)
4766                   IF_COP(, CPU));
4767 #else
4768     tc_stat.initialized = true;
4769     allocate_tempbuf();
4770     commit();
4771     free_tempbuf();
4772     tc_stat.ready = check_all_headers();
4773 #endif
4774 }
4775
4776 #ifdef __PCTOOL__
4777 void tagcache_reverse_scan(void)
4778 {
4779     logf("Checking for deleted files");
4780     check_deleted_files();
4781 }
4782 #endif
4783
4784 bool tagcache_is_initialized(void)
4785 {
4786     return tc_stat.initialized;
4787 }
4788 bool tagcache_is_fully_initialized(void)
4789 {
4790     return tc_stat.readyvalid;
4791 }
4792 bool tagcache_is_usable(void)
4793 {
4794     return tc_stat.initialized && tc_stat.ready;
4795 }
4796 int tagcache_get_commit_step(void)
4797 {
4798     return tc_stat.commit_step;
4799 }
4800 int tagcache_get_max_commit_step(void)
4801 {
4802     return (int)(SORTED_TAGS_COUNT)+1;
4803 }
4804