apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /* #define LOGF_ENABLE */
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 #define UNTAGGED "<Untagged>"
  99
 100 /* Previous path when scanning directory tree recursively. */
 101 static char curpath[TAG_MAXLEN+32];
 102
 103 /* Used when removing duplicates. */
 104 static char *tempbuf;     /* Allocated when needed. */
 105 static long tempbufidx;   /* Current location in buffer. */
 106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 107 static long tempbuf_left; /* Buffer space left. */
 108 static long tempbuf_pos;
 109
 110 #define SORTED_TAGS_COUNT 8
 111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 114     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 115 /* Tags we want to get sorted (loaded to the tempbuf). */
 116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 117     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 118     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 119
 120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 122     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 123     (1LU << tag_albumartist) | (1LU << tag_grouping))
 124
 125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 126 static const char *tags_str[] = { "artist", "album", "genre", "title",
 127     "filename", "composer", "comment", "albumartist", "grouping", "year",
 128     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 129     "playtime", "lastplayed", "commitid", "mtime" };
 130
 131 /* Status information of the tagcache. */
 132 static struct tagcache_stat tc_stat;
 133
 134 /* Queue commands. */
 135 enum tagcache_queue {
 136     Q_STOP_SCAN = 0,
 137     Q_START_SCAN,
 138     Q_IMPORT_CHANGELOG,
 139     Q_UPDATE,
 140     Q_REBUILD,
 141
 142     /* Internal tagcache command queue. */
 143     CMD_UPDATE_MASTER_HEADER,
 144     CMD_UPDATE_NUMERIC,
 145 };
 146
 147 struct tagcache_command_entry {
 148     int32_t command;
 149     int32_t idx_id;
 150     int32_t tag;
 151     int32_t data;
 152 };
 153
 154 #ifndef __PCTOOL__
 155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 156 static volatile int command_queue_widx = 0;
 157 static volatile int command_queue_ridx = 0;
 158 static struct mutex command_queue_mutex;
 159 #endif
 160
 161 /* Tag database structures. */
 162
 163 /* Variable-length tag entry in tag files. */
 164 struct tagfile_entry {
 165     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 166     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 167     char tag_data[0];  /* Begin of the tag data */
 168 };
 169
 170 /* Fixed-size tag entry in master db index. */
 171 struct index_entry {
 172     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 173     int32_t flag;                /* Status flags */
 174 };
 175
 176 /* Header is the same in every file. */
 177 struct tagcache_header {
 178     int32_t magic;       /* Header version number */
 179     int32_t datasize;    /* Data size in bytes */
 180     int32_t entry_count; /* Number of entries in this file */
 181 };
 182
 183 struct master_header {
 184     struct tagcache_header tch;
 185     int32_t serial; /* Increasing counting number */
 186     int32_t commitid; /* Number of commits so far */
 187     int32_t dirty;
 188 };
 189
 190 /* For the endianess correction */
 191 static const char *tagfile_entry_ec   = "ll";
 192 /**
 193  Note: This should be (1 + TAG_COUNT) amount of l's.
 194  */
 195 static const char *index_entry_ec     = "lllllllllllllllllllll";
 196
 197 static const char *tagcache_header_ec = "lll";
 198 static const char *master_header_ec   = "llllll";
 199
 200 static struct master_header current_tcmh;
 201
 202 #ifdef HAVE_TC_RAMCACHE
 203 /* Header is created when loading database to ram. */
 204 struct ramcache_header {
 205     struct master_header h;      /* Header from the master index */
 206     struct index_entry *indices; /* Master index file content */
 207     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 208     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 209 };
 210
 211 # ifdef HAVE_EEPROM_SETTINGS
 212 struct statefile_header {
 213     struct ramcache_header *hdr;
 214     struct tagcache_stat tc_stat;
 215 };
 216 # endif
 217
 218 /* Pointer to allocated ramcache_header */
 219 static struct ramcache_header *hdr;
 220 #endif
 221
 222 /**
 223  * Full tag entries stored in a temporary file waiting
 224  * for commit to the cache. */
 225 struct temp_file_entry {
 226     long tag_offset[TAG_COUNT];
 227     short tag_length[TAG_COUNT];
 228     long flag;
 229
 230     long data_length;
 231 };
 232
 233 struct tempbuf_id_list {
 234     long id;
 235     struct tempbuf_id_list *next;
 236 };
 237
 238 struct tempbuf_searchidx {
 239     long idx_id;
 240     char *str;
 241     int seek;
 242     struct tempbuf_id_list idlist;
 243 };
 244
 245 /* Lookup buffer for fixing messed up index while after sorting. */
 246 static long commit_entry_count;
 247 static long lookup_buffer_depth;
 248 static struct tempbuf_searchidx **lookup;
 249
 250 /* Used when building the temporary file. */
 251 static int cachefd = -1, filenametag_fd;
 252 static int total_entry_count = 0;
 253 static int data_size = 0;
 254 static int processed_dir_count;
 255
 256 /* Thread safe locking */
 257 static volatile int write_lock;
 258 static volatile int read_lock;
 259
 260 static bool delete_entry(long idx_id);
 261
 262 const char* tagcache_tag_to_str(int tag)
 263 {
 264     return tags_str[tag];
 265 }
 266
 267 #ifdef HAVE_DIRCACHE
 268 /**
 269  * Returns true if specified flag is still present, i.e., dircache
 270  * has not been reloaded.
 271  */
 272 static bool is_dircache_intact(void)
 273 {
 274     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 275 }
 276 #endif
 277
 278 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 279 {
 280     int fd;
 281     char buf[MAX_PATH];
 282     int rc;
 283
 284     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 285         return -1;
 286
 287     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 288
 289     fd = open(buf, write ? O_RDWR : O_RDONLY);
 290     if (fd < 0)
 291     {
 292         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 293         tc_stat.ready = false;
 294         return fd;
 295     }
 296
 297     /* Check the header. */
 298     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 299     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 300     {
 301         logf("header error");
 302         tc_stat.ready = false;
 303         close(fd);
 304         return -2;
 305     }
 306
 307     return fd;
 308 }
 309
 310 static int open_master_fd(struct master_header *hdr, bool write)
 311 {
 312     int fd;
 313     int rc;
 314
 315     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 316     if (fd < 0)
 317     {
 318         logf("master file open failed for R/W");
 319         tc_stat.ready = false;
 320         return fd;
 321     }
 322
 323     tc_stat.econ = false;
 324
 325     /* Check the header. */
 326     rc = read(fd, hdr, sizeof(struct master_header));
 327     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 328     {
 329         /* Success. */
 330         return fd;
 331     }
 332
 333     /* Trying to read again, this time with endianess correction enabled. */
 334     lseek(fd, 0, SEEK_SET);
 335
 336     rc = ecread(fd, hdr, 1, master_header_ec, true);
 337     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 338     {
 339         logf("header error");
 340         tc_stat.ready = false;
 341         close(fd);
 342         return -2;
 343     }
 344
 345     tc_stat.econ = true;
 346
 347     return fd;
 348 }
 349
 350 #ifndef __PCTOOL__
 351 static bool do_timed_yield(void)
 352 {
 353     /* Sorting can lock up for quite a while, so yield occasionally */
 354     static long wakeup_tick = 0;
 355     if (current_tick >= wakeup_tick)
 356     {
 357         wakeup_tick = current_tick + (HZ/4);
 358         yield();
 359         return true;
 360     }
 361     return false;
 362 }
 363 #endif
 364
 365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 366 static long find_entry_ram(const char *filename,
 367                            const struct dirent *dc)
 368 {
 369     static long last_pos = 0;
 370     int i;
 371
 372     /* Check if we tagcache is loaded into ram. */
 373     if (!tc_stat.ramcache)
 374         return -1;
 375
 376     if (dc == NULL)
 377         dc = dircache_get_entry_ptr(filename);
 378
 379     if (dc == NULL)
 380     {
 381         logf("tagcache: file not found.");
 382         return -1;
 383     }
 384
 385     try_again:
 386
 387     if (last_pos > 0)
 388         i = last_pos;
 389     else
 390         i = 0;
 391
 392     for (; i < hdr->h.tch.entry_count; i++)
 393     {
 394         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 395         {
 396             last_pos = MAX(0, i - 3);
 397             return i;
 398         }
 399
 400         do_timed_yield();
 401     }
 402
 403     if (last_pos > 0)
 404     {
 405         last_pos = 0;
 406         goto try_again;
 407     }
 408
 409     return -1;
 410 }
 411 #endif
 412
 413 static long find_entry_disk(const char *filename)
 414 {
 415     struct tagcache_header tch;
 416     static long last_pos = -1;
 417     long pos_history[POS_HISTORY_COUNT];
 418     long pos_history_idx = 0;
 419     bool found = false;
 420     struct tagfile_entry tfe;
 421     int fd;
 422     char buf[TAG_MAXLEN+32];
 423     int i;
 424     int pos = -1;
 425
 426     if (!tc_stat.ready)
 427         return -2;
 428
 429     fd = filenametag_fd;
 430     if (fd < 0)
 431     {
 432         last_pos = -1;
 433         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 434             return -1;
 435     }
 436
 437     check_again:
 438
 439     if (last_pos > 0)
 440         lseek(fd, last_pos, SEEK_SET);
 441     else
 442         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 443
 444     while (true)
 445     {
 446         pos = lseek(fd, 0, SEEK_CUR);
 447         for (i = pos_history_idx-1; i >= 0; i--)
 448             pos_history[i+1] = pos_history[i];
 449         pos_history[0] = pos;
 450
 451         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 452             != sizeof(struct tagfile_entry))
 453         {
 454             break ;
 455         }
 456
 457         if (tfe.tag_length >= (long)sizeof(buf))
 458         {
 459             logf("too long tag #1");
 460             close(fd);
 461             last_pos = -1;
 462             return -2;
 463         }
 464
 465         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 466         {
 467             logf("read error #2");
 468             close(fd);
 469             last_pos = -1;
 470             return -3;
 471         }
 472
 473         if (!strcasecmp(filename, buf))
 474         {
 475             last_pos = pos_history[pos_history_idx];
 476             found = true;
 477             break ;
 478         }
 479
 480         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 481             pos_history_idx++;
 482     }
 483
 484     /* Not found? */
 485     if (!found)
 486     {
 487         if (last_pos > 0)
 488         {
 489             last_pos = -1;
 490             logf("seek again");
 491             goto check_again;
 492         }
 493
 494         if (fd != filenametag_fd)
 495             close(fd);
 496         return -4;
 497     }
 498
 499     if (fd != filenametag_fd)
 500         close(fd);
 501
 502     return tfe.idx_id;
 503 }
 504
 505 static int find_index(const char *filename)
 506 {
 507     long idx_id = -1;
 508
 509 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 510     if (tc_stat.ramcache && is_dircache_intact())
 511         idx_id = find_entry_ram(filename, NULL);
 512 #endif
 513
 514     if (idx_id < 0)
 515         idx_id = find_entry_disk(filename);
 516
 517     return idx_id;
 518 }
 519
 520 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 521 {
 522     int idx_id;
 523
 524     if (!tc_stat.ready)
 525         return false;
 526
 527     idx_id = find_index(filename);
 528     if (idx_id < 0)
 529         return false;
 530
 531     if (!tagcache_search(tcs, tag_filename))
 532         return false;
 533
 534     tcs->entry_count = 0;
 535     tcs->idx_id = idx_id;
 536
 537     return true;
 538 }
 539
 540 static bool get_index(int masterfd, int idxid,
 541                       struct index_entry *idx, bool use_ram)
 542 {
 543     bool localfd = false;
 544
 545     if (idxid < 0)
 546     {
 547         logf("Incorrect idxid: %d", idxid);
 548         return false;
 549     }
 550
 551 #ifdef HAVE_TC_RAMCACHE
 552     if (tc_stat.ramcache && use_ram)
 553     {
 554         if (hdr->indices[idxid].flag & FLAG_DELETED)
 555             return false;
 556
 557         memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 558         return true;
 559     }
 560 #else
 561     (void)use_ram;
 562 #endif
 563
 564     if (masterfd < 0)
 565     {
 566         struct master_header tcmh;
 567
 568         localfd = true;
 569         masterfd = open_master_fd(&tcmh, false);
 570         if (masterfd < 0)
 571             return false;
 572     }
 573
 574     lseek(masterfd, idxid * sizeof(struct index_entry)
 575           + sizeof(struct master_header), SEEK_SET);
 576     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 577         != sizeof(struct index_entry))
 578     {
 579         logf("read error #3");
 580         if (localfd)
 581             close(masterfd);
 582
 583         return false;
 584     }
 585
 586     if (localfd)
 587         close(masterfd);
 588
 589     if (idx->flag & FLAG_DELETED)
 590         return false;
 591
 592     return true;
 593 }
 594
 595 #ifndef __PCTOOL__
 596
 597 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 598 {
 599     /* We need to exclude all memory only flags & tags when writing to disk. */
 600     if (idx->flag & FLAG_DIRCACHE)
 601     {
 602         logf("memory only flags!");
 603         return false;
 604     }
 605
 606 #ifdef HAVE_TC_RAMCACHE
 607     /* Only update numeric data. Writing the whole index to RAM by memcpy
 608      * destroys dircache pointers!
 609      */
 610     if (tc_stat.ramcache)
 611     {
 612         int tag;
 613         struct index_entry *idx_ram = &hdr->indices[idxid];
 614
 615         for (tag = 0; tag < TAG_COUNT; tag++)
 616         {
 617             if (TAGCACHE_IS_NUMERIC(tag))
 618             {
 619                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 620             }
 621         }
 622
 623         /* Don't touch the dircache flag or attributes. */
 624         idx_ram->flag = (idx->flag & 0x0000ffff)
 625             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 626     }
 627 #endif
 628
 629     lseek(masterfd, idxid * sizeof(struct index_entry)
 630           + sizeof(struct master_header), SEEK_SET);
 631     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 632         != sizeof(struct index_entry))
 633     {
 634         logf("write error #3");
 635         logf("idxid: %d", idxid);
 636         return false;
 637     }
 638
 639     return true;
 640 }
 641
 642 #endif /* !__PCTOOL__ */
 643
 644 static bool open_files(struct tagcache_search *tcs, int tag)
 645 {
 646     if (tcs->idxfd[tag] < 0)
 647     {
 648         char fn[MAX_PATH];
 649
 650         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 651         tcs->idxfd[tag] = open(fn, O_RDONLY);
 652     }
 653
 654     if (tcs->idxfd[tag] < 0)
 655     {
 656         logf("File not open!");
 657         return false;
 658     }
 659
 660     return true;
 661 }
 662
 663 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 664                      int tag, char *buf, long size)
 665 {
 666     struct tagfile_entry tfe;
 667     long seek;
 668
 669     *buf = '\0';
 670
 671     if (TAGCACHE_IS_NUMERIC(tag))
 672         return false;
 673
 674     seek = idx->tag_seek[tag];
 675     if (seek < 0)
 676     {
 677         logf("Retrieve failed");
 678         return false;
 679     }
 680
 681 #ifdef HAVE_TC_RAMCACHE
 682     if (tcs->ramsearch)
 683     {
 684         struct tagfile_entry *ep;
 685
 686 # ifdef HAVE_DIRCACHE
 687         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 688             && is_dircache_intact())
 689         {
 690             dircache_copy_path((struct dirent *)seek,
 691                                buf, size);
 692             return true;
 693         }
 694         else
 695 # endif
 696         if (tag != tag_filename)
 697         {
 698             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 699             strncpy(buf, ep->tag_data, size-1);
 700
 701             return true;
 702         }
 703     }
 704 #endif
 705
 706     if (!open_files(tcs, tag))
 707         return false;
 708
 709     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 710     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 711         != sizeof(struct tagfile_entry))
 712     {
 713         logf("read error #5");
 714         return false;
 715     }
 716
 717     if (tfe.tag_length >= size)
 718     {
 719         logf("too small buffer");
 720         return false;
 721     }
 722
 723     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 724         tfe.tag_length)
 725     {
 726         logf("read error #6");
 727         return false;
 728     }
 729
 730     buf[tfe.tag_length] = '\0';
 731
 732     return true;
 733 }
 734
 735 static long check_virtual_tags(int tag, const struct index_entry *idx)
 736 {
 737     long data = 0;
 738
 739     switch (tag)
 740     {
 741         case tag_virt_length_sec:
 742             data = (idx->tag_seek[tag_length]/1000) % 60;
 743             break;
 744
 745         case tag_virt_length_min:
 746             data = (idx->tag_seek[tag_length]/1000) / 60;
 747             break;
 748
 749         case tag_virt_playtime_sec:
 750             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 751             break;
 752
 753         case tag_virt_playtime_min:
 754             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 755             break;
 756
 757         case tag_virt_autoscore:
 758             if (idx->tag_seek[tag_length] == 0
 759                 || idx->tag_seek[tag_playcount] == 0)
 760             {
 761                 data = 0;
 762             }
 763             else
 764             {
 765                 data = 100 * idx->tag_seek[tag_playtime]
 766                     / idx->tag_seek[tag_length]
 767                     / idx->tag_seek[tag_playcount];
 768             }
 769             break;
 770
 771         /* How many commits before the file has been added to the DB. */
 772         case tag_virt_entryage:
 773             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 774             break;
 775
 776         default:
 777             data = idx->tag_seek[tag];
 778     }
 779
 780     return data;
 781 }
 782
 783 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 784 {
 785     struct index_entry idx;
 786
 787     if (!tc_stat.ready)
 788         return false;
 789
 790     if (!TAGCACHE_IS_NUMERIC(tag))
 791         return -1;
 792
 793     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 794         return -2;
 795
 796     return check_virtual_tags(tag, &idx);
 797 }
 798
 799 inline static bool str_ends_with(const char *str1, const char *str2)
 800 {
 801     int str_len = strlen(str1);
 802     int clause_len = strlen(str2);
 803
 804     if (clause_len > str_len)
 805         return false;
 806
 807     return !strcasecmp(&str1[str_len - clause_len], str2);
 808 }
 809
 810 inline static bool str_oneof(const char *str, const char *list)
 811 {
 812     const char *sep;
 813     int l, len = strlen(str);
 814
 815     while (*list)
 816     {
 817         sep = strchr(list, '|');
 818         l = sep ? (long)sep - (long)list : (int)strlen(list);
 819         if ((l==len) && !strncasecmp(str, list, len))
 820             return true;
 821         list += sep ? l + 1 : l;
 822     }
 823
 824     return false;
 825 }
 826
 827 static bool check_against_clause(long numeric, const char *str,
 828                                  const struct tagcache_search_clause *clause)
 829 {
 830     if (clause->numeric)
 831     {
 832         switch (clause->type)
 833         {
 834             case clause_is:
 835                 return numeric == clause->numeric_data;
 836             case clause_is_not:
 837                 return numeric != clause->numeric_data;
 838             case clause_gt:
 839                 return numeric > clause->numeric_data;
 840             case clause_gteq:
 841                 return numeric >= clause->numeric_data;
 842             case clause_lt:
 843                 return numeric < clause->numeric_data;
 844             case clause_lteq:
 845                 return numeric <= clause->numeric_data;
 846             default:
 847                 logf("Incorrect numeric tag: %d", clause->type);
 848         }
 849     }
 850     else
 851     {
 852         switch (clause->type)
 853         {
 854             case clause_is:
 855                 return !strcasecmp(clause->str, str);
 856             case clause_is_not:
 857                 return strcasecmp(clause->str, str);
 858             case clause_gt:
 859                 return 0>strcasecmp(clause->str, str);
 860             case clause_gteq:
 861                 return 0>=strcasecmp(clause->str, str);
 862             case clause_lt:
 863                 return 0<strcasecmp(clause->str, str);
 864             case clause_lteq:
 865                 return 0<=strcasecmp(clause->str, str);
 866             case clause_contains:
 867                 return (strcasestr(str, clause->str) != NULL);
 868             case clause_not_contains:
 869                 return (strcasestr(str, clause->str) == NULL);
 870             case clause_begins_with:
 871                 return (strcasestr(str, clause->str) == str);
 872             case clause_not_begins_with:
 873                 return (strcasestr(str, clause->str) != str);
 874             case clause_ends_with:
 875                 return str_ends_with(str, clause->str);
 876             case clause_not_ends_with:
 877                 return !str_ends_with(str, clause->str);
 878             case clause_oneof:
 879                 return str_oneof(str, clause->str);
 880
 881             default:
 882                 logf("Incorrect tag: %d", clause->type);
 883         }
 884     }
 885
 886     return false;
 887 }
 888
 889 static bool check_clauses(struct tagcache_search *tcs,
 890                           struct index_entry *idx,
 891                           struct tagcache_search_clause **clause, int count)
 892 {
 893     int i;
 894
 895 #ifdef HAVE_TC_RAMCACHE
 896     if (tcs->ramsearch)
 897     {
 898         /* Go through all conditional clauses. */
 899         for (i = 0; i < count; i++)
 900         {
 901             struct tagfile_entry *tfe;
 902             int seek;
 903             char buf[256];
 904             char *str = NULL;
 905
 906             seek = check_virtual_tags(clause[i]->tag, idx);
 907
 908             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 909             {
 910                 if (clause[i]->tag == tag_filename)
 911                 {
 912                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 913                     str = buf;
 914                 }
 915                 else
 916                 {
 917                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 918                     str = tfe->tag_data;
 919                 }
 920             }
 921
 922             if (!check_against_clause(seek, str, clause[i]))
 923                 return false;
 924         }
 925     }
 926     else
 927 #endif
 928     {
 929         /* Check for conditions. */
 930         for (i = 0; i < count; i++)
 931         {
 932             struct tagfile_entry tfe;
 933             int seek;
 934             char str[256];
 935
 936             seek = check_virtual_tags(clause[i]->tag, idx);
 937
 938             memset(str, 0, sizeof str);
 939             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 940             {
 941                 int fd = tcs->idxfd[clause[i]->tag];
 942                 lseek(fd, seek, SEEK_SET);
 943                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 944                 if (tfe.tag_length >= (int)sizeof(str))
 945                 {
 946                     logf("Too long tag read!");
 947                     break ;
 948                 }
 949
 950                 read(fd, str, tfe.tag_length);
 951
 952                 /* Check if entry has been deleted. */
 953                 if (str[0] == '\0')
 954                     break;
 955             }
 956
 957             if (!check_against_clause(seek, str, clause[i]))
 958                 return false;
 959         }
 960     }
 961
 962     return true;
 963 }
 964
 965 bool tagcache_check_clauses(struct tagcache_search *tcs,
 966                             struct tagcache_search_clause **clause, int count)
 967 {
 968     struct index_entry idx;
 969
 970     if (count == 0)
 971         return true;
 972
 973     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 974         return false;
 975
 976     return check_clauses(tcs, &idx, clause, count);
 977 }
 978
 979 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
 980 {
 981     int i;
 982
 983     /* If uniq buffer is not defined we must return true for search to work. */
 984     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
 985         && !TAGCACHE_IS_NUMERIC(tcs->type)))
 986     {
 987         return true;
 988     }
 989
 990     for (i = 0; i < tcs->unique_list_count; i++)
 991     {
 992         /* Return false if entry is found. */
 993         if (tcs->unique_list[i] == id)
 994             return false;
 995     }
 996
 997     if (tcs->unique_list_count < tcs->unique_list_capacity)
 998     {
 999         tcs->unique_list[i] = id;
1000         tcs->unique_list_count++;
1001     }
1002
1003     return true;
1004 }
1005
1006 static bool build_lookup_list(struct tagcache_search *tcs)
1007 {
1008     struct index_entry entry;
1009     int i;
1010
1011     tcs->seek_list_count = 0;
1012
1013 #ifdef HAVE_TC_RAMCACHE
1014     if (tcs->ramsearch)
1015     {
1016         int j;
1017
1018         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1019         {
1020             struct index_entry *idx = &hdr->indices[i];
1021             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1022                 break ;
1023
1024             /* Skip deleted files. */
1025             if (idx->flag & FLAG_DELETED)
1026                 continue;
1027
1028             /* Go through all filters.. */
1029             for (j = 0; j < tcs->filter_count; j++)
1030             {
1031                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1032                 {
1033                     break ;
1034                 }
1035             }
1036
1037             if (j < tcs->filter_count)
1038                 continue ;
1039
1040             /* Check for conditions. */
1041             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1042                 continue;
1043
1044             /* Add to the seek list if not already in uniq buffer. */
1045             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1046                 continue;
1047
1048             /* Lets add it. */
1049             tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1050             tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1051             tcs->seek_list_count++;
1052         }
1053
1054         tcs->seek_pos = i;
1055
1056         return tcs->seek_list_count > 0;
1057     }
1058 #endif
1059
1060     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1061             sizeof(struct master_header), SEEK_SET);
1062
1063     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1064            == sizeof(struct index_entry))
1065     {
1066         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1067             break ;
1068
1069         tcs->seek_pos++;
1070
1071         /* Check if entry has been deleted. */
1072         if (entry.flag & FLAG_DELETED)
1073             continue;
1074
1075         /* Go through all filters.. */
1076         for (i = 0; i < tcs->filter_count; i++)
1077         {
1078             if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1079                 break ;
1080         }
1081
1082         if (i < tcs->filter_count)
1083             continue ;
1084
1085         /* Check for conditions. */
1086         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1087             continue;
1088
1089         /* Add to the seek list if not already in uniq buffer. */
1090         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1091             continue;
1092
1093         /* Lets add it. */
1094         tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1095         tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1096         tcs->seek_list_count++;
1097
1098         yield();
1099     }
1100
1101     return tcs->seek_list_count > 0;
1102 }
1103
1104
1105 static void remove_files(void)
1106 {
1107     int i;
1108     char buf[MAX_PATH];
1109
1110     tc_stat.ready = false;
1111     tc_stat.ramcache = false;
1112     tc_stat.econ = false;
1113     remove(TAGCACHE_FILE_MASTER);
1114     for (i = 0; i < TAG_COUNT; i++)
1115     {
1116         if (TAGCACHE_IS_NUMERIC(i))
1117             continue;
1118
1119         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1120         remove(buf);
1121     }
1122 }
1123
1124
1125 static bool check_all_headers(void)
1126 {
1127     struct master_header myhdr;
1128     struct tagcache_header tch;
1129     int tag;
1130     int fd;
1131
1132     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1133         return false;
1134
1135     close(fd);
1136     if (myhdr.dirty)
1137     {
1138         logf("tagcache is dirty!");
1139         return false;
1140     }
1141
1142     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1143
1144     for (tag = 0; tag < TAG_COUNT; tag++)
1145     {
1146         if (TAGCACHE_IS_NUMERIC(tag))
1147             continue;
1148
1149         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1150             return false;
1151
1152         close(fd);
1153     }
1154
1155     return true;
1156 }
1157
1158 bool tagcache_search(struct tagcache_search *tcs, int tag)
1159 {
1160     struct tagcache_header tag_hdr;
1161     struct master_header   master_hdr;
1162     int i;
1163
1164     if (tcs->initialized)
1165         tagcache_search_finish(tcs);
1166
1167     while (read_lock)
1168         sleep(1);
1169
1170     memset(tcs, 0, sizeof(struct tagcache_search));
1171     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1172         return false;
1173
1174     tcs->position = sizeof(struct tagcache_header);
1175     tcs->type = tag;
1176     tcs->seek_pos = 0;
1177     tcs->seek_list_count = 0;
1178     tcs->filter_count = 0;
1179     tcs->masterfd = -1;
1180
1181     for (i = 0; i < TAG_COUNT; i++)
1182         tcs->idxfd[i] = -1;
1183
1184 #ifndef HAVE_TC_RAMCACHE
1185     tcs->ramsearch = false;
1186 #else
1187     tcs->ramsearch = tc_stat.ramcache;
1188     if (tcs->ramsearch)
1189     {
1190         tcs->entry_count = hdr->entry_count[tcs->type];
1191     }
1192     else
1193 #endif
1194     {
1195         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1196         {
1197             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1198             if (tcs->idxfd[tcs->type] < 0)
1199                 return false;
1200         }
1201
1202         /* Always open as R/W so we can pass tcs to functions that modify data also
1203          * without failing. */
1204         tcs->masterfd = open_master_fd(&master_hdr, true);
1205
1206         if (tcs->masterfd < 0)
1207             return false;
1208     }
1209
1210     tcs->valid = true;
1211     tcs->initialized = true;
1212     write_lock++;
1213
1214     return true;
1215 }
1216
1217 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1218                                  void *buffer, long length)
1219 {
1220     tcs->unique_list = (unsigned long *)buffer;
1221     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1222     tcs->unique_list_count = 0;
1223 }
1224
1225 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1226                                 int tag, int seek)
1227 {
1228     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1229         return false;
1230
1231     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1232         return false;
1233
1234     tcs->filter_tag[tcs->filter_count] = tag;
1235     tcs->filter_seek[tcs->filter_count] = seek;
1236     tcs->filter_count++;
1237
1238     return true;
1239 }
1240
1241 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1242                                 struct tagcache_search_clause *clause)
1243 {
1244     int i;
1245
1246     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1247     {
1248         logf("Too many clauses");
1249         return false;
1250     }
1251
1252     /* Check if there is already a similar filter in present (filters are
1253      * much faster than clauses).
1254      */
1255     for (i = 0; i < tcs->filter_count; i++)
1256     {
1257         if (tcs->filter_tag[i] == clause->tag)
1258             return true;
1259     }
1260
1261     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1262     {
1263         char buf[MAX_PATH];
1264
1265         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1266         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1267     }
1268
1269     tcs->clause[tcs->clause_count] = clause;
1270     tcs->clause_count++;
1271
1272     return true;
1273 }
1274
1275 /* TODO: Remove this mess. */
1276 #ifdef HAVE_DIRCACHE
1277 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1278     ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1279 #else
1280 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1281 #endif
1282
1283 static bool get_next(struct tagcache_search *tcs)
1284 {
1285     static char buf[TAG_MAXLEN+32];
1286     struct tagfile_entry entry;
1287     long flag = 0;
1288
1289     if (!tcs->valid || !tc_stat.ready)
1290         return false;
1291
1292     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1293 #ifdef HAVE_TC_RAMCACHE
1294         && !tcs->ramsearch
1295 #endif
1296         )
1297         return false;
1298
1299     /* Relative fetch. */
1300     if (tcs->filter_count > 0 || tcs->clause_count > 0
1301         || TAGCACHE_IS_NUMERIC(tcs->type))
1302     {
1303         /* Check for end of list. */
1304         if (tcs->seek_list_count == 0)
1305         {
1306             /* Try to fetch more. */
1307             if (!build_lookup_list(tcs))
1308             {
1309                 tcs->valid = false;
1310                 return false;
1311             }
1312         }
1313
1314         tcs->seek_list_count--;
1315         flag = tcs->seek_flags[tcs->seek_list_count];
1316
1317         /* Seek stream to the correct position and continue to direct fetch. */
1318         if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1319             && !TAGCACHE_IS_NUMERIC(tcs->type))
1320         {
1321             if (!open_files(tcs, tcs->type))
1322                 return false;
1323
1324             lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1325         }
1326         else
1327             tcs->position = tcs->seek_list[tcs->seek_list_count];
1328     }
1329
1330     if (TAGCACHE_IS_NUMERIC(tcs->type))
1331     {
1332         snprintf(buf, sizeof(buf), "%d", tcs->position);
1333         tcs->result_seek = tcs->position;
1334         tcs->result = buf;
1335         tcs->result_len = strlen(buf) + 1;
1336         return true;
1337     }
1338
1339     /* Direct fetch. */
1340 #ifdef HAVE_TC_RAMCACHE
1341     if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1342     {
1343         struct tagfile_entry *ep;
1344
1345         if (tcs->entry_count == 0)
1346         {
1347             tcs->valid = false;
1348             return false;
1349         }
1350         tcs->entry_count--;
1351
1352         tcs->result_seek = tcs->position;
1353
1354 # ifdef HAVE_DIRCACHE
1355         if (tcs->type == tag_filename)
1356         {
1357             dircache_copy_path((struct dirent *)tcs->position,
1358                                buf, sizeof buf);
1359             tcs->result = buf;
1360             tcs->result_len = strlen(buf) + 1;
1361             tcs->idx_id = FLAG_GET_ATTR(flag);
1362             tcs->ramresult = false;
1363
1364             return true;
1365         }
1366 # endif
1367
1368         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1369         tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1370         tcs->result = ep->tag_data;
1371         tcs->result_len = strlen(tcs->result) + 1;
1372         tcs->idx_id = ep->idx_id;
1373         tcs->ramresult = true;
1374
1375         return true;
1376     }
1377     else
1378 #endif
1379     {
1380         if (!open_files(tcs, tcs->type))
1381             return false;
1382
1383         tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1384         if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1385                  tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1386         {
1387             /* End of data. */
1388             tcs->valid = false;
1389             return false;
1390         }
1391     }
1392
1393     if (entry.tag_length > (long)sizeof(buf))
1394     {
1395         tcs->valid = false;
1396         logf("too long tag #2");
1397         return false;
1398     }
1399
1400     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1401     {
1402         tcs->valid = false;
1403         logf("read error #4");
1404         return false;
1405     }
1406
1407     tcs->result = buf;
1408     tcs->result_len = strlen(tcs->result) + 1;
1409     tcs->idx_id = entry.idx_id;
1410     tcs->ramresult = false;
1411
1412     return true;
1413 }
1414
1415 bool tagcache_get_next(struct tagcache_search *tcs)
1416 {
1417     while (get_next(tcs))
1418     {
1419         if (tcs->result_len > 1)
1420             return true;
1421     }
1422
1423     return false;
1424 }
1425
1426 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1427                        int tag, char *buf, long size)
1428 {
1429     struct index_entry idx;
1430
1431     *buf = '\0';
1432     if (!get_index(tcs->masterfd, idxid, &idx, true))
1433         return false;
1434
1435     return retrieve(tcs, &idx, tag, buf, size);
1436 }
1437
1438 static bool update_master_header(void)
1439 {
1440     struct master_header myhdr;
1441     int fd;
1442
1443     if (!tc_stat.ready)
1444         return false;
1445
1446     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1447         return false;
1448
1449     myhdr.serial = current_tcmh.serial;
1450     myhdr.commitid = current_tcmh.commitid;
1451     myhdr.dirty = current_tcmh.dirty;
1452
1453     /* Write it back */
1454     lseek(fd, 0, SEEK_SET);
1455     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1456     close(fd);
1457
1458 #ifdef HAVE_TC_RAMCACHE
1459     if (hdr)
1460     {
1461         hdr->h.serial = current_tcmh.serial;
1462         hdr->h.commitid = current_tcmh.commitid;
1463         hdr->h.dirty = current_tcmh.dirty;
1464     }
1465 #endif
1466
1467     return true;
1468 }
1469
1470 #if 0
1471
1472 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1473 {
1474     struct tagentry *entry;
1475
1476     if (tcs->type != tag_title)
1477         return ;
1478
1479     /* We will need reserve buffer for this. */
1480     if (tcs->ramcache)
1481     {
1482         struct tagfile_entry *ep;
1483
1484         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1485         tcs->seek_list[tcs->seek_list_count];
1486     }
1487
1488     entry = find_entry_ram();
1489
1490 }
1491 #endif
1492
1493 void tagcache_search_finish(struct tagcache_search *tcs)
1494 {
1495     int i;
1496
1497     if (!tcs->initialized)
1498         return;
1499
1500     if (tcs->masterfd >= 0)
1501     {
1502         close(tcs->masterfd);
1503         tcs->masterfd = -1;
1504     }
1505
1506     for (i = 0; i < TAG_COUNT; i++)
1507     {
1508         if (tcs->idxfd[i] >= 0)
1509         {
1510             close(tcs->idxfd[i]);
1511             tcs->idxfd[i] = -1;
1512         }
1513     }
1514
1515     tcs->ramsearch = false;
1516     tcs->valid = false;
1517     tcs->initialized = 0;
1518     if (write_lock > 0)
1519         write_lock--;
1520 }
1521
1522 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1523 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1524 {
1525     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1526 }
1527
1528 static long get_tag_numeric(const struct index_entry *entry, int tag)
1529 {
1530     return check_virtual_tags(tag, entry);
1531 }
1532
1533 static char* get_tag_string(const struct index_entry *entry, int tag)
1534 {
1535     char* s = get_tag(entry, tag)->tag_data;
1536     return strcmp(s, UNTAGGED) ? s : NULL;
1537 }
1538
1539 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1540 {
1541     struct index_entry *entry;
1542     int idx_id;
1543
1544     if (!tc_stat.ready)
1545         return false;
1546
1547     /* Find the corresponding entry in tagcache. */
1548     idx_id = find_entry_ram(filename, NULL);
1549     if (idx_id < 0 || !tc_stat.ramcache)
1550         return false;
1551
1552     entry = &hdr->indices[idx_id];
1553
1554     id3->title        = get_tag_string(entry, tag_title);
1555     id3->artist       = get_tag_string(entry, tag_artist);
1556     id3->album        = get_tag_string(entry, tag_album);
1557     id3->genre_string = get_tag_string(entry, tag_genre);
1558     id3->composer     = get_tag_string(entry, tag_composer);
1559     id3->comment      = get_tag_string(entry, tag_comment);
1560     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1561     id3->grouping     = get_tag_string(entry, tag_grouping);
1562
1563     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1564     id3->rating     = get_tag_numeric(entry, tag_rating);
1565     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1566     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1567     id3->year       = get_tag_numeric(entry, tag_year);
1568
1569     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1570     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1571     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1572     if (id3->bitrate == 0)
1573         id3->bitrate = 1;
1574
1575     return true;
1576 }
1577 #endif
1578
1579 static inline void write_item(const char *item)
1580 {
1581     int len = strlen(item) + 1;
1582
1583     data_size += len;
1584     write(cachefd, item, len);
1585 }
1586
1587 static int check_if_empty(char **tag)
1588 {
1589     int length;
1590
1591     if (*tag == NULL || **tag == '\0')
1592     {
1593         *tag = UNTAGGED;
1594         return sizeof(UNTAGGED); /* Tag length */
1595     }
1596
1597     length = strlen(*tag);
1598     if (length > TAG_MAXLEN)
1599     {
1600         logf("over length tag: %s", *tag);
1601         length = TAG_MAXLEN;
1602         (*tag)[length] = '\0';
1603     }
1604
1605     return length + 1;
1606 }
1607
1608 #define ADD_TAG(entry,tag,data) \
1609     /* Adding tag */ \
1610     entry.tag_offset[tag] = offset; \
1611     entry.tag_length[tag] = check_if_empty(data); \
1612     offset += entry.tag_length[tag]
1613
1614 static void add_tagcache(char *path, unsigned long mtime
1615 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1616                          ,const struct dirent *dc
1617 #endif
1618                          )
1619 {
1620     struct mp3entry id3;
1621     struct temp_file_entry entry;
1622     bool ret;
1623     int fd;
1624     int idx_id = -1;
1625     char tracknumfix[3];
1626     int offset = 0;
1627     int path_length = strlen(path);
1628     bool has_albumartist;
1629     bool has_grouping;
1630
1631     if (cachefd < 0)
1632         return ;
1633
1634     /* Check for overlength file path. */
1635     if (path_length > TAG_MAXLEN)
1636     {
1637         /* Path can't be shortened. */
1638         logf("Too long path: %s", path);
1639         return ;
1640     }
1641
1642     /* Check if the file is supported. */
1643     if (probe_file_format(path) == AFMT_UNKNOWN)
1644         return ;
1645
1646     /* Check if the file is already cached. */
1647 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1648     if (tc_stat.ramcache && is_dircache_intact())
1649     {
1650         idx_id = find_entry_ram(path, dc);
1651     }
1652     else
1653 #endif
1654     {
1655         if (filenametag_fd >= 0)
1656         {
1657             idx_id = find_entry_disk(path);
1658         }
1659     }
1660
1661     /* Check if file has been modified. */
1662     if (idx_id >= 0)
1663     {
1664         struct index_entry idx;
1665
1666         /* TODO: Mark that the index exists (for fast reverse scan) */
1667         //found_idx[idx_id/8] |= idx_id%8;
1668
1669         if (!get_index(-1, idx_id, &idx, true))
1670         {
1671             logf("failed to retrieve index entry");
1672             return ;
1673         }
1674
1675         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1676         {
1677             /* No changes to file. */
1678             return ;
1679         }
1680
1681         /* Metadata might have been changed. Delete the entry. */
1682         logf("Re-adding: %s", path);
1683         if (!delete_entry(idx_id))
1684         {
1685             logf("delete_entry failed: %d", idx_id);
1686             return ;
1687         }
1688     }
1689
1690     fd = open(path, O_RDONLY);
1691     if (fd < 0)
1692     {
1693         logf("open fail: %s", path);
1694         return ;
1695     }
1696
1697     memset(&id3, 0, sizeof(struct mp3entry));
1698     memset(&entry, 0, sizeof(struct temp_file_entry));
1699     memset(&tracknumfix, 0, sizeof(tracknumfix));
1700     ret = get_metadata(&id3, fd, path);
1701     close(fd);
1702
1703     if (!ret)
1704         return ;
1705
1706     logf("-> %s", path);
1707
1708     /* Generate track number if missing. */
1709     if (id3.tracknum <= 0)
1710     {
1711         const char *p = strrchr(path, '.');
1712
1713         if (p == NULL)
1714             p = &path[strlen(path)-1];
1715
1716         while (*p != '/')
1717         {
1718             if (isdigit(*p) && isdigit(*(p-1)))
1719             {
1720                 tracknumfix[1] = *p--;
1721                 tracknumfix[0] = *p;
1722                 break;
1723             }
1724             p--;
1725         }
1726
1727         if (tracknumfix[0] != '\0')
1728         {
1729             id3.tracknum = atoi(tracknumfix);
1730             /* Set a flag to indicate track number has been generated. */
1731             entry.flag |= FLAG_TRKNUMGEN;
1732         }
1733         else
1734         {
1735             /* Unable to generate track number. */
1736             id3.tracknum = -1;
1737         }
1738     }
1739
1740     /* Numeric tags */
1741     entry.tag_offset[tag_year] = id3.year;
1742     entry.tag_offset[tag_discnumber] = id3.discnum;
1743     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1744     entry.tag_offset[tag_length] = id3.length;
1745     entry.tag_offset[tag_bitrate] = id3.bitrate;
1746     entry.tag_offset[tag_mtime] = mtime;
1747
1748     /* String tags. */
1749     has_albumartist = id3.albumartist != NULL
1750         && strlen(id3.albumartist) > 0;
1751     has_grouping = id3.grouping != NULL
1752         && strlen(id3.grouping) > 0;
1753
1754     ADD_TAG(entry, tag_filename, &path);
1755     ADD_TAG(entry, tag_title, &id3.title);
1756     ADD_TAG(entry, tag_artist, &id3.artist);
1757     ADD_TAG(entry, tag_album, &id3.album);
1758     ADD_TAG(entry, tag_genre, &id3.genre_string);
1759     ADD_TAG(entry, tag_composer, &id3.composer);
1760     ADD_TAG(entry, tag_comment, &id3.comment);
1761     if (has_albumartist)
1762     {
1763         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1764     }
1765     else
1766     {
1767         ADD_TAG(entry, tag_albumartist, &id3.artist);
1768     }
1769     if (has_grouping)
1770     {
1771         ADD_TAG(entry, tag_grouping, &id3.grouping);
1772     }
1773     else
1774     {
1775         ADD_TAG(entry, tag_grouping, &id3.title);
1776     }
1777     entry.data_length = offset;
1778
1779     /* Write the header */
1780     write(cachefd, &entry, sizeof(struct temp_file_entry));
1781
1782     /* And tags also... Correct order is critical */
1783     write_item(path);
1784     write_item(id3.title);
1785     write_item(id3.artist);
1786     write_item(id3.album);
1787     write_item(id3.genre_string);
1788     write_item(id3.composer);
1789     write_item(id3.comment);
1790     if (has_albumartist)
1791     {
1792         write_item(id3.albumartist);
1793     }
1794     else
1795     {
1796         write_item(id3.artist);
1797     }
1798     if (has_grouping)
1799     {
1800         write_item(id3.grouping);
1801     }
1802     else
1803     {
1804         write_item(id3.title);
1805     }
1806     total_entry_count++;
1807 }
1808
1809 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1810 {
1811     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1812     int len = strlen(str)+1;
1813     int i;
1814     unsigned crc32;
1815     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1816     char buf[TAG_MAXLEN+32];
1817
1818     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1819         buf[i] = tolower(str[i]);
1820     buf[i] = '\0';
1821
1822     crc32 = crc_32(buf, i, 0xffffffff);
1823
1824     if (unique)
1825     {
1826         /* Check if the crc does not exist -> entry does not exist for sure. */
1827         for (i = 0; i < tempbufidx; i++)
1828         {
1829             if (crcbuf[-i] != crc32)
1830                 continue;
1831
1832             if (!strcasecmp(str, index[i].str))
1833             {
1834                 if (id < 0 || id >= lookup_buffer_depth)
1835                 {
1836                     logf("lookup buf overf.: %d", id);
1837                     return false;
1838                 }
1839
1840                 lookup[id] = &index[i];
1841                 return true;
1842             }
1843         }
1844     }
1845
1846     /* Insert to CRC buffer. */
1847     crcbuf[-tempbufidx] = crc32;
1848     tempbuf_left -= 4;
1849
1850     /* Insert it to the buffer. */
1851     tempbuf_left -= len;
1852     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1853         return false;
1854
1855     if (id >= lookup_buffer_depth)
1856     {
1857         logf("lookup buf overf. #2: %d", id);
1858         return false;
1859     }
1860
1861     if (id >= 0)
1862     {
1863         lookup[id] = &index[tempbufidx];
1864         index[tempbufidx].idlist.id = id;
1865     }
1866     else
1867         index[tempbufidx].idlist.id = -1;
1868
1869     index[tempbufidx].idlist.next = NULL;
1870     index[tempbufidx].idx_id = idx_id;
1871     index[tempbufidx].seek = -1;
1872     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1873     memcpy(index[tempbufidx].str, str, len);
1874     tempbuf_pos += len;
1875     tempbufidx++;
1876
1877     return true;
1878 }
1879
1880 static int compare(const void *p1, const void *p2)
1881 {
1882     do_timed_yield();
1883
1884     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1885     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1886
1887     if (strcmp(e1->str, UNTAGGED) == 0)
1888     {
1889         if (strcmp(e2->str, UNTAGGED) == 0)
1890             return 0;
1891         return -1;
1892     }
1893     else if (strcmp(e2->str, UNTAGGED) == 0)
1894         return 1;
1895
1896     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1897 }
1898
1899 static int tempbuf_sort(int fd)
1900 {
1901     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1902     struct tagfile_entry fe;
1903     int i;
1904     int length;
1905
1906     /* Generate reverse lookup entries. */
1907     for (i = 0; i < lookup_buffer_depth; i++)
1908     {
1909         struct tempbuf_id_list *idlist;
1910
1911         if (!lookup[i])
1912             continue;
1913
1914         if (lookup[i]->idlist.id == i)
1915             continue;
1916
1917         idlist = &lookup[i]->idlist;
1918         while (idlist->next != NULL)
1919             idlist = idlist->next;
1920
1921         tempbuf_left -= sizeof(struct tempbuf_id_list);
1922         if (tempbuf_left - 4 < 0)
1923             return -1;
1924
1925         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1926         if (tempbuf_pos & 0x03)
1927         {
1928             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1929             tempbuf_left -= 3;
1930             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1931         }
1932         tempbuf_pos += sizeof(struct tempbuf_id_list);
1933
1934         idlist = idlist->next;
1935         idlist->id = i;
1936         idlist->next = NULL;
1937
1938         do_timed_yield();
1939     }
1940
1941     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1942     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1943
1944     for (i = 0; i < tempbufidx; i++)
1945     {
1946         struct tempbuf_id_list *idlist = &index[i].idlist;
1947
1948         /* Fix the lookup list. */
1949         while (idlist != NULL)
1950         {
1951             if (idlist->id >= 0)
1952                 lookup[idlist->id] = &index[i];
1953             idlist = idlist->next;
1954         }
1955
1956         index[i].seek = lseek(fd, 0, SEEK_CUR);
1957         length = strlen(index[i].str) + 1;
1958         fe.tag_length = length;
1959         fe.idx_id = index[i].idx_id;
1960
1961         /* Check the chunk alignment. */
1962         if ((fe.tag_length + sizeof(struct tagfile_entry))
1963             % TAGFILE_ENTRY_CHUNK_LENGTH)
1964         {
1965             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1966                 ((fe.tag_length + sizeof(struct tagfile_entry))
1967                  % TAGFILE_ENTRY_CHUNK_LENGTH);
1968         }
1969
1970 #ifdef TAGCACHE_STRICT_ALIGN
1971         /* Make sure the entry is long aligned. */
1972         if (index[i].seek & 0x03)
1973         {
1974             logf("tempbuf_sort: alignment error!");
1975             return -3;
1976         }
1977 #endif
1978
1979         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
1980             sizeof(struct tagfile_entry))
1981         {
1982             logf("tempbuf_sort: write error #1");
1983             return -1;
1984         }
1985
1986         if (write(fd, index[i].str, length) != length)
1987         {
1988             logf("tempbuf_sort: write error #2");
1989             return -2;
1990         }
1991
1992         /* Write some padding. */
1993         if (fe.tag_length - length > 0)
1994             write(fd, "XXXXXXXX", fe.tag_length - length);
1995     }
1996
1997     return i;
1998 }
1999
2000 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2001 {
2002     if (id < 0 || id >= lookup_buffer_depth)
2003         return NULL;
2004
2005     return lookup[id];
2006 }
2007
2008
2009 inline static int tempbuf_find_location(int id)
2010 {
2011     struct tempbuf_searchidx *entry;
2012
2013     entry = tempbuf_locate(id);
2014     if (entry == NULL)
2015         return -1;
2016
2017     return entry->seek;
2018 }
2019
2020 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2021 {
2022     struct master_header tcmh;
2023     struct index_entry idx;
2024     int masterfd;
2025     int masterfd_pos;
2026     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2027     int max_entries;
2028     int entries_processed = 0;
2029     int i, j;
2030     char buf[TAG_MAXLEN];
2031
2032     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2033
2034     logf("Building numeric indices...");
2035     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2036
2037     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2038         return false;
2039
2040     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2041                          SEEK_CUR);
2042     if (masterfd_pos == filesize(masterfd))
2043     {
2044         logf("we can't append!");
2045         close(masterfd);
2046         return false;
2047     }
2048
2049     while (entries_processed < h->entry_count)
2050     {
2051         int count = MIN(h->entry_count - entries_processed, max_entries);
2052
2053         /* Read in as many entries as possible. */
2054         for (i = 0; i < count; i++)
2055         {
2056             struct temp_file_entry *tfe = &entrybuf[i];
2057             int datastart;
2058
2059             /* Read in numeric data. */
2060             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2061                 sizeof(struct temp_file_entry))
2062             {
2063                 logf("read fail #1");
2064                 close(masterfd);
2065                 return false;
2066             }
2067
2068             datastart = lseek(tmpfd, 0, SEEK_CUR);
2069
2070             /**
2071              * Read string data from the following tags:
2072              * - tag_filename
2073              * - tag_artist
2074              * - tag_album
2075              * - tag_title
2076              *
2077              * A crc32 hash is calculated from the read data
2078              * and stored back to the data offset field kept in memory.
2079              */
2080 #define tmpdb_read_string_tag(tag) \
2081     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2082     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2083     { \
2084         logf("read fail: buffer overflow"); \
2085         close(masterfd); \
2086         return false; \
2087     } \
2088     \
2089     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2090         tfe->tag_length[tag]) \
2091     { \
2092         logf("read fail #2"); \
2093         close(masterfd); \
2094         return false; \
2095     } \
2096     \
2097     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2098     lseek(tmpfd, datastart, SEEK_SET)
2099
2100             tmpdb_read_string_tag(tag_filename);
2101             tmpdb_read_string_tag(tag_artist);
2102             tmpdb_read_string_tag(tag_album);
2103             tmpdb_read_string_tag(tag_title);
2104
2105             /* Seek to the end of the string data. */
2106             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2107         }
2108
2109         /* Backup the master index position. */
2110         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2111         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2112
2113         /* Check if we can resurrect some deleted runtime statistics data. */
2114         for (i = 0; i < tcmh.tch.entry_count; i++)
2115         {
2116             /* Read the index entry. */
2117             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2118                 != sizeof(struct index_entry))
2119             {
2120                 logf("read fail #3");
2121                 close(masterfd);
2122                 return false;
2123             }
2124
2125             /**
2126              * Skip unless the entry is marked as being deleted
2127              * or the data has already been resurrected.
2128              */
2129             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2130                 continue;
2131
2132             /* Now try to match the entry. */
2133             /**
2134              * To succesfully match a song, the following conditions
2135              * must apply:
2136              *
2137              * For numeric fields: tag_length
2138              * - Full identical match is required
2139              *
2140              * If tag_filename matches, no further checking necessary.
2141              *
2142              * For string hashes: tag_artist, tag_album, tag_title
2143              * - Two of these must match
2144              */
2145             for (j = 0; j < count; j++)
2146             {
2147                 struct temp_file_entry *tfe = &entrybuf[j];
2148
2149                 /* Try to match numeric fields first. */
2150                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2151                     continue;
2152
2153                 /* Now it's time to do the hash matching. */
2154                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2155                 {
2156                     int match_count = 0;
2157
2158                     /* No filename match, check if we can match two other tags. */
2159 #define tmpdb_match(tag) \
2160     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2161         match_count++
2162
2163                     tmpdb_match(tag_artist);
2164                     tmpdb_match(tag_album);
2165                     tmpdb_match(tag_title);
2166
2167                     if (match_count < 2)
2168                     {
2169                         /* Still no match found, give up. */
2170                         continue;
2171                     }
2172                 }
2173
2174                 /* A match found, now copy & resurrect the statistical data. */
2175 #define tmpdb_copy_tag(tag) \
2176     tfe->tag_offset[tag] = idx.tag_seek[tag]
2177
2178                 tmpdb_copy_tag(tag_playcount);
2179                 tmpdb_copy_tag(tag_rating);
2180                 tmpdb_copy_tag(tag_playtime);
2181                 tmpdb_copy_tag(tag_lastplayed);
2182                 tmpdb_copy_tag(tag_commitid);
2183
2184                 /* Avoid processing this entry again. */
2185                 idx.flag |= FLAG_RESURRECTED;
2186
2187                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2188                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2189                     != sizeof(struct index_entry))
2190                 {
2191                     logf("masterfd writeback fail #1");
2192                     close(masterfd);
2193                     return false;
2194                 }
2195
2196                 logf("Entry resurrected");
2197             }
2198         }
2199
2200
2201         /* Restore the master index position. */
2202         lseek(masterfd, masterfd_pos, SEEK_SET);
2203
2204         /* Commit the data to the index. */
2205         for (i = 0; i < count; i++)
2206         {
2207             int loc = lseek(masterfd, 0, SEEK_CUR);
2208
2209             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2210                 != sizeof(struct index_entry))
2211             {
2212                 logf("read fail #3");
2213                 close(masterfd);
2214                 return false;
2215             }
2216
2217             for (j = 0; j < TAG_COUNT; j++)
2218             {
2219                 if (!TAGCACHE_IS_NUMERIC(j))
2220                     continue;
2221
2222                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2223             }
2224             idx.flag = entrybuf[i].flag;
2225
2226             if (idx.tag_seek[tag_commitid])
2227             {
2228                 /* Data has been resurrected. */
2229                 idx.flag |= FLAG_DIRTYNUM;
2230             }
2231             else if (tc_stat.ready && current_tcmh.commitid > 0)
2232             {
2233                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2234                 idx.flag |= FLAG_DIRTYNUM;
2235             }
2236
2237             /* Write back the updated index. */
2238             lseek(masterfd, loc, SEEK_SET);
2239             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2240                 != sizeof(struct index_entry))
2241             {
2242                 logf("write fail");
2243                 close(masterfd);
2244                 return false;
2245             }
2246         }
2247
2248         entries_processed += count;
2249         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2250     }
2251
2252     close(masterfd);
2253
2254     return true;
2255 }
2256
2257 /**
2258  * Return values:
2259  *     > 0   success
2260  *    == 0   temporary failure
2261  *     < 0   fatal error
2262  */
2263 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2264 {
2265     int i;
2266     struct tagcache_header tch;
2267     struct master_header   tcmh;
2268     struct index_entry idxbuf[IDX_BUF_DEPTH];
2269     int idxbuf_pos;
2270     char buf[TAG_MAXLEN+32];
2271     int fd = -1, masterfd;
2272     bool error = false;
2273     int init;
2274     int masterfd_pos;
2275
2276     logf("Building index: %d", index_type);
2277
2278     /* Check the number of entries we need to allocate ram for. */
2279     commit_entry_count = h->entry_count + 1;
2280
2281     masterfd = open_master_fd(&tcmh, false);
2282     if (masterfd >= 0)
2283     {
2284         commit_entry_count += tcmh.tch.entry_count;
2285         close(masterfd);
2286     }
2287     else
2288         remove_files(); /* Just to be sure we are clean. */
2289
2290     /* Open the index file, which contains the tag names. */
2291     fd = open_tag_fd(&tch, index_type, true);
2292     if (fd >= 0)
2293     {
2294         logf("tch.datasize=%ld", tch.datasize);
2295         lookup_buffer_depth = 1 +
2296         /* First part */ commit_entry_count +
2297         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2298     }
2299     else
2300     {
2301         lookup_buffer_depth = 1 +
2302         /* First part */ commit_entry_count +
2303         /* Second part */ 0;
2304     }
2305
2306     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2307     logf("commit_entry_count=%ld", commit_entry_count);
2308
2309     /* Allocate buffer for all index entries from both old and new
2310      * tag files. */
2311     tempbufidx = 0;
2312     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2313
2314     /* Allocate lookup buffer. The first portion of commit_entry_count
2315      * contains the new tags in the temporary file and the second
2316      * part for locating entries already in the db.
2317      *
2318      *  New tags  Old tags
2319      * +---------+---------------------------+
2320      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2321      * +---------+---------------------------+
2322      *
2323      * Old tags are inserted to a temporary buffer with position:
2324      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2325      * And new tags with index:
2326      *     tempbuf_insert(idx, ...);
2327      *
2328      * The buffer is sorted and written into tag file:
2329      *     tempbuf_sort(...);
2330      * leaving master index locations messed up.
2331      *
2332      * That is fixed using the lookup buffer for old tags:
2333      *     new_seek = tempbuf_find_location(old_seek, ...);
2334      * and for new tags:
2335      *     new_seek = tempbuf_find_location(idx);
2336      */
2337     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2338     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2339     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2340
2341     /* And calculate the remaining data space used mainly for storing
2342      * tag data (strings). */
2343     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2344     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2345     {
2346         logf("Buffer way too small!");
2347         return 0;
2348     }
2349
2350     if (fd >= 0)
2351     {
2352         /**
2353          * If tag file contains unique tags (sorted index), we will load
2354          * it entirely into memory so we can resort it later for use with
2355          * chunked browsing.
2356          */
2357         if (TAGCACHE_IS_SORTED(index_type))
2358         {
2359             logf("loading tags...");
2360             for (i = 0; i < tch.entry_count; i++)
2361             {
2362                 struct tagfile_entry entry;
2363                 int loc = lseek(fd, 0, SEEK_CUR);
2364                 bool ret;
2365
2366                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2367                     != sizeof(struct tagfile_entry))
2368                 {
2369                     logf("read error #7");
2370                     close(fd);
2371                     return -2;
2372                 }
2373
2374                 if (entry.tag_length >= (int)sizeof(buf))
2375                 {
2376                     logf("too long tag #3");
2377                     close(fd);
2378                     return -2;
2379                 }
2380
2381                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2382                 {
2383                     logf("read error #8");
2384                     close(fd);
2385                     return -2;
2386                 }
2387
2388                 /* Skip deleted entries. */
2389                 if (buf[0] == '\0')
2390                     continue;
2391
2392                 /**
2393                  * Save the tag and tag id in the memory buffer. Tag id
2394                  * is saved so we can later reindex the master lookup
2395                  * table when the index gets resorted.
2396                  */
2397                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2398                                      + commit_entry_count, entry.idx_id,
2399                                      TAGCACHE_IS_UNIQUE(index_type));
2400                 if (!ret)
2401                 {
2402                     close(fd);
2403                     return -3;
2404                 }
2405                 do_timed_yield();
2406             }
2407             logf("done");
2408         }
2409         else
2410             tempbufidx = tch.entry_count;
2411     }
2412     else
2413     {
2414         /**
2415          * Creating new index file to store the tags. No need to preload
2416          * anything whether the index type is sorted or not.
2417          */
2418         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2419         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2420         if (fd < 0)
2421         {
2422             logf("%s open fail", buf);
2423             return -2;
2424         }
2425
2426         tch.magic = TAGCACHE_MAGIC;
2427         tch.entry_count = 0;
2428         tch.datasize = 0;
2429
2430         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2431             != sizeof(struct tagcache_header))
2432         {
2433             logf("header write failed");
2434             close(fd);
2435             return -2;
2436         }
2437     }
2438
2439     /* Loading the tag lookup file as "master file". */
2440     logf("Loading index file");
2441     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2442
2443     if (masterfd < 0)
2444     {
2445         logf("Creating new DB");
2446         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2447
2448         if (masterfd < 0)
2449         {
2450             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2451             close(fd);
2452             return -2;
2453         }
2454
2455         /* Write the header (write real values later). */
2456         memset(&tcmh, 0, sizeof(struct master_header));
2457         tcmh.tch = *h;
2458         tcmh.tch.entry_count = 0;
2459         tcmh.tch.datasize = 0;
2460         tcmh.dirty = true;
2461         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2462         init = true;
2463         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2464     }
2465     else
2466     {
2467         /**
2468          * Master file already exists so we need to process the current
2469          * file first.
2470          */
2471         init = false;
2472
2473         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2474             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2475         {
2476             logf("header error");
2477             close(fd);
2478             close(masterfd);
2479             return -2;
2480         }
2481
2482         /**
2483          * If we reach end of the master file, we need to expand it to
2484          * hold new tags. If the current index is not sorted, we can
2485          * simply append new data to end of the file.
2486          * However, if the index is sorted, we need to update all tag
2487          * pointers in the master file for the current index.
2488          */
2489         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2490             SEEK_CUR);
2491         if (masterfd_pos == filesize(masterfd))
2492         {
2493             logf("appending...");
2494             init = true;
2495         }
2496     }
2497
2498     /**
2499      * Load new unique tags in memory to be sorted later and added
2500      * to the master lookup file.
2501      */
2502     if (TAGCACHE_IS_SORTED(index_type))
2503     {
2504         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2505         /* h is the header of the temporary file containing new tags. */
2506         logf("inserting new tags...");
2507         for (i = 0; i < h->entry_count; i++)
2508         {
2509             struct temp_file_entry entry;
2510
2511             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2512                 sizeof(struct temp_file_entry))
2513             {
2514                 logf("read fail #3");
2515                 error = true;
2516                 goto error_exit;
2517             }
2518
2519             /* Read data. */
2520             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2521             {
2522                 logf("too long entry!");
2523                 error = true;
2524                 goto error_exit;
2525             }
2526
2527             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2528             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2529                 entry.tag_length[index_type])
2530             {
2531                 logf("read fail #4");
2532                 error = true;
2533                 goto error_exit;
2534             }
2535
2536             if (TAGCACHE_IS_UNIQUE(index_type))
2537                 error = !tempbuf_insert(buf, i, -1, true);
2538             else
2539                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2540
2541             if (error)
2542             {
2543                 logf("insert error");
2544                 goto error_exit;
2545             }
2546
2547             /* Skip to next. */
2548             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2549                     entry.tag_length[index_type], SEEK_CUR);
2550             do_timed_yield();
2551         }
2552         logf("done");
2553
2554         /* Sort the buffer data and write it to the index file. */
2555         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2556         i = tempbuf_sort(fd);
2557         if (i < 0)
2558             goto error_exit;
2559         logf("sorted %d tags", i);
2560
2561         /**
2562          * Now update all indexes in the master lookup file.
2563          */
2564         logf("updating indices...");
2565         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2566         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2567         {
2568             int j;
2569             int loc = lseek(masterfd, 0, SEEK_CUR);
2570
2571             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2572
2573             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2574                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2575             {
2576                 logf("read fail #5");
2577                 error = true;
2578                 goto error_exit ;
2579             }
2580             lseek(masterfd, loc, SEEK_SET);
2581
2582             for (j = 0; j < idxbuf_pos; j++)
2583             {
2584                 if (idxbuf[j].flag & FLAG_DELETED)
2585                 {
2586                     /* We can just ignore deleted entries. */
2587                     // idxbuf[j].tag_seek[index_type] = 0;
2588                     continue;
2589                 }
2590
2591                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2592                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2593                     + commit_entry_count);
2594
2595                 if (idxbuf[j].tag_seek[index_type] < 0)
2596                 {
2597                     logf("update error: %d/%d/%d",
2598                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2599                     error = true;
2600                     goto error_exit;
2601                 }
2602
2603                 do_timed_yield();
2604             }
2605
2606             /* Write back the updated index. */
2607             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2608                         index_entry_ec, tc_stat.econ) !=
2609                 (int)sizeof(struct index_entry)*idxbuf_pos)
2610             {
2611                 logf("write fail");
2612                 error = true;
2613                 goto error_exit;
2614             }
2615         }
2616         logf("done");
2617     }
2618
2619     /**
2620      * Walk through the temporary file containing the new tags.
2621      */
2622     // build_normal_index(h, tmpfd, masterfd, idx);
2623     logf("updating new indices...");
2624     lseek(masterfd, masterfd_pos, SEEK_SET);
2625     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2626     lseek(fd, 0, SEEK_END);
2627     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2628     {
2629         int j;
2630
2631         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2632         if (init)
2633         {
2634             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2635         }
2636         else
2637         {
2638             int loc = lseek(masterfd, 0, SEEK_CUR);
2639
2640             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2641                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2642             {
2643                 logf("read fail #6");
2644                 error = true;
2645                 break ;
2646             }
2647             lseek(masterfd, loc, SEEK_SET);
2648         }
2649
2650         /* Read entry headers. */
2651         for (j = 0; j < idxbuf_pos; j++)
2652         {
2653             if (!TAGCACHE_IS_SORTED(index_type))
2654             {
2655                 struct temp_file_entry entry;
2656                 struct tagfile_entry fe;
2657
2658                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2659                     sizeof(struct temp_file_entry))
2660                 {
2661                     logf("read fail #7");
2662                     error = true;
2663                     break ;
2664                 }
2665
2666                 /* Read data. */
2667                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2668                 {
2669                     logf("too long entry!");
2670                     logf("length=%d", entry.tag_length[index_type]);
2671                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2672                     error = true;
2673                     break ;
2674                 }
2675
2676                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2677                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2678                     entry.tag_length[index_type])
2679                 {
2680                     logf("read fail #8");
2681                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2682                     logf("length=0x%02x", entry.tag_length[index_type]);
2683                     error = true;
2684                     break ;
2685                 }
2686
2687                 /* Write to index file. */
2688                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2689                 fe.tag_length = entry.tag_length[index_type];
2690                 fe.idx_id = tcmh.tch.entry_count + i + j;
2691                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2692                 write(fd, buf, fe.tag_length);
2693                 tempbufidx++;
2694
2695                 /* Skip to next. */
2696                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2697                       entry.tag_length[index_type], SEEK_CUR);
2698             }
2699             else
2700             {
2701                 /* Locate the correct entry from the sorted array. */
2702                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2703                 if (idxbuf[j].tag_seek[index_type] < 0)
2704                 {
2705                     logf("entry not found (%d)", j);
2706                     error = true;
2707                     break ;
2708                 }
2709             }
2710         }
2711
2712         /* Write index. */
2713         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2714                   index_entry_ec, tc_stat.econ) !=
2715             (int)sizeof(struct index_entry)*idxbuf_pos)
2716         {
2717             logf("tagcache: write fail #4");
2718             error = true;
2719             break ;
2720         }
2721
2722         do_timed_yield();
2723     }
2724     logf("done");
2725
2726     /* Finally write the header. */
2727     tch.magic = TAGCACHE_MAGIC;
2728     tch.entry_count = tempbufidx;
2729     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2730     lseek(fd, 0, SEEK_SET);
2731     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2732
2733     if (index_type != tag_filename)
2734         h->datasize += tch.datasize;
2735     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2736     error_exit:
2737
2738     close(fd);
2739     close(masterfd);
2740
2741     if (error)
2742         return -2;
2743
2744     return 1;
2745 }
2746
2747 static bool commit(void)
2748 {
2749     struct tagcache_header tch;
2750     struct master_header   tcmh;
2751     int i, len, rc;
2752     int tmpfd;
2753     int masterfd;
2754 #ifdef HAVE_DIRCACHE
2755     bool dircache_buffer_stolen = false;
2756 #endif
2757     bool local_allocation = false;
2758
2759     logf("committing tagcache");
2760
2761     while (write_lock)
2762         sleep(1);
2763
2764     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2765     if (tmpfd < 0)
2766     {
2767         logf("nothing to commit");
2768         return true;
2769     }
2770
2771
2772     /* Load the header. */
2773     len = sizeof(struct tagcache_header);
2774     rc = read(tmpfd, &tch, len);
2775
2776     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2777     {
2778         logf("incorrect header");
2779         close(tmpfd);
2780         remove(TAGCACHE_FILE_TEMP);
2781         return false;
2782     }
2783
2784     if (tch.entry_count == 0)
2785     {
2786         logf("nothing to commit");
2787         close(tmpfd);
2788         remove(TAGCACHE_FILE_TEMP);
2789         return true;
2790     }
2791
2792 #ifdef HAVE_EEPROM_SETTINGS
2793     remove(TAGCACHE_STATEFILE);
2794 #endif
2795
2796     /* At first be sure to unload the ramcache! */
2797 #ifdef HAVE_TC_RAMCACHE
2798     tc_stat.ramcache = false;
2799 #endif
2800
2801     read_lock++;
2802
2803     /* Try to steal every buffer we can :) */
2804     if (tempbuf_size == 0)
2805         local_allocation = true;
2806
2807 #ifdef HAVE_DIRCACHE
2808     if (tempbuf_size == 0)
2809     {
2810         /* Try to steal the dircache buffer. */
2811         tempbuf = dircache_steal_buffer(&tempbuf_size);
2812         tempbuf_size &= ~0x03;
2813
2814         if (tempbuf_size > 0)
2815         {
2816             dircache_buffer_stolen = true;
2817         }
2818     }
2819 #endif
2820
2821 #ifdef HAVE_TC_RAMCACHE
2822     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2823     {
2824         tempbuf = (char *)(hdr + 1);
2825         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2826         tempbuf_size &= ~0x03;
2827     }
2828 #endif
2829
2830     /* And finally fail if there are no buffers available. */
2831     if (tempbuf_size == 0)
2832     {
2833         logf("delaying commit until next boot");
2834         tc_stat.commit_delayed = true;
2835         close(tmpfd);
2836         read_lock--;
2837         return false;
2838     }
2839
2840     logf("commit %ld entries...", tch.entry_count);
2841
2842     /* Mark DB dirty so it will stay disabled if commit fails. */
2843     current_tcmh.dirty = true;
2844     update_master_header();
2845
2846     /* Now create the index files. */
2847     tc_stat.commit_step = 0;
2848     tch.datasize = 0;
2849     tc_stat.commit_delayed = false;
2850
2851     for (i = 0; i < TAG_COUNT; i++)
2852     {
2853         int ret;
2854
2855         if (TAGCACHE_IS_NUMERIC(i))
2856             continue;
2857
2858         tc_stat.commit_step++;
2859         ret = build_index(i, &tch, tmpfd);
2860         if (ret <= 0)
2861         {
2862             close(tmpfd);
2863             logf("tagcache failed init");
2864             if (ret == 0)
2865                 tc_stat.commit_delayed = true;
2866
2867             tc_stat.commit_step = 0;
2868             read_lock--;
2869             return false;
2870         }
2871     }
2872
2873     if (!build_numeric_indices(&tch, tmpfd))
2874     {
2875         logf("Failure to commit numeric indices");
2876         close(tmpfd);
2877         tc_stat.commit_step = 0;
2878         read_lock--;
2879         return false;
2880     }
2881
2882     close(tmpfd);
2883     remove(TAGCACHE_FILE_TEMP);
2884
2885     tc_stat.commit_step = 0;
2886
2887     /* Update the master index headers. */
2888     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2889     {
2890         read_lock--;
2891         return false;
2892     }
2893
2894     tcmh.tch.entry_count += tch.entry_count;
2895     tcmh.tch.datasize = sizeof(struct master_header)
2896         + sizeof(struct index_entry) * tcmh.tch.entry_count
2897         + tch.datasize;
2898     tcmh.dirty = false;
2899     tcmh.commitid++;
2900
2901     lseek(masterfd, 0, SEEK_SET);
2902     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2903     close(masterfd);
2904
2905     logf("tagcache committed");
2906     tc_stat.ready = check_all_headers();
2907     tc_stat.readyvalid = true;
2908
2909     if (local_allocation)
2910     {
2911         tempbuf = NULL;
2912         tempbuf_size = 0;
2913     }
2914
2915 #ifdef HAVE_DIRCACHE
2916     /* Rebuild the dircache, if we stole the buffer. */
2917     if (dircache_buffer_stolen)
2918         dircache_build(0);
2919 #endif
2920
2921 #ifdef HAVE_TC_RAMCACHE
2922     /* Reload tagcache. */
2923     if (tc_stat.ramcache_allocated > 0)
2924         tagcache_start_scan();
2925 #endif
2926
2927     read_lock--;
2928
2929     return true;
2930 }
2931
2932 static void allocate_tempbuf(void)
2933 {
2934     /* Yeah, malloc would be really nice now :) */
2935 #ifdef __PCTOOL__
2936     tempbuf_size = 32*1024*1024;
2937     tempbuf = malloc(tempbuf_size);
2938 #else
2939     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2940     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2941     audiobuf += tempbuf_size;
2942 #endif
2943 }
2944
2945 static void free_tempbuf(void)
2946 {
2947     if (tempbuf_size == 0)
2948         return ;
2949
2950 #ifdef __PCTOOL__
2951     free(tempbuf);
2952 #else
2953     audiobuf -= tempbuf_size;
2954 #endif
2955     tempbuf = NULL;
2956     tempbuf_size = 0;
2957 }
2958
2959 #ifndef __PCTOOL__
2960
2961 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2962 {
2963     struct index_entry idx;
2964
2965     if (!tc_stat.ready)
2966         return false;
2967
2968     if (!TAGCACHE_IS_NUMERIC(tag))
2969         return false;
2970
2971     if (!get_index(masterfd, idx_id, &idx, false))
2972         return false;
2973
2974     idx.tag_seek[tag] = data;
2975     idx.flag |= FLAG_DIRTYNUM;
2976
2977     return write_index(masterfd, idx_id, &idx);
2978 }
2979
2980 #if 0
2981 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
2982                                    int tag, long data)
2983 {
2984     struct master_header myhdr;
2985
2986     if (tcs->masterfd < 0)
2987     {
2988         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
2989             return false;
2990     }
2991
2992     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
2993 }
2994 #endif
2995
2996 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
2997
2998 static bool command_queue_is_full(void)
2999 {
3000     int next;
3001
3002     next = command_queue_widx + 1;
3003     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3004         next = 0;
3005
3006     return (next == command_queue_ridx);
3007 }
3008
3009 static bool command_queue_sync_callback(void)
3010 {
3011
3012     struct master_header myhdr;
3013     int masterfd;
3014
3015     mutex_lock(&command_queue_mutex);
3016
3017     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3018         return false;
3019
3020     while (command_queue_ridx != command_queue_widx)
3021     {
3022         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3023
3024         switch (ce->command)
3025         {
3026             case CMD_UPDATE_MASTER_HEADER:
3027             {
3028                 close(masterfd);
3029                 update_master_header();
3030
3031                 /* Re-open the masterfd. */
3032                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3033                     return true;
3034
3035                 break;
3036             }
3037             case CMD_UPDATE_NUMERIC:
3038             {
3039                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3040                 break;
3041             }
3042         }
3043
3044         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3045             command_queue_ridx = 0;
3046     }
3047
3048     close(masterfd);
3049
3050     tc_stat.queue_length = 0;
3051     mutex_unlock(&command_queue_mutex);
3052     return true;
3053 }
3054
3055 static void run_command_queue(bool force)
3056 {
3057     if (COMMAND_QUEUE_IS_EMPTY)
3058         return;
3059
3060     if (force || command_queue_is_full())
3061         command_queue_sync_callback();
3062     else
3063         register_storage_idle_func(command_queue_sync_callback);
3064 }
3065
3066 static void queue_command(int cmd, long idx_id, int tag, long data)
3067 {
3068     while (1)
3069     {
3070         int next;
3071
3072         mutex_lock(&command_queue_mutex);
3073         next = command_queue_widx + 1;
3074         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3075             next = 0;
3076
3077         /* Make sure queue is not full. */
3078         if (next != command_queue_ridx)
3079         {
3080             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3081
3082             ce->command = cmd;
3083             ce->idx_id = idx_id;
3084             ce->tag = tag;
3085             ce->data = data;
3086
3087             command_queue_widx = next;
3088
3089             tc_stat.queue_length++;
3090
3091             mutex_unlock(&command_queue_mutex);
3092             break;
3093         }
3094
3095         /* Queue is full, try again later... */
3096         mutex_unlock(&command_queue_mutex);
3097         sleep(1);
3098     }
3099 }
3100
3101 long tagcache_increase_serial(void)
3102 {
3103     long old;
3104
3105     if (!tc_stat.ready)
3106         return -2;
3107
3108     while (read_lock)
3109         sleep(1);
3110
3111     old = current_tcmh.serial++;
3112     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3113
3114     return old;
3115 }
3116
3117 void tagcache_update_numeric(int idx_id, int tag, long data)
3118 {
3119     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3120 }
3121 #endif /* !__PCTOOL__ */
3122
3123 long tagcache_get_serial(void)
3124 {
3125     return current_tcmh.serial;
3126 }
3127
3128 long tagcache_get_commitid(void)
3129 {
3130     return current_tcmh.commitid;
3131 }
3132
3133 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3134 {
3135     char buf[512];
3136     int i;
3137
3138     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3139     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3140     {
3141         if (*datastr == '\0')
3142             break;
3143
3144         if (*datastr == '"' || *datastr == '\\')
3145             buf[i++] = '\\';
3146
3147         buf[i] = *(datastr++);
3148     }
3149
3150     strcpy(&buf[i], "\" ");
3151
3152     write(fd, buf, i + 2);
3153
3154     return true;
3155 }
3156
3157 #ifndef __PCTOOL__
3158
3159 static bool read_tag(char *dest, long size,
3160                      const char *src, const char *tagstr)
3161 {
3162     int pos;
3163     char current_tag[32];
3164
3165     while (*src != '\0')
3166     {
3167         /* Skip all whitespace */
3168         while (*src == ' ')
3169             src++;
3170
3171         if (*src == '\0')
3172             break;
3173
3174         pos = 0;
3175         /* Read in tag name */
3176         while (*src != '=' && *src != ' ')
3177         {
3178             current_tag[pos] = *src;
3179             src++;
3180             pos++;
3181
3182             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3183                 return false;
3184         }
3185         current_tag[pos] = '\0';
3186
3187         /* Read in tag data */
3188
3189         /* Find the start. */
3190         while (*src != '"' && *src != '\0')
3191             src++;
3192
3193         if (*src == '\0' || *(++src) == '\0')
3194             return false;
3195
3196         /* Read the data. */
3197         for (pos = 0; pos < size; pos++)
3198         {
3199             if (*src == '\0')
3200                 break;
3201
3202             if (*src == '\\')
3203             {
3204                 dest[pos] = *(src+1);
3205                 src += 2;
3206                 continue;
3207             }
3208
3209             dest[pos] = *src;
3210
3211             if (*src == '"')
3212             {
3213                 src++;
3214                 break;
3215             }
3216
3217             if (*src == '\0')
3218                 break;
3219
3220             src++;
3221         }
3222         dest[pos] = '\0';
3223
3224         if (!strcasecmp(tagstr, current_tag))
3225             return true;
3226     }
3227
3228     return false;
3229 }
3230
3231 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3232 {
3233     struct index_entry idx;
3234     char tag_data[TAG_MAXLEN+32];
3235     int idx_id;
3236     long masterfd = (long)parameters;
3237     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3238         tag_commitid };
3239     int i;
3240     (void)line_n;
3241
3242     if (*buf == '#')
3243         return 0;
3244
3245     logf("%d/%s", line_n, buf);
3246     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3247     {
3248         logf("filename missing");
3249         logf("-> %s", buf);
3250         return 0;
3251     }
3252
3253     idx_id = find_index(tag_data);
3254     if (idx_id < 0)
3255     {
3256         logf("entry not found");
3257         return 0;
3258     }
3259
3260     if (!get_index(masterfd, idx_id, &idx, false))
3261     {
3262         logf("failed to retrieve index entry");
3263         return 0;
3264     }
3265
3266     /* Stop if tag has already been modified. */
3267     if (idx.flag & FLAG_DIRTYNUM)
3268         return 0;
3269
3270     logf("import: %s", tag_data);
3271
3272     idx.flag |= FLAG_DIRTYNUM;
3273     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3274     {
3275         int data;
3276
3277         if (!read_tag(tag_data, sizeof tag_data, buf,
3278                       tagcache_tag_to_str(import_tags[i])))
3279         {
3280             continue;
3281         }
3282
3283         data = atoi(tag_data);
3284         if (data < 0)
3285             continue;
3286
3287         idx.tag_seek[import_tags[i]] = data;
3288
3289         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3290             current_tcmh.serial = data + 1;
3291         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3292             current_tcmh.commitid = data + 1;
3293     }
3294
3295     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3296 }
3297
3298 bool tagcache_import_changelog(void)
3299 {
3300     struct master_header myhdr;
3301     struct tagcache_header tch;
3302     int clfd;
3303     long masterfd;
3304     char buf[2048];
3305
3306     if (!tc_stat.ready)
3307         return false;
3308
3309     while (read_lock)
3310         sleep(1);
3311
3312     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3313     if (clfd < 0)
3314     {
3315         logf("failure to open changelog");
3316         return false;
3317     }
3318
3319     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3320     {
3321         close(clfd);
3322         return false;
3323     }
3324
3325     write_lock++;
3326
3327     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3328
3329     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3330                   parse_changelog_line);
3331
3332     close(clfd);
3333     close(masterfd);
3334
3335     if (filenametag_fd >= 0)
3336         close(filenametag_fd);
3337
3338     write_lock--;
3339
3340     update_master_header();
3341
3342     return true;
3343 }
3344
3345 #endif /* !__PCTOOL__ */
3346
3347 bool tagcache_create_changelog(struct tagcache_search *tcs)
3348 {
3349     struct master_header myhdr;
3350     struct index_entry idx;
3351     char buf[TAG_MAXLEN+32];
3352     char temp[32];
3353     int clfd;
3354     int i, j;
3355
3356     if (!tc_stat.ready)
3357         return false;
3358
3359     if (!tagcache_search(tcs, tag_filename))
3360         return false;
3361
3362     /* Initialize the changelog */
3363     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3364     if (clfd < 0)
3365     {
3366         logf("failure to open changelog");
3367         return false;
3368     }
3369
3370     if (tcs->masterfd < 0)
3371     {
3372         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3373             return false;
3374     }
3375     else
3376     {
3377         lseek(tcs->masterfd, 0, SEEK_SET);
3378         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3379     }
3380
3381     write(clfd, "## Changelog version 1\n", 23);
3382
3383     for (i = 0; i < myhdr.tch.entry_count; i++)
3384     {
3385         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3386             != sizeof(struct index_entry))
3387         {
3388             logf("read error #9");
3389             tagcache_search_finish(tcs);
3390             close(clfd);
3391             return false;
3392         }
3393
3394         /* Skip until the entry found has been modified. */
3395         if (! (idx.flag & FLAG_DIRTYNUM) )
3396             continue;
3397
3398         /* Skip deleted entries too. */
3399         if (idx.flag & FLAG_DELETED)
3400             continue;
3401
3402         /* Now retrieve all tags. */
3403         for (j = 0; j < TAG_COUNT; j++)
3404         {
3405             if (TAGCACHE_IS_NUMERIC(j))
3406             {
3407                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3408                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3409                 continue;
3410             }
3411
3412             tcs->type = j;
3413             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3414             write_tag(clfd, tagcache_tag_to_str(j), buf);
3415         }
3416
3417         write(clfd, "\n", 1);
3418         do_timed_yield();
3419     }
3420
3421     close(clfd);
3422
3423     tagcache_search_finish(tcs);
3424
3425     return true;
3426 }
3427
3428 static bool delete_entry(long idx_id)
3429 {
3430     int fd = -1;
3431     int masterfd = -1;
3432     int tag, i;
3433     struct index_entry idx, myidx;
3434     struct master_header myhdr;
3435     char buf[TAG_MAXLEN+32];
3436     int in_use[TAG_COUNT];
3437
3438     logf("delete_entry(): %ld", idx_id);
3439
3440 #ifdef HAVE_TC_RAMCACHE
3441     /* At first mark the entry removed from ram cache. */
3442     if (tc_stat.ramcache)
3443         hdr->indices[idx_id].flag |= FLAG_DELETED;
3444 #endif
3445
3446     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3447         return false;
3448
3449     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3450     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3451         != sizeof(struct index_entry))
3452     {
3453         logf("delete_entry(): read error");
3454         goto cleanup;
3455     }
3456
3457     if (myidx.flag & FLAG_DELETED)
3458     {
3459         logf("delete_entry(): already deleted!");
3460         goto cleanup;
3461     }
3462
3463     myidx.flag |= FLAG_DELETED;
3464     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3465     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3466         != sizeof(struct index_entry))
3467     {
3468         logf("delete_entry(): write_error #1");
3469         goto cleanup;
3470     }
3471
3472     /* Now check which tags are no longer in use (if any) */
3473     for (tag = 0; tag < TAG_COUNT; tag++)
3474         in_use[tag] = 0;
3475
3476     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3477     for (i = 0; i < myhdr.tch.entry_count; i++)
3478     {
3479         struct index_entry *idxp;
3480
3481 #ifdef HAVE_TC_RAMCACHE
3482         /* Use RAM DB if available for greater speed */
3483         if (tc_stat.ramcache)
3484             idxp = &hdr->indices[i];
3485         else
3486 #endif
3487         {
3488             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3489                 != sizeof(struct index_entry))
3490             {
3491                 logf("delete_entry(): read error #2");
3492                 goto cleanup;
3493             }
3494             idxp = &idx;
3495         }
3496
3497         if (idxp->flag & FLAG_DELETED)
3498             continue;
3499
3500         for (tag = 0; tag < TAG_COUNT; tag++)
3501         {
3502             if (TAGCACHE_IS_NUMERIC(tag))
3503                 continue;
3504
3505             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3506                 in_use[tag]++;
3507         }
3508     }
3509
3510     /* Now delete all tags no longer in use. */
3511     for (tag = 0; tag < TAG_COUNT; tag++)
3512     {
3513         struct tagcache_header tch;
3514         int oldseek = myidx.tag_seek[tag];
3515
3516         if (TAGCACHE_IS_NUMERIC(tag))
3517             continue;
3518
3519         /**
3520          * Replace tag seek with a hash value of the field string data.
3521          * That way runtime statistics of moved or altered files can be
3522          * resurrected.
3523          */
3524 #ifdef HAVE_TC_RAMCACHE
3525         if (tc_stat.ramcache && tag != tag_filename)
3526         {
3527             struct tagfile_entry *tfe;
3528             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3529
3530             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3531             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3532             myidx.tag_seek[tag] = *seek;
3533         }
3534         else
3535 #endif
3536         {
3537             struct tagfile_entry tfe;
3538
3539             /* Open the index file, which contains the tag names. */
3540             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3541                 goto cleanup;
3542
3543             /* Skip the header block */
3544             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3545             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3546                 != sizeof(struct tagfile_entry))
3547             {
3548                 logf("delete_entry(): read error #3");
3549                 goto cleanup;
3550             }
3551
3552             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3553             {
3554                 logf("delete_entry(): read error #4");
3555                 goto cleanup;
3556             }
3557
3558             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3559         }
3560
3561         if (in_use[tag])
3562         {
3563             logf("in use: %d/%d", tag, in_use[tag]);
3564             if (fd >= 0)
3565             {
3566                 close(fd);
3567                 fd = -1;
3568             }
3569             continue;
3570         }
3571
3572 #ifdef HAVE_TC_RAMCACHE
3573         /* Delete from ram. */
3574         if (tc_stat.ramcache && tag != tag_filename)
3575         {
3576             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3577             tagentry->tag_data[0] = '\0';
3578         }
3579 #endif
3580
3581         /* Open the index file, which contains the tag names. */
3582         if (fd < 0)
3583         {
3584             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3585                 goto cleanup;
3586         }
3587
3588         /* Skip the header block */
3589         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3590
3591         /* Debug, print 10 first characters of the tag
3592         read(fd, buf, 10);
3593         buf[10]='\0';
3594         logf("TAG:%s", buf);
3595         lseek(fd, -10, SEEK_CUR);
3596         */
3597
3598         /* Write first data byte in tag as \0 */
3599         write(fd, "", 1);
3600
3601         /* Now tag data has been removed */
3602         close(fd);
3603         fd = -1;
3604     }
3605
3606     /* Write index entry back into master index. */
3607     lseek(masterfd, sizeof(struct master_header) +
3608           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3609     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3610         != sizeof(struct index_entry))
3611     {
3612         logf("delete_entry(): write_error #2");
3613         goto cleanup;
3614     }
3615
3616     close(masterfd);
3617
3618     return true;
3619
3620     cleanup:
3621     if (fd >= 0)
3622         close(fd);
3623     if (masterfd >= 0)
3624         close(masterfd);
3625
3626     return false;
3627 }
3628
3629 #ifndef __PCTOOL__
3630 /**
3631  * Returns true if there is an event waiting in the queue
3632  * that requires the current operation to be aborted.
3633  */
3634 static bool check_event_queue(void)
3635 {
3636     struct queue_event ev;
3637
3638     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3639     switch (ev.id)
3640     {
3641         case Q_STOP_SCAN:
3642         case SYS_POWEROFF:
3643         case SYS_USB_CONNECTED:
3644             /* Put the event back into the queue. */
3645             queue_post(&tagcache_queue, ev.id, ev.data);
3646             return true;
3647     }
3648
3649     return false;
3650 }
3651 #endif
3652
3653 #ifdef HAVE_TC_RAMCACHE
3654 static bool allocate_tagcache(void)
3655 {
3656     struct master_header tcmh;
3657     int fd;
3658
3659     /* Load the header. */
3660     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3661     {
3662         hdr = NULL;
3663         return false;
3664     }
3665
3666     close(fd);
3667
3668     /**
3669      * Now calculate the required cache size plus
3670      * some extra space for alignment fixes.
3671      */
3672     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3673         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3674     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3675     memset(hdr, 0, sizeof(struct ramcache_header));
3676     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3677     hdr->indices = (struct index_entry *)(hdr + 1);
3678     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3679
3680     return true;
3681 }
3682
3683 # ifdef HAVE_EEPROM_SETTINGS
3684 static bool tagcache_dumpload(void)
3685 {
3686     struct statefile_header shdr;
3687     int fd, rc;
3688     long offpos;
3689     int i;
3690
3691     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3692     if (fd < 0)
3693     {
3694         logf("no tagcache statedump");
3695         return false;
3696     }
3697
3698     /* Check the statefile memory placement */
3699     hdr = buffer_alloc(0);
3700     rc = read(fd, &shdr, sizeof(struct statefile_header));
3701     if (rc != sizeof(struct statefile_header)
3702         /* || (long)hdr != (long)shdr.hdr */)
3703     {
3704         logf("incorrect statefile");
3705         hdr = NULL;
3706         close(fd);
3707         return false;
3708     }
3709
3710     offpos = (long)hdr - (long)shdr.hdr;
3711
3712     /* Lets allocate real memory and load it */
3713     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3714     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3715     close(fd);
3716
3717     if (rc != shdr.tc_stat.ramcache_allocated)
3718     {
3719         logf("read failure!");
3720         hdr = NULL;
3721         return false;
3722     }
3723
3724     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3725
3726     /* Now fix the pointers */
3727     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3728     for (i = 0; i < TAG_COUNT; i++)
3729         hdr->tags[i] += offpos;
3730
3731     return true;
3732 }
3733
3734 static bool tagcache_dumpsave(void)
3735 {
3736     struct statefile_header shdr;
3737     int fd;
3738
3739     if (!tc_stat.ramcache)
3740         return false;
3741
3742     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3743     if (fd < 0)
3744     {
3745         logf("failed to create a statedump");
3746         return false;
3747     }
3748
3749     /* Create the header */
3750     shdr.hdr = hdr;
3751     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3752     write(fd, &shdr, sizeof(struct statefile_header));
3753
3754     /* And dump the data too */
3755     write(fd, hdr, tc_stat.ramcache_allocated);
3756     close(fd);
3757
3758     return true;
3759 }
3760 # endif
3761
3762 static bool load_tagcache(void)
3763 {
3764     struct tagcache_header *tch;
3765     long bytesleft = tc_stat.ramcache_allocated;
3766     struct index_entry *idx;
3767     int rc, fd;
3768     char *p;
3769     int i, tag;
3770
3771 # ifdef HAVE_DIRCACHE
3772     while (dircache_is_initializing())
3773         sleep(1);
3774
3775     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3776 # endif
3777
3778     logf("loading tagcache to ram...");
3779
3780     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3781     if (fd < 0)
3782     {
3783         logf("tagcache open failed");
3784         return false;
3785     }
3786
3787     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3788         != sizeof(struct master_header)
3789         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3790     {
3791         logf("incorrect header");
3792         return false;
3793     }
3794
3795     idx = hdr->indices;
3796
3797     /* Load the master index table. */
3798     for (i = 0; i < hdr->h.tch.entry_count; i++)
3799     {
3800         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3801         if (rc != sizeof(struct index_entry))
3802         {
3803             logf("read error #10");
3804             close(fd);
3805             return false;
3806         }
3807
3808         bytesleft -= sizeof(struct index_entry);
3809         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3810         {
3811             logf("too big tagcache.");
3812             close(fd);
3813             return false;
3814         }
3815
3816         idx++;
3817     }
3818
3819     close(fd);
3820
3821     /* Load the tags. */
3822     p = (char *)idx;
3823     for (tag = 0; tag < TAG_COUNT; tag++)
3824     {
3825         struct tagfile_entry *fe;
3826         char buf[TAG_MAXLEN+32];
3827
3828         if (TAGCACHE_IS_NUMERIC(tag))
3829             continue ;
3830
3831         //p = ((void *)p+1);
3832         p = (char *)((long)p & ~0x03) + 0x04;
3833         hdr->tags[tag] = p;
3834
3835         /* Check the header. */
3836         tch = (struct tagcache_header *)p;
3837         p += sizeof(struct tagcache_header);
3838
3839         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3840             return false;
3841
3842         for (hdr->entry_count[tag] = 0;
3843              hdr->entry_count[tag] < tch->entry_count;
3844              hdr->entry_count[tag]++)
3845         {
3846             long pos;
3847
3848             if (do_timed_yield())
3849             {
3850                 /* Abort if we got a critical event in queue */
3851                 if (check_event_queue())
3852                     return false;
3853             }
3854
3855             fe = (struct tagfile_entry *)p;
3856             pos = lseek(fd, 0, SEEK_CUR);
3857             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3858             if (rc != sizeof(struct tagfile_entry))
3859             {
3860                 /* End of lookup table. */
3861                 logf("read error #11");
3862                 close(fd);
3863                 return false;
3864             }
3865
3866             /* We have a special handling for the filename tags. */
3867             if (tag == tag_filename)
3868             {
3869 # ifdef HAVE_DIRCACHE
3870                 const struct dirent *dc;
3871 # endif
3872
3873                 // FIXME: This is wrong!
3874                 // idx = &hdr->indices[hdr->entry_count[i]];
3875                 idx = &hdr->indices[fe->idx_id];
3876
3877                 if (fe->tag_length >= (long)sizeof(buf)-1)
3878                 {
3879                     read(fd, buf, 10);
3880                     buf[10] = '\0';
3881                     logf("TAG:%s", buf);
3882                     logf("too long filename");
3883                     close(fd);
3884                     return false;
3885                 }
3886
3887                 rc = read(fd, buf, fe->tag_length);
3888                 if (rc != fe->tag_length)
3889                 {
3890                     logf("read error #12");
3891                     close(fd);
3892                     return false;
3893                 }
3894
3895                 /* Check if the entry has already been removed */
3896                 if (idx->flag & FLAG_DELETED)
3897                     continue;
3898
3899                 /* This flag must not be used yet. */
3900                 if (idx->flag & FLAG_DIRCACHE)
3901                 {
3902                     logf("internal error!");
3903                     close(fd);
3904                     return false;
3905                 }
3906
3907                 if (idx->tag_seek[tag] != pos)
3908                 {
3909                     logf("corrupt data structures!");
3910                     close(fd);
3911                     return false;
3912                 }
3913
3914 # ifdef HAVE_DIRCACHE
3915                 if (dircache_is_enabled())
3916                 {
3917                     dc = dircache_get_entry_ptr(buf);
3918                     if (dc == NULL)
3919                     {
3920                         logf("Entry no longer valid.");
3921                         logf("-> %s", buf);
3922                         delete_entry(fe->idx_id);
3923                         continue ;
3924                     }
3925
3926                     idx->flag |= FLAG_DIRCACHE;
3927                     FLAG_SET_ATTR(idx->flag, fe->idx_id);
3928                     idx->tag_seek[tag_filename] = (long)dc;
3929                 }
3930                 else
3931 # endif
3932                 {
3933                     /* This will be very slow unless dircache is enabled
3934                        or target is flash based, but do it anyway for
3935                        consistency. */
3936                     /* Check if entry has been removed. */
3937                     if (global_settings.tagcache_autoupdate)
3938                     {
3939                         if (!file_exists(buf))
3940                         {
3941                             logf("Entry no longer valid.");
3942                             logf("-> %s", buf);
3943                             delete_entry(fe->idx_id);
3944                             continue;
3945                         }
3946                     }
3947                 }
3948
3949                 continue ;
3950             }
3951
3952             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3953             if (bytesleft < 0)
3954             {
3955                 logf("too big tagcache #2");
3956                 logf("tl: %d", fe->tag_length);
3957                 logf("bl: %ld", bytesleft);
3958                 close(fd);
3959                 return false;
3960             }
3961
3962             p = fe->tag_data;
3963             rc = read(fd, fe->tag_data, fe->tag_length);
3964             p += rc;
3965
3966             if (rc != fe->tag_length)
3967             {
3968                 logf("read error #13");
3969                 logf("rc=0x%04x", rc); // 0x431
3970                 logf("len=0x%04x", fe->tag_length); // 0x4000
3971                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3972                 logf("tag=0x%02x", tag); // 0x00
3973                 close(fd);
3974                 return false;
3975             }
3976         }
3977         close(fd);
3978     }
3979
3980     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
3981     logf("tagcache loaded into ram!");
3982
3983     return true;
3984 }
3985 #endif /* HAVE_TC_RAMCACHE */
3986
3987 static bool check_deleted_files(void)
3988 {
3989     int fd;
3990     char buf[TAG_MAXLEN+32];
3991     struct tagfile_entry tfe;
3992
3993     logf("reverse scan...");
3994     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
3995     fd = open(buf, O_RDONLY);
3996
3997     if (fd < 0)
3998     {
3999         logf("%s open fail", buf);
4000         return false;
4001     }
4002
4003     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4004     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4005            == sizeof(struct tagfile_entry)
4006 #ifndef __PCTOOL__
4007            && !check_event_queue()
4008 #endif
4009            )
4010     {
4011         if (tfe.tag_length >= (long)sizeof(buf)-1)
4012         {
4013             logf("too long tag");
4014             close(fd);
4015             return false;
4016         }
4017
4018         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4019         {
4020             logf("read error #14");
4021             close(fd);
4022             return false;
4023         }
4024
4025         /* Check if the file has already deleted from the db. */
4026         if (*buf == '\0')
4027             continue;
4028
4029         /* Now check if the file exists. */
4030         if (!file_exists(buf))
4031         {
4032             logf("Entry no longer valid.");
4033             logf("-> %s / %d", buf, tfe.tag_length);
4034             delete_entry(tfe.idx_id);
4035         }
4036     }
4037
4038     close(fd);
4039
4040     logf("done");
4041
4042     return true;
4043 }
4044
4045 static bool check_dir(const char *dirname, int add_files)
4046 {
4047     DIR *dir;
4048     int len;
4049     int success = false;
4050     int ignore, unignore;
4051     char newpath[MAX_PATH];
4052
4053     dir = opendir(dirname);
4054     if (!dir)
4055     {
4056         logf("tagcache: opendir() failed");
4057         return false;
4058     }
4059
4060     /* check for a database.ignore file */
4061     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4062     ignore = file_exists(newpath);
4063     /* check for a database.unignore file */
4064     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4065     unignore = file_exists(newpath);
4066
4067     /* don't do anything if both ignore and unignore are there */
4068     if (ignore != unignore)
4069         add_files = unignore;
4070
4071     /* Recursively scan the dir. */
4072 #ifdef __PCTOOL__
4073     while (1)
4074 #else
4075     while (!check_event_queue())
4076 #endif
4077     {
4078         struct dirent *entry;
4079
4080         entry = readdir(dir);
4081
4082         if (entry == NULL)
4083         {
4084             success = true;
4085             break ;
4086         }
4087
4088         if (!strcmp((char *)entry->d_name, ".") ||
4089             !strcmp((char *)entry->d_name, ".."))
4090             continue;
4091
4092         yield();
4093
4094         len = strlen(curpath);
4095         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4096                  entry->d_name);
4097
4098         processed_dir_count++;
4099         if (entry->attribute & ATTR_DIRECTORY)
4100             check_dir(curpath, add_files);
4101         else if (add_files)
4102         {
4103             tc_stat.curentry = curpath;
4104
4105             /* Add a new entry to the temporary db file. */
4106             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4107 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4108                          , dir->internal_entry
4109 #endif
4110                          );
4111
4112             /* Wait until current path for debug screen is read and unset. */
4113             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4114                 yield();
4115
4116             tc_stat.curentry = NULL;
4117         }
4118
4119         curpath[len] = '\0';
4120     }
4121
4122     closedir(dir);
4123
4124     return success;
4125 }
4126
4127 void tagcache_screensync_event(void)
4128 {
4129     tc_stat.curentry = NULL;
4130 }
4131
4132 void tagcache_screensync_enable(bool state)
4133 {
4134     tc_stat.syncscreen = state;
4135 }
4136
4137 void tagcache_build(const char *path)
4138 {
4139     struct tagcache_header header;
4140     bool ret;
4141
4142     curpath[0] = '\0';
4143     data_size = 0;
4144     total_entry_count = 0;
4145     processed_dir_count = 0;
4146
4147 #ifdef HAVE_DIRCACHE
4148     while (dircache_is_initializing())
4149         sleep(1);
4150 #endif
4151
4152     logf("updating tagcache");
4153
4154     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4155     if (cachefd >= 0)
4156     {
4157         logf("skipping, cache already waiting for commit");
4158         close(cachefd);
4159         return ;
4160     }
4161
4162     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4163     if (cachefd < 0)
4164     {
4165         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4166         return ;
4167     }
4168
4169     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4170
4171     cpu_boost(true);
4172
4173     logf("Scanning files...");
4174     /* Scan for new files. */
4175     memset(&header, 0, sizeof(struct tagcache_header));
4176     write(cachefd, &header, sizeof(struct tagcache_header));
4177
4178     if (strcmp("/", path) != 0)
4179         strcpy(curpath, path);
4180     ret = check_dir(path, true);
4181
4182     /* Write the header. */
4183     header.magic = TAGCACHE_MAGIC;
4184     header.datasize = data_size;
4185     header.entry_count = total_entry_count;
4186     lseek(cachefd, 0, SEEK_SET);
4187     write(cachefd, &header, sizeof(struct tagcache_header));
4188     close(cachefd);
4189
4190     if (filenametag_fd >= 0)
4191     {
4192         close(filenametag_fd);
4193         filenametag_fd = -1;
4194     }
4195
4196     if (!ret)
4197     {
4198         logf("Aborted.");
4199         cpu_boost(false);
4200         return ;
4201     }
4202
4203     /* Commit changes to the database. */
4204 #ifdef __PCTOOL__
4205     allocate_tempbuf();
4206 #endif
4207     if (commit())
4208     {
4209         remove(TAGCACHE_FILE_TEMP);
4210         logf("tagcache built!");
4211     }
4212 #ifdef __PCTOOL__
4213     free_tempbuf();
4214 #endif
4215
4216 #ifdef HAVE_TC_RAMCACHE
4217     if (hdr)
4218     {
4219         /* Import runtime statistics if we just initialized the db. */
4220         if (hdr->h.serial == 0)
4221             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4222     }
4223 #endif
4224
4225     cpu_boost(false);
4226 }
4227
4228 #ifdef HAVE_TC_RAMCACHE
4229 static void load_ramcache(void)
4230 {
4231     if (!hdr)
4232         return ;
4233
4234     cpu_boost(true);
4235
4236     /* At first we should load the cache (if exists). */
4237     tc_stat.ramcache = load_tagcache();
4238
4239     if (!tc_stat.ramcache)
4240     {
4241         /* If loading failed, it must indicate some problem with the db
4242          * so disable it entirely to prevent further issues. */
4243         tc_stat.ready = false;
4244         hdr = NULL;
4245     }
4246
4247     cpu_boost(false);
4248 }
4249
4250 void tagcache_unload_ramcache(void)
4251 {
4252     tc_stat.ramcache = false;
4253     /* Just to make sure there is no statefile present. */
4254     // remove(TAGCACHE_STATEFILE);
4255 }
4256 #endif
4257
4258 #ifndef __PCTOOL__
4259 static void tagcache_thread(void)
4260 {
4261     struct queue_event ev;
4262     bool check_done = false;
4263
4264     /* If the previous cache build/update was interrupted, commit
4265      * the changes first in foreground. */
4266     cpu_boost(true);
4267     allocate_tempbuf();
4268     commit();
4269     free_tempbuf();
4270
4271 #ifdef HAVE_TC_RAMCACHE
4272 # ifdef HAVE_EEPROM_SETTINGS
4273     if (firmware_settings.initialized && firmware_settings.disk_clean)
4274         check_done = tagcache_dumpload();
4275
4276     remove(TAGCACHE_STATEFILE);
4277 # endif
4278
4279     /* Allocate space for the tagcache if found on disk. */
4280     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4281         allocate_tagcache();
4282 #endif
4283
4284     cpu_boost(false);
4285     tc_stat.initialized = true;
4286
4287     /* Don't delay bootup with the header check but do it on background. */
4288     sleep(HZ);
4289     tc_stat.ready = check_all_headers();
4290     tc_stat.readyvalid = true;
4291
4292     while (1)
4293     {
4294         run_command_queue(false);
4295
4296         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4297
4298         switch (ev.id)
4299         {
4300             case Q_IMPORT_CHANGELOG:
4301                 tagcache_import_changelog();
4302                 break;
4303
4304             case Q_REBUILD:
4305                 remove_files();
4306                 remove(TAGCACHE_FILE_TEMP);
4307                 tagcache_build("/");
4308                 break;
4309
4310             case Q_UPDATE:
4311                 tagcache_build("/");
4312 #ifdef HAVE_TC_RAMCACHE
4313                 load_ramcache();
4314 #endif
4315                 check_deleted_files();
4316                 break ;
4317
4318             case Q_START_SCAN:
4319                 check_done = false;
4320             case SYS_TIMEOUT:
4321                 if (check_done || !tc_stat.ready)
4322                     break ;
4323
4324 #ifdef HAVE_TC_RAMCACHE
4325                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4326                 {
4327                     load_ramcache();
4328                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4329                         tagcache_build("/");
4330                 }
4331                 else
4332 #endif
4333                 if (global_settings.tagcache_autoupdate)
4334                 {
4335                     tagcache_build("/");
4336
4337                     /* This will be very slow unless dircache is enabled
4338                        or target is flash based, but do it anyway for
4339                        consistency. */
4340                     check_deleted_files();
4341                 }
4342
4343                 logf("tagcache check done");
4344
4345                 check_done = true;
4346                 break ;
4347
4348             case Q_STOP_SCAN:
4349                 break ;
4350
4351             case SYS_POWEROFF:
4352                 break ;
4353
4354 #ifndef SIMULATOR
4355             case SYS_USB_CONNECTED:
4356                 logf("USB: TagCache");
4357                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4358                 usb_wait_for_disconnect(&tagcache_queue);
4359                 break ;
4360 #endif
4361         }
4362     }
4363 }
4364
4365 bool tagcache_prepare_shutdown(void)
4366 {
4367     if (tagcache_get_commit_step() > 0)
4368         return false;
4369
4370     tagcache_stop_scan();
4371     while (read_lock || write_lock)
4372         sleep(1);
4373
4374     return true;
4375 }
4376
4377 void tagcache_shutdown(void)
4378 {
4379     /* Flush the command queue. */
4380     run_command_queue(true);
4381
4382 #ifdef HAVE_EEPROM_SETTINGS
4383     if (tc_stat.ramcache)
4384         tagcache_dumpsave();
4385 #endif
4386 }
4387
4388 static int get_progress(void)
4389 {
4390     int total_count = -1;
4391
4392 #ifdef HAVE_DIRCACHE
4393     if (dircache_is_enabled())
4394     {
4395         total_count = dircache_get_entry_count();
4396     }
4397     else
4398 #endif
4399 #ifdef HAVE_TC_RAMCACHE
4400     {
4401         if (hdr && tc_stat.ramcache)
4402             total_count = hdr->h.tch.entry_count;
4403     }
4404 #endif
4405
4406     if (total_count < 0)
4407         return -1;
4408
4409     return processed_dir_count * 100 / total_count;
4410 }
4411
4412 struct tagcache_stat* tagcache_get_stat(void)
4413 {
4414     tc_stat.progress = get_progress();
4415     tc_stat.processed_entries = processed_dir_count;
4416
4417     return &tc_stat;
4418 }
4419
4420 void tagcache_start_scan(void)
4421 {
4422     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4423 }
4424
4425 bool tagcache_update(void)
4426 {
4427     if (!tc_stat.ready)
4428         return false;
4429
4430     queue_post(&tagcache_queue, Q_UPDATE, 0);
4431     return false;
4432 }
4433
4434 bool tagcache_rebuild()
4435 {
4436     queue_post(&tagcache_queue, Q_REBUILD, 0);
4437     return false;
4438 }
4439
4440 void tagcache_stop_scan(void)
4441 {
4442     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4443 }
4444
4445 #ifdef HAVE_TC_RAMCACHE
4446 bool tagcache_is_ramcache(void)
4447 {
4448     return tc_stat.ramcache;
4449 }
4450 #endif
4451
4452 #endif /* !__PCTOOL__ */
4453
4454
4455 void tagcache_init(void)
4456 {
4457     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4458     memset(&current_tcmh, 0, sizeof(struct master_header));
4459     filenametag_fd = -1;
4460     write_lock = read_lock = 0;
4461
4462 #ifndef __PCTOOL__
4463     mutex_init(&command_queue_mutex);
4464     queue_init(&tagcache_queue, true);
4465     create_thread(tagcache_thread, tagcache_stack,
4466                   sizeof(tagcache_stack), 0, tagcache_thread_name
4467                   IF_PRIO(, PRIORITY_BACKGROUND)
4468                           IF_COP(, CPU));
4469 #else
4470     tc_stat.initialized = true;
4471     allocate_tempbuf();
4472     commit();
4473     free_tempbuf();
4474     tc_stat.ready = check_all_headers();
4475 #endif
4476 }
4477
4478 #ifdef __PCTOOL__
4479 void tagcache_reverse_scan(void)
4480 {
4481     logf("Checking for deleted files");
4482     check_deleted_files();
4483 }
4484 #endif
4485
4486 bool tagcache_is_initialized(void)
4487 {
4488     return tc_stat.initialized;
4489 }
4490 bool tagcache_is_usable(void)
4491 {
4492     return tc_stat.initialized && tc_stat.ready;
4493 }
4494 int tagcache_get_commit_step(void)
4495 {
4496     return tc_stat.commit_step;
4497 }
4498 int tagcache_get_max_commit_step(void)
4499 {
4500     return (int)(SORTED_TAGS_COUNT)+1;
4501 }
4502