apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string-extra.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 /* Previous path when scanning directory tree recursively. */
  99 static char curpath[TAG_MAXLEN+32];
 100
 101 /* Used when removing duplicates. */
 102 static char *tempbuf;     /* Allocated when needed. */
 103 static long tempbufidx;   /* Current location in buffer. */
 104 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 105 static long tempbuf_left; /* Buffer space left. */
 106 static long tempbuf_pos;
 107
 108 #define SORTED_TAGS_COUNT 8
 109 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 110 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 111 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 112     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 113 /* Tags we want to get sorted (loaded to the tempbuf). */
 114 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 115     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 116     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 117
 118 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 119 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 120     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 121     (1LU << tag_albumartist) | (1LU << tag_grouping))
 122
 123 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 124 static const char *tags_str[] = { "artist", "album", "genre", "title",
 125     "filename", "composer", "comment", "albumartist", "grouping", "year",
 126     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 127     "playtime", "lastplayed", "commitid", "mtime" };
 128
 129 /* Status information of the tagcache. */
 130 static struct tagcache_stat tc_stat;
 131
 132 /* Queue commands. */
 133 enum tagcache_queue {
 134     Q_STOP_SCAN = 0,
 135     Q_START_SCAN,
 136     Q_IMPORT_CHANGELOG,
 137     Q_UPDATE,
 138     Q_REBUILD,
 139
 140     /* Internal tagcache command queue. */
 141     CMD_UPDATE_MASTER_HEADER,
 142     CMD_UPDATE_NUMERIC,
 143 };
 144
 145 struct tagcache_command_entry {
 146     int32_t command;
 147     int32_t idx_id;
 148     int32_t tag;
 149     int32_t data;
 150 };
 151
 152 #ifndef __PCTOOL__
 153 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 154 static volatile int command_queue_widx = 0;
 155 static volatile int command_queue_ridx = 0;
 156 static struct mutex command_queue_mutex;
 157 #endif
 158
 159 /* Tag database structures. */
 160
 161 /* Variable-length tag entry in tag files. */
 162 struct tagfile_entry {
 163     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 164     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 165     char tag_data[0];  /* Begin of the tag data */
 166 };
 167
 168 /* Fixed-size tag entry in master db index. */
 169 struct index_entry {
 170     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 171     int32_t flag;                /* Status flags */
 172 };
 173
 174 /* Header is the same in every file. */
 175 struct tagcache_header {
 176     int32_t magic;       /* Header version number */
 177     int32_t datasize;    /* Data size in bytes */
 178     int32_t entry_count; /* Number of entries in this file */
 179 };
 180
 181 struct master_header {
 182     struct tagcache_header tch;
 183     int32_t serial; /* Increasing counting number */
 184     int32_t commitid; /* Number of commits so far */
 185     int32_t dirty;
 186 };
 187
 188 /* For the endianess correction */
 189 static const char *tagfile_entry_ec   = "ll";
 190 /**
 191  Note: This should be (1 + TAG_COUNT) amount of l's.
 192  */
 193 static const char *index_entry_ec     = "lllllllllllllllllllll";
 194
 195 static const char *tagcache_header_ec = "lll";
 196 static const char *master_header_ec   = "llllll";
 197
 198 static struct master_header current_tcmh;
 199
 200 #ifdef HAVE_TC_RAMCACHE
 201 /* Header is created when loading database to ram. */
 202 struct ramcache_header {
 203     struct master_header h;      /* Header from the master index */
 204     struct index_entry *indices; /* Master index file content */
 205     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 206     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 207 };
 208
 209 # ifdef HAVE_EEPROM_SETTINGS
 210 struct statefile_header {
 211     struct ramcache_header *hdr;
 212     struct tagcache_stat tc_stat;
 213 };
 214 # endif
 215
 216 /* Pointer to allocated ramcache_header */
 217 static struct ramcache_header *hdr;
 218 #endif
 219
 220 /**
 221  * Full tag entries stored in a temporary file waiting
 222  * for commit to the cache. */
 223 struct temp_file_entry {
 224     long tag_offset[TAG_COUNT];
 225     short tag_length[TAG_COUNT];
 226     long flag;
 227
 228     long data_length;
 229 };
 230
 231 struct tempbuf_id_list {
 232     long id;
 233     struct tempbuf_id_list *next;
 234 };
 235
 236 struct tempbuf_searchidx {
 237     long idx_id;
 238     char *str;
 239     int seek;
 240     struct tempbuf_id_list idlist;
 241 };
 242
 243 /* Lookup buffer for fixing messed up index while after sorting. */
 244 static long commit_entry_count;
 245 static long lookup_buffer_depth;
 246 static struct tempbuf_searchidx **lookup;
 247
 248 /* Used when building the temporary file. */
 249 static int cachefd = -1, filenametag_fd;
 250 static int total_entry_count = 0;
 251 static int data_size = 0;
 252 static int processed_dir_count;
 253
 254 /* Thread safe locking */
 255 static volatile int write_lock;
 256 static volatile int read_lock;
 257
 258 static bool delete_entry(long idx_id);
 259
 260 const char* tagcache_tag_to_str(int tag)
 261 {
 262     return tags_str[tag];
 263 }
 264
 265 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
 266 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
 267 {
 268     return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
 269 }
 270
 271 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
 272 {
 273     return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
 274 }
 275
 276 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
 277 {
 278     return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
 279 }
 280
 281 #ifdef HAVE_DIRCACHE
 282 /**
 283  * Returns true if specified flag is still present, i.e., dircache
 284  * has not been reloaded.
 285  */
 286 static bool is_dircache_intact(void)
 287 {
 288     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 289 }
 290 #endif
 291
 292 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 293 {
 294     int fd;
 295     char buf[MAX_PATH];
 296     int rc;
 297
 298     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 299         return -1;
 300
 301     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 302
 303     fd = open(buf, write ? O_RDWR : O_RDONLY);
 304     if (fd < 0)
 305     {
 306         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 307         tc_stat.ready = false;
 308         return fd;
 309     }
 310
 311     /* Check the header. */
 312     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 313     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 314     {
 315         logf("header error");
 316         tc_stat.ready = false;
 317         close(fd);
 318         return -2;
 319     }
 320
 321     return fd;
 322 }
 323
 324 static int open_master_fd(struct master_header *hdr, bool write)
 325 {
 326     int fd;
 327     int rc;
 328
 329     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 330     if (fd < 0)
 331     {
 332         logf("master file open failed for R/W");
 333         tc_stat.ready = false;
 334         return fd;
 335     }
 336
 337     tc_stat.econ = false;
 338
 339     /* Check the header. */
 340     rc = read(fd, hdr, sizeof(struct master_header));
 341     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 342     {
 343         /* Success. */
 344         return fd;
 345     }
 346
 347     /* Trying to read again, this time with endianess correction enabled. */
 348     lseek(fd, 0, SEEK_SET);
 349
 350     rc = ecread(fd, hdr, 1, master_header_ec, true);
 351     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 352     {
 353         logf("header error");
 354         tc_stat.ready = false;
 355         close(fd);
 356         return -2;
 357     }
 358
 359     tc_stat.econ = true;
 360
 361     return fd;
 362 }
 363
 364 #ifndef __PCTOOL__
 365 static bool do_timed_yield(void)
 366 {
 367     /* Sorting can lock up for quite a while, so yield occasionally */
 368     static long wakeup_tick = 0;
 369     if (TIME_AFTER(current_tick, wakeup_tick))
 370     {
 371         wakeup_tick = current_tick + (HZ/4);
 372         yield();
 373         return true;
 374     }
 375     return false;
 376 }
 377 #endif
 378
 379 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 380 static long find_entry_ram(const char *filename,
 381                            const struct dircache_entry *dc)
 382 {
 383     static long last_pos = 0;
 384     int i;
 385
 386     /* Check if we tagcache is loaded into ram. */
 387     if (!tc_stat.ramcache)
 388         return -1;
 389
 390     if (dc == NULL)
 391         dc = dircache_get_entry_ptr(filename);
 392
 393     if (dc == NULL)
 394     {
 395         logf("tagcache: file not found.");
 396         return -1;
 397     }
 398
 399     try_again:
 400
 401     if (last_pos > 0)
 402         i = last_pos;
 403     else
 404         i = 0;
 405
 406     for (; i < hdr->h.tch.entry_count; i++)
 407     {
 408         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 409         {
 410             last_pos = MAX(0, i - 3);
 411             return i;
 412         }
 413
 414         do_timed_yield();
 415     }
 416
 417     if (last_pos > 0)
 418     {
 419         last_pos = 0;
 420         goto try_again;
 421     }
 422
 423     return -1;
 424 }
 425 #endif
 426
 427 static long find_entry_disk(const char *filename, bool localfd)
 428 {
 429     struct tagcache_header tch;
 430     static long last_pos = -1;
 431     long pos_history[POS_HISTORY_COUNT];
 432     long pos_history_idx = 0;
 433     bool found = false;
 434     struct tagfile_entry tfe;
 435     int fd;
 436     char buf[TAG_MAXLEN+32];
 437     int i;
 438     int pos = -1;
 439
 440     if (!tc_stat.ready)
 441         return -2;
 442
 443     fd = filenametag_fd;
 444     if (fd < 0 || localfd)
 445     {
 446         last_pos = -1;
 447         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 448             return -1;
 449     }
 450
 451     check_again:
 452
 453     if (last_pos > 0)
 454         lseek(fd, last_pos, SEEK_SET);
 455     else
 456         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 457
 458     while (true)
 459     {
 460         pos = lseek(fd, 0, SEEK_CUR);
 461         for (i = pos_history_idx-1; i >= 0; i--)
 462             pos_history[i+1] = pos_history[i];
 463         pos_history[0] = pos;
 464
 465         if (ecread_tagfile_entry(fd, &tfe)
 466             != sizeof(struct tagfile_entry))
 467         {
 468             break ;
 469         }
 470
 471         if (tfe.tag_length >= (long)sizeof(buf))
 472         {
 473             logf("too long tag #1");
 474             close(fd);
 475             if (!localfd)
 476                 filenametag_fd = -1;
 477             last_pos = -1;
 478             return -2;
 479         }
 480
 481         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 482         {
 483             logf("read error #2");
 484             close(fd);
 485             if (!localfd)
 486                 filenametag_fd = -1;
 487             last_pos = -1;
 488             return -3;
 489         }
 490
 491         if (!strcasecmp(filename, buf))
 492         {
 493             last_pos = pos_history[pos_history_idx];
 494             found = true;
 495             break ;
 496         }
 497
 498         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 499             pos_history_idx++;
 500     }
 501
 502     /* Not found? */
 503     if (!found)
 504     {
 505         if (last_pos > 0)
 506         {
 507             last_pos = -1;
 508             logf("seek again");
 509             goto check_again;
 510         }
 511
 512         if (fd != filenametag_fd || localfd)
 513             close(fd);
 514         return -4;
 515     }
 516
 517     if (fd != filenametag_fd || localfd)
 518         close(fd);
 519
 520     return tfe.idx_id;
 521 }
 522
 523 static int find_index(const char *filename)
 524 {
 525     long idx_id = -1;
 526
 527 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 528     if (tc_stat.ramcache && is_dircache_intact())
 529         idx_id = find_entry_ram(filename, NULL);
 530 #endif
 531
 532     if (idx_id < 0)
 533         idx_id = find_entry_disk(filename, true);
 534
 535     return idx_id;
 536 }
 537
 538 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 539 {
 540     int idx_id;
 541
 542     if (!tc_stat.ready)
 543         return false;
 544
 545     idx_id = find_index(filename);
 546     if (idx_id < 0)
 547         return false;
 548
 549     if (!tagcache_search(tcs, tag_filename))
 550         return false;
 551
 552     tcs->entry_count = 0;
 553     tcs->idx_id = idx_id;
 554
 555     return true;
 556 }
 557
 558 static bool get_index(int masterfd, int idxid,
 559                       struct index_entry *idx, bool use_ram)
 560 {
 561     bool localfd = false;
 562
 563     if (idxid < 0)
 564     {
 565         logf("Incorrect idxid: %d", idxid);
 566         return false;
 567     }
 568
 569 #ifdef HAVE_TC_RAMCACHE
 570     if (tc_stat.ramcache && use_ram)
 571     {
 572         if (hdr->indices[idxid].flag & FLAG_DELETED)
 573             return false;
 574
 575 # ifdef HAVE_DIRCACHE
 576         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 577             || is_dircache_intact())
 578 #endif
 579         {
 580             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 581             return true;
 582         }
 583     }
 584 #else
 585     (void)use_ram;
 586 #endif
 587
 588     if (masterfd < 0)
 589     {
 590         struct master_header tcmh;
 591
 592         localfd = true;
 593         masterfd = open_master_fd(&tcmh, false);
 594         if (masterfd < 0)
 595             return false;
 596     }
 597
 598     lseek(masterfd, idxid * sizeof(struct index_entry)
 599           + sizeof(struct master_header), SEEK_SET);
 600     if (ecread_index_entry(masterfd, idx)
 601         != sizeof(struct index_entry))
 602     {
 603         logf("read error #3");
 604         if (localfd)
 605             close(masterfd);
 606
 607         return false;
 608     }
 609
 610     if (localfd)
 611         close(masterfd);
 612
 613     if (idx->flag & FLAG_DELETED)
 614         return false;
 615
 616     return true;
 617 }
 618
 619 #ifndef __PCTOOL__
 620
 621 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 622 {
 623     /* We need to exclude all memory only flags & tags when writing to disk. */
 624     if (idx->flag & FLAG_DIRCACHE)
 625     {
 626         logf("memory only flags!");
 627         return false;
 628     }
 629
 630 #ifdef HAVE_TC_RAMCACHE
 631     /* Only update numeric data. Writing the whole index to RAM by memcpy
 632      * destroys dircache pointers!
 633      */
 634     if (tc_stat.ramcache)
 635     {
 636         int tag;
 637         struct index_entry *idx_ram = &hdr->indices[idxid];
 638
 639         for (tag = 0; tag < TAG_COUNT; tag++)
 640         {
 641             if (TAGCACHE_IS_NUMERIC(tag))
 642             {
 643                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 644             }
 645         }
 646
 647         /* Don't touch the dircache flag or attributes. */
 648         idx_ram->flag = (idx->flag & 0x0000ffff)
 649             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 650     }
 651 #endif
 652
 653     lseek(masterfd, idxid * sizeof(struct index_entry)
 654           + sizeof(struct master_header), SEEK_SET);
 655     if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
 656     {
 657         logf("write error #3");
 658         logf("idxid: %d", idxid);
 659         return false;
 660     }
 661
 662     return true;
 663 }
 664
 665 #endif /* !__PCTOOL__ */
 666
 667 static bool open_files(struct tagcache_search *tcs, int tag)
 668 {
 669     if (tcs->idxfd[tag] < 0)
 670     {
 671         char fn[MAX_PATH];
 672
 673         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 674         tcs->idxfd[tag] = open(fn, O_RDONLY);
 675     }
 676
 677     if (tcs->idxfd[tag] < 0)
 678     {
 679         logf("File not open!");
 680         return false;
 681     }
 682
 683     return true;
 684 }
 685
 686 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 687                      int tag, char *buf, long size)
 688 {
 689     struct tagfile_entry tfe;
 690     long seek;
 691
 692     *buf = '\0';
 693
 694     if (TAGCACHE_IS_NUMERIC(tag))
 695         return false;
 696
 697     seek = idx->tag_seek[tag];
 698     if (seek < 0)
 699     {
 700         logf("Retrieve failed");
 701         return false;
 702     }
 703
 704 #ifdef HAVE_TC_RAMCACHE
 705     if (tcs->ramsearch)
 706     {
 707         struct tagfile_entry *ep;
 708
 709 # ifdef HAVE_DIRCACHE
 710         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 711             && is_dircache_intact())
 712         {
 713             dircache_copy_path((struct dircache_entry *)seek,
 714                                buf, size);
 715             return true;
 716         }
 717         else
 718 # endif
 719         if (tag != tag_filename)
 720         {
 721             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 722             strlcpy(buf, ep->tag_data, size);
 723
 724             return true;
 725         }
 726     }
 727 #endif
 728
 729     if (!open_files(tcs, tag))
 730         return false;
 731
 732     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 733     if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
 734         != sizeof(struct tagfile_entry))
 735     {
 736         logf("read error #5");
 737         return false;
 738     }
 739
 740     if (tfe.tag_length >= size)
 741     {
 742         logf("too small buffer");
 743         return false;
 744     }
 745
 746     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 747         tfe.tag_length)
 748     {
 749         logf("read error #6");
 750         return false;
 751     }
 752
 753     buf[tfe.tag_length] = '\0';
 754
 755     return true;
 756 }
 757
 758 static long check_virtual_tags(int tag, const struct index_entry *idx)
 759 {
 760     long data = 0;
 761
 762     switch (tag)
 763     {
 764         case tag_virt_length_sec:
 765             data = (idx->tag_seek[tag_length]/1000) % 60;
 766             break;
 767
 768         case tag_virt_length_min:
 769             data = (idx->tag_seek[tag_length]/1000) / 60;
 770             break;
 771
 772         case tag_virt_playtime_sec:
 773             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 774             break;
 775
 776         case tag_virt_playtime_min:
 777             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 778             break;
 779
 780         case tag_virt_autoscore:
 781             if (idx->tag_seek[tag_length] == 0
 782                 || idx->tag_seek[tag_playcount] == 0)
 783             {
 784                 data = 0;
 785             }
 786             else
 787             {
 788                 /* A straight calculus gives:
 789                      autoscore = 100 * playtime / length / playcout (1)
 790                    Now, consider the euclidian division of playtime by length:
 791                      playtime = alpha * length + beta
 792                    With:
 793                      0 <= beta < length
 794                    Now, (1) becomes:
 795                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 796                    Both terms should be small enough to avoid any overflow
 797                 */
 798                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 799                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 800                 data /=  idx->tag_seek[tag_playcount];
 801             }
 802             break;
 803
 804         /* How many commits before the file has been added to the DB. */
 805         case tag_virt_entryage:
 806             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 807             break;
 808
 809         default:
 810             data = idx->tag_seek[tag];
 811     }
 812
 813     return data;
 814 }
 815
 816 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 817 {
 818     struct index_entry idx;
 819
 820     if (!tc_stat.ready)
 821         return false;
 822
 823     if (!TAGCACHE_IS_NUMERIC(tag))
 824         return -1;
 825
 826     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 827         return -2;
 828
 829     return check_virtual_tags(tag, &idx);
 830 }
 831
 832 inline static bool str_ends_with(const char *str1, const char *str2)
 833 {
 834     int str_len = strlen(str1);
 835     int clause_len = strlen(str2);
 836
 837     if (clause_len > str_len)
 838         return false;
 839
 840     return !strcasecmp(&str1[str_len - clause_len], str2);
 841 }
 842
 843 inline static bool str_oneof(const char *str, const char *list)
 844 {
 845     const char *sep;
 846     int l, len = strlen(str);
 847
 848     while (*list)
 849     {
 850         sep = strchr(list, '|');
 851         l = sep ? (long)sep - (long)list : (int)strlen(list);
 852         if ((l==len) && !strncasecmp(str, list, len))
 853             return true;
 854         list += sep ? l + 1 : l;
 855     }
 856
 857     return false;
 858 }
 859
 860 static bool check_against_clause(long numeric, const char *str,
 861                                  const struct tagcache_search_clause *clause)
 862 {
 863     if (clause->numeric)
 864     {
 865         switch (clause->type)
 866         {
 867             case clause_is:
 868                 return numeric == clause->numeric_data;
 869             case clause_is_not:
 870                 return numeric != clause->numeric_data;
 871             case clause_gt:
 872                 return numeric > clause->numeric_data;
 873             case clause_gteq:
 874                 return numeric >= clause->numeric_data;
 875             case clause_lt:
 876                 return numeric < clause->numeric_data;
 877             case clause_lteq:
 878                 return numeric <= clause->numeric_data;
 879             default:
 880                 logf("Incorrect numeric tag: %d", clause->type);
 881         }
 882     }
 883     else
 884     {
 885         switch (clause->type)
 886         {
 887             case clause_is:
 888                 return !strcasecmp(clause->str, str);
 889             case clause_is_not:
 890                 return strcasecmp(clause->str, str);
 891             case clause_gt:
 892                 return 0>strcasecmp(clause->str, str);
 893             case clause_gteq:
 894                 return 0>=strcasecmp(clause->str, str);
 895             case clause_lt:
 896                 return 0<strcasecmp(clause->str, str);
 897             case clause_lteq:
 898                 return 0<=strcasecmp(clause->str, str);
 899             case clause_contains:
 900                 return (strcasestr(str, clause->str) != NULL);
 901             case clause_not_contains:
 902                 return (strcasestr(str, clause->str) == NULL);
 903             case clause_begins_with:
 904                 return (strcasestr(str, clause->str) == str);
 905             case clause_not_begins_with:
 906                 return (strcasestr(str, clause->str) != str);
 907             case clause_ends_with:
 908                 return str_ends_with(str, clause->str);
 909             case clause_not_ends_with:
 910                 return !str_ends_with(str, clause->str);
 911             case clause_oneof:
 912                 return str_oneof(str, clause->str);
 913
 914             default:
 915                 logf("Incorrect tag: %d", clause->type);
 916         }
 917     }
 918
 919     return false;
 920 }
 921
 922 static bool check_clauses(struct tagcache_search *tcs,
 923                           struct index_entry *idx,
 924                           struct tagcache_search_clause **clause, int count)
 925 {
 926     int i;
 927
 928 #ifdef HAVE_TC_RAMCACHE
 929     if (tcs->ramsearch)
 930     {
 931         /* Go through all conditional clauses. */
 932         for (i = 0; i < count; i++)
 933         {
 934             struct tagfile_entry *tfe;
 935             int seek;
 936             char buf[256];
 937             char *str = NULL;
 938
 939             seek = check_virtual_tags(clause[i]->tag, idx);
 940
 941             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 942             {
 943                 if (clause[i]->tag == tag_filename)
 944                 {
 945                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 946                     str = buf;
 947                 }
 948                 else
 949                 {
 950                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 951                     str = tfe->tag_data;
 952                 }
 953             }
 954
 955             if (!check_against_clause(seek, str, clause[i]))
 956                 return false;
 957         }
 958     }
 959     else
 960 #endif
 961     {
 962         /* Check for conditions. */
 963         for (i = 0; i < count; i++)
 964         {
 965             struct tagfile_entry tfe;
 966             int seek;
 967             char str[256];
 968
 969             seek = check_virtual_tags(clause[i]->tag, idx);
 970
 971             memset(str, 0, sizeof str);
 972             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 973             {
 974                 int fd = tcs->idxfd[clause[i]->tag];
 975                 lseek(fd, seek, SEEK_SET);
 976                 ecread_tagfile_entry(fd, &tfe);
 977                 if (tfe.tag_length >= (int)sizeof(str))
 978                 {
 979                     logf("Too long tag read!");
 980                     break ;
 981                 }
 982
 983                 read(fd, str, tfe.tag_length);
 984
 985                 /* Check if entry has been deleted. */
 986                 if (str[0] == '\0')
 987                     break;
 988             }
 989
 990             if (!check_against_clause(seek, str, clause[i]))
 991                 return false;
 992         }
 993     }
 994
 995     return true;
 996 }
 997
 998 bool tagcache_check_clauses(struct tagcache_search *tcs,
 999                             struct tagcache_search_clause **clause, int count)
1000 {
1001     struct index_entry idx;
1002
1003     if (count == 0)
1004         return true;
1005
1006     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1007         return false;
1008
1009     return check_clauses(tcs, &idx, clause, count);
1010 }
1011
1012 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1013 {
1014     int i;
1015
1016     /* If uniq buffer is not defined we must return true for search to work. */
1017     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1018         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1019     {
1020         return true;
1021     }
1022
1023     for (i = 0; i < tcs->unique_list_count; i++)
1024     {
1025         /* Return false if entry is found. */
1026         if (tcs->unique_list[i] == id)
1027             return false;
1028     }
1029
1030     if (tcs->unique_list_count < tcs->unique_list_capacity)
1031     {
1032         tcs->unique_list[i] = id;
1033         tcs->unique_list_count++;
1034     }
1035
1036     return true;
1037 }
1038
1039 static bool build_lookup_list(struct tagcache_search *tcs)
1040 {
1041     struct index_entry entry;
1042     int i, j;
1043
1044     tcs->seek_list_count = 0;
1045
1046 #ifdef HAVE_TC_RAMCACHE
1047     if (tcs->ramsearch
1048 # ifdef HAVE_DIRCACHE
1049         && (tcs->type != tag_filename || is_dircache_intact())
1050 # endif
1051         )
1052     {
1053         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1054         {
1055             struct tagcache_seeklist_entry *seeklist;
1056             struct index_entry *idx = &hdr->indices[i];
1057             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1058                 break ;
1059
1060             /* Skip deleted files. */
1061             if (idx->flag & FLAG_DELETED)
1062                 continue;
1063
1064             /* Go through all filters.. */
1065             for (j = 0; j < tcs->filter_count; j++)
1066             {
1067                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1068                 {
1069                     break ;
1070                 }
1071             }
1072
1073             if (j < tcs->filter_count)
1074                 continue ;
1075
1076             /* Check for conditions. */
1077             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1078                 continue;
1079
1080             /* Add to the seek list if not already in uniq buffer. */
1081             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1082                 continue;
1083
1084             /* Lets add it. */
1085             seeklist = &tcs->seeklist[tcs->seek_list_count];
1086             seeklist->seek = idx->tag_seek[tcs->type];
1087             seeklist->flag = idx->flag;
1088             seeklist->idx_id = i;
1089             tcs->seek_list_count++;
1090         }
1091
1092         tcs->seek_pos = i;
1093
1094         return tcs->seek_list_count > 0;
1095     }
1096 #endif
1097
1098     if (tcs->masterfd < 0)
1099     {
1100         struct master_header tcmh;
1101         tcs->masterfd = open_master_fd(&tcmh, false);
1102     }
1103
1104     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1105             sizeof(struct master_header), SEEK_SET);
1106
1107     while (ecread_index_entry(tcs->masterfd, &entry)
1108            == sizeof(struct index_entry))
1109     {
1110         struct tagcache_seeklist_entry *seeklist;
1111
1112         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1113             break ;
1114
1115         i = tcs->seek_pos;
1116         tcs->seek_pos++;
1117
1118         /* Check if entry has been deleted. */
1119         if (entry.flag & FLAG_DELETED)
1120             continue;
1121
1122         /* Go through all filters.. */
1123         for (j = 0; j < tcs->filter_count; j++)
1124         {
1125             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1126                 break ;
1127         }
1128
1129         if (j < tcs->filter_count)
1130             continue ;
1131
1132         /* Check for conditions. */
1133         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1134             continue;
1135
1136         /* Add to the seek list if not already in uniq buffer. */
1137         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1138             continue;
1139
1140         /* Lets add it. */
1141         seeklist = &tcs->seeklist[tcs->seek_list_count];
1142         seeklist->seek = entry.tag_seek[tcs->type];
1143         seeklist->flag = entry.flag;
1144         seeklist->idx_id = i;
1145         tcs->seek_list_count++;
1146
1147         yield();
1148     }
1149
1150     return tcs->seek_list_count > 0;
1151 }
1152
1153
1154 static void remove_files(void)
1155 {
1156     int i;
1157     char buf[MAX_PATH];
1158
1159     tc_stat.ready = false;
1160     tc_stat.ramcache = false;
1161     tc_stat.econ = false;
1162     remove(TAGCACHE_FILE_MASTER);
1163     for (i = 0; i < TAG_COUNT; i++)
1164     {
1165         if (TAGCACHE_IS_NUMERIC(i))
1166             continue;
1167
1168         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1169         remove(buf);
1170     }
1171 }
1172
1173
1174 static bool check_all_headers(void)
1175 {
1176     struct master_header myhdr;
1177     struct tagcache_header tch;
1178     int tag;
1179     int fd;
1180
1181     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1182         return false;
1183
1184     close(fd);
1185     if (myhdr.dirty)
1186     {
1187         logf("tagcache is dirty!");
1188         return false;
1189     }
1190
1191     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1192
1193     for (tag = 0; tag < TAG_COUNT; tag++)
1194     {
1195         if (TAGCACHE_IS_NUMERIC(tag))
1196             continue;
1197
1198         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1199             return false;
1200
1201         close(fd);
1202     }
1203
1204     return true;
1205 }
1206
1207 bool tagcache_is_busy(void)
1208 {
1209     return read_lock || write_lock;
1210 }
1211
1212 bool tagcache_search(struct tagcache_search *tcs, int tag)
1213 {
1214     struct tagcache_header tag_hdr;
1215     struct master_header   master_hdr;
1216     int i;
1217
1218     while (read_lock)
1219         sleep(1);
1220
1221     memset(tcs, 0, sizeof(struct tagcache_search));
1222     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1223         return false;
1224
1225     tcs->position = sizeof(struct tagcache_header);
1226     tcs->type = tag;
1227     tcs->seek_pos = 0;
1228     tcs->list_position = 0;
1229     tcs->seek_list_count = 0;
1230     tcs->filter_count = 0;
1231     tcs->masterfd = -1;
1232
1233     for (i = 0; i < TAG_COUNT; i++)
1234         tcs->idxfd[i] = -1;
1235
1236 #ifndef HAVE_TC_RAMCACHE
1237     tcs->ramsearch = false;
1238 #else
1239     tcs->ramsearch = tc_stat.ramcache;
1240     if (tcs->ramsearch)
1241     {
1242         tcs->entry_count = hdr->entry_count[tcs->type];
1243     }
1244     else
1245 #endif
1246     {
1247         /* Always open as R/W so we can pass tcs to functions that modify data also
1248          * without failing. */
1249         tcs->masterfd = open_master_fd(&master_hdr, true);
1250         if (tcs->masterfd < 0)
1251             return false;
1252
1253         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1254         {
1255             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1256             if (tcs->idxfd[tcs->type] < 0)
1257                 return false;
1258
1259             tcs->entry_count = tag_hdr.entry_count;
1260         }
1261         else
1262         {
1263             tcs->entry_count = master_hdr.tch.entry_count;
1264         }
1265     }
1266
1267     tcs->valid = true;
1268     tcs->initialized = true;
1269     write_lock++;
1270
1271     return true;
1272 }
1273
1274 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1275                                  void *buffer, long length)
1276 {
1277     tcs->unique_list = (unsigned long *)buffer;
1278     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1279     tcs->unique_list_count = 0;
1280 }
1281
1282 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1283                                 int tag, int seek)
1284 {
1285     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1286         return false;
1287
1288     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1289         return false;
1290
1291     tcs->filter_tag[tcs->filter_count] = tag;
1292     tcs->filter_seek[tcs->filter_count] = seek;
1293     tcs->filter_count++;
1294
1295     return true;
1296 }
1297
1298 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1299                                 struct tagcache_search_clause *clause)
1300 {
1301     int i;
1302
1303     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1304     {
1305         logf("Too many clauses");
1306         return false;
1307     }
1308
1309     /* Check if there is already a similar filter in present (filters are
1310      * much faster than clauses).
1311      */
1312     for (i = 0; i < tcs->filter_count; i++)
1313     {
1314         if (tcs->filter_tag[i] == clause->tag)
1315             return true;
1316     }
1317
1318     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1319     {
1320         char buf[MAX_PATH];
1321
1322         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1323         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1324     }
1325
1326     tcs->clause[tcs->clause_count] = clause;
1327     tcs->clause_count++;
1328
1329     return true;
1330 }
1331
1332 static bool get_next(struct tagcache_search *tcs)
1333 {
1334     static char buf[TAG_MAXLEN+32];
1335     struct tagfile_entry entry;
1336     long flag = 0;
1337
1338     if (!tcs->valid || !tc_stat.ready)
1339         return false;
1340
1341     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1342 #ifdef HAVE_TC_RAMCACHE
1343         && !tcs->ramsearch
1344 #endif
1345         )
1346         return false;
1347
1348     /* Relative fetch. */
1349     if (tcs->filter_count > 0 || tcs->clause_count > 0
1350         || TAGCACHE_IS_NUMERIC(tcs->type)
1351 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1352         /* We need to retrieve flag status for dircache. */
1353         || (tcs->ramsearch && tcs->type == tag_filename)
1354 #endif
1355         )
1356     {
1357         struct tagcache_seeklist_entry *seeklist;
1358
1359         /* Check for end of list. */
1360         if (tcs->list_position == tcs->seek_list_count)
1361         {
1362             tcs->list_position = 0;
1363
1364             /* Try to fetch more. */
1365             if (!build_lookup_list(tcs))
1366             {
1367                 tcs->valid = false;
1368                 return false;
1369             }
1370         }
1371
1372         seeklist = &tcs->seeklist[tcs->list_position];
1373         flag = seeklist->flag;
1374         tcs->position = seeklist->seek;
1375         tcs->idx_id = seeklist->idx_id;
1376         tcs->list_position++;
1377     }
1378     else
1379     {
1380         if (tcs->entry_count == 0)
1381         {
1382             tcs->valid = false;
1383             return false;
1384         }
1385
1386         tcs->entry_count--;
1387     }
1388
1389     tcs->result_seek = tcs->position;
1390
1391     if (TAGCACHE_IS_NUMERIC(tcs->type))
1392     {
1393         snprintf(buf, sizeof(buf), "%ld", tcs->position);
1394         tcs->result = buf;
1395         tcs->result_len = strlen(buf) + 1;
1396         return true;
1397     }
1398
1399     /* Direct fetch. */
1400 #ifdef HAVE_TC_RAMCACHE
1401     if (tcs->ramsearch)
1402     {
1403
1404 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1405         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1406             && is_dircache_intact())
1407         {
1408             dircache_copy_path((struct dircache_entry *)tcs->position,
1409                                buf, sizeof buf);
1410             tcs->result = buf;
1411             tcs->result_len = strlen(buf) + 1;
1412             tcs->ramresult = false;
1413
1414             return true;
1415         }
1416         else
1417 #endif
1418         if (tcs->type != tag_filename)
1419         {
1420             struct tagfile_entry *ep;
1421
1422             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1423             tcs->result = ep->tag_data;
1424             tcs->result_len = strlen(tcs->result) + 1;
1425             tcs->idx_id = ep->idx_id;
1426             tcs->ramresult = true;
1427
1428             /* Increase position for the next run. This may get overwritten. */
1429             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1430
1431             return true;
1432         }
1433     }
1434 #endif
1435
1436     if (!open_files(tcs, tcs->type))
1437     {
1438         tcs->valid = false;
1439         return false;
1440     }
1441
1442     /* Seek stream to the correct position and continue to direct fetch. */
1443     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1444
1445     if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1446     {
1447         logf("read error #5");
1448         tcs->valid = false;
1449         return false;
1450     }
1451
1452     if (entry.tag_length > (long)sizeof(buf))
1453     {
1454         tcs->valid = false;
1455         logf("too long tag #2");
1456         logf("P:%lX/%lX", tcs->position, entry.tag_length);
1457         return false;
1458     }
1459
1460     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1461     {
1462         tcs->valid = false;
1463         logf("read error #4");
1464         return false;
1465     }
1466
1467     /**
1468      Update the position for the next read (this may be overridden
1469      if filters or clauses are being used).
1470      */
1471     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1472     tcs->result = buf;
1473     tcs->result_len = strlen(tcs->result) + 1;
1474     tcs->idx_id = entry.idx_id;
1475     tcs->ramresult = false;
1476
1477     return true;
1478 }
1479
1480 bool tagcache_get_next(struct tagcache_search *tcs)
1481 {
1482     while (get_next(tcs))
1483     {
1484         if (tcs->result_len > 1)
1485             return true;
1486     }
1487
1488     return false;
1489 }
1490
1491 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1492                        int tag, char *buf, long size)
1493 {
1494     struct index_entry idx;
1495
1496     *buf = '\0';
1497     if (!get_index(tcs->masterfd, idxid, &idx, true))
1498         return false;
1499
1500     return retrieve(tcs, &idx, tag, buf, size);
1501 }
1502
1503 static bool update_master_header(void)
1504 {
1505     struct master_header myhdr;
1506     int fd;
1507
1508     if (!tc_stat.ready)
1509         return false;
1510
1511     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1512         return false;
1513
1514     myhdr.serial = current_tcmh.serial;
1515     myhdr.commitid = current_tcmh.commitid;
1516     myhdr.dirty = current_tcmh.dirty;
1517
1518     /* Write it back */
1519     lseek(fd, 0, SEEK_SET);
1520     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1521     close(fd);
1522
1523 #ifdef HAVE_TC_RAMCACHE
1524     if (hdr)
1525     {
1526         hdr->h.serial = current_tcmh.serial;
1527         hdr->h.commitid = current_tcmh.commitid;
1528         hdr->h.dirty = current_tcmh.dirty;
1529     }
1530 #endif
1531
1532     return true;
1533 }
1534
1535 #if 0
1536
1537 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1538 {
1539     struct tagentry *entry;
1540
1541     if (tcs->type != tag_title)
1542         return ;
1543
1544     /* We will need reserve buffer for this. */
1545     if (tcs->ramcache)
1546     {
1547         struct tagfile_entry *ep;
1548
1549         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1550         tcs->seek_list[tcs->seek_list_count];
1551     }
1552
1553     entry = find_entry_ram();
1554
1555 }
1556 #endif
1557
1558 void tagcache_search_finish(struct tagcache_search *tcs)
1559 {
1560     int i;
1561
1562     if (!tcs->initialized)
1563         return;
1564
1565     if (tcs->masterfd >= 0)
1566     {
1567         close(tcs->masterfd);
1568         tcs->masterfd = -1;
1569     }
1570
1571     for (i = 0; i < TAG_COUNT; i++)
1572     {
1573         if (tcs->idxfd[i] >= 0)
1574         {
1575             close(tcs->idxfd[i]);
1576             tcs->idxfd[i] = -1;
1577         }
1578     }
1579
1580     tcs->ramsearch = false;
1581     tcs->valid = false;
1582     tcs->initialized = 0;
1583     if (write_lock > 0)
1584         write_lock--;
1585 }
1586
1587 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1588 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1589 {
1590     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1591 }
1592
1593 static long get_tag_numeric(const struct index_entry *entry, int tag)
1594 {
1595     return check_virtual_tags(tag, entry);
1596 }
1597
1598 static char* get_tag_string(const struct index_entry *entry, int tag)
1599 {
1600     char* s = get_tag(entry, tag)->tag_data;
1601     return strcmp(s, UNTAGGED) ? s : NULL;
1602 }
1603
1604 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1605 {
1606     struct index_entry *entry;
1607     int idx_id;
1608
1609     if (!tc_stat.ready || !tc_stat.ramcache)
1610         return false;
1611
1612     /* Find the corresponding entry in tagcache. */
1613     idx_id = find_entry_ram(filename, NULL);
1614     if (idx_id < 0)
1615         return false;
1616
1617     entry = &hdr->indices[idx_id];
1618
1619     memset(id3, 0, sizeof(struct mp3entry));
1620
1621     id3->title        = get_tag_string(entry, tag_title);
1622     id3->artist       = get_tag_string(entry, tag_artist);
1623     id3->album        = get_tag_string(entry, tag_album);
1624     id3->genre_string = get_tag_string(entry, tag_genre);
1625     id3->composer     = get_tag_string(entry, tag_composer);
1626     id3->comment      = get_tag_string(entry, tag_comment);
1627     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1628     id3->grouping     = get_tag_string(entry, tag_grouping);
1629
1630     id3->length     = get_tag_numeric(entry, tag_length);
1631     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1632     id3->rating     = get_tag_numeric(entry, tag_rating);
1633     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1634     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1635     id3->year       = get_tag_numeric(entry, tag_year);
1636
1637     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1638     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1639     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1640     if (id3->bitrate == 0)
1641         id3->bitrate = 1;
1642
1643     return true;
1644 }
1645 #endif
1646
1647 static inline void write_item(const char *item)
1648 {
1649     int len = strlen(item) + 1;
1650
1651     data_size += len;
1652     write(cachefd, item, len);
1653 }
1654
1655 static int check_if_empty(char **tag)
1656 {
1657     int length;
1658
1659     if (*tag == NULL || **tag == '\0')
1660     {
1661         *tag = UNTAGGED;
1662         return sizeof(UNTAGGED); /* Tag length */
1663     }
1664
1665     length = strlen(*tag);
1666     if (length > TAG_MAXLEN)
1667     {
1668         logf("over length tag: %s", *tag);
1669         length = TAG_MAXLEN;
1670         (*tag)[length] = '\0';
1671     }
1672
1673     return length + 1;
1674 }
1675
1676 #define ADD_TAG(entry,tag,data) \
1677     /* Adding tag */ \
1678     entry.tag_offset[tag] = offset; \
1679     entry.tag_length[tag] = check_if_empty(data); \
1680     offset += entry.tag_length[tag]
1681 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1682  * idea, as it uses lots of stack and is called from a recursive function
1683  * (check_dir).
1684  */
1685 static void __attribute__ ((noinline)) add_tagcache(char *path,
1686                                                     unsigned long mtime
1687 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1688                                                     ,const struct dircache_entry *dc
1689 #endif
1690                                                    )
1691 {
1692     struct mp3entry id3;
1693     struct temp_file_entry entry;
1694     bool ret;
1695     int fd;
1696     int idx_id = -1;
1697     char tracknumfix[3];
1698     int offset = 0;
1699     int path_length = strlen(path);
1700     bool has_albumartist;
1701     bool has_grouping;
1702
1703 #ifdef SIMULATOR
1704     /* Crude logging for the sim - to aid in debugging */
1705     int logfd = open(ROCKBOX_DIR "/database.log",
1706                      O_WRONLY | O_APPEND | O_CREAT, 0666);
1707     if (logfd >= 0) {
1708         write(logfd, path, strlen(path));
1709         write(logfd, "\n", 1);
1710         close(logfd);
1711     }
1712 #endif
1713
1714     if (cachefd < 0)
1715         return ;
1716
1717     /* Check for overlength file path. */
1718     if (path_length > TAG_MAXLEN)
1719     {
1720         /* Path can't be shortened. */
1721         logf("Too long path: %s", path);
1722         return ;
1723     }
1724
1725     /* Check if the file is supported. */
1726     if (probe_file_format(path) == AFMT_UNKNOWN)
1727         return ;
1728
1729     /* Check if the file is already cached. */
1730 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1731     if (tc_stat.ramcache && is_dircache_intact())
1732     {
1733         idx_id = find_entry_ram(path, dc);
1734     }
1735 #endif
1736
1737     /* Be sure the entry doesn't exist. */
1738     if (filenametag_fd >= 0 && idx_id < 0)
1739         idx_id = find_entry_disk(path, false);
1740
1741     /* Check if file has been modified. */
1742     if (idx_id >= 0)
1743     {
1744         struct index_entry idx;
1745
1746         /* TODO: Mark that the index exists (for fast reverse scan) */
1747         //found_idx[idx_id/8] |= idx_id%8;
1748
1749         if (!get_index(-1, idx_id, &idx, true))
1750         {
1751             logf("failed to retrieve index entry");
1752             return ;
1753         }
1754
1755         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1756         {
1757             /* No changes to file. */
1758             return ;
1759         }
1760
1761         /* Metadata might have been changed. Delete the entry. */
1762         logf("Re-adding: %s", path);
1763         if (!delete_entry(idx_id))
1764         {
1765             logf("delete_entry failed: %d", idx_id);
1766             return ;
1767         }
1768     }
1769
1770     fd = open(path, O_RDONLY);
1771     if (fd < 0)
1772     {
1773         logf("open fail: %s", path);
1774         return ;
1775     }
1776
1777     memset(&id3, 0, sizeof(struct mp3entry));
1778     memset(&entry, 0, sizeof(struct temp_file_entry));
1779     memset(&tracknumfix, 0, sizeof(tracknumfix));
1780     ret = get_metadata(&id3, fd, path);
1781     close(fd);
1782
1783     if (!ret)
1784         return ;
1785
1786     logf("-> %s", path);
1787
1788     /* Generate track number if missing. */
1789     if (id3.tracknum <= 0)
1790     {
1791         const char *p = strrchr(path, '.');
1792
1793         if (p == NULL)
1794             p = &path[strlen(path)-1];
1795
1796         while (*p != '/')
1797         {
1798             if (isdigit(*p) && isdigit(*(p-1)))
1799             {
1800                 tracknumfix[1] = *p--;
1801                 tracknumfix[0] = *p;
1802                 break;
1803             }
1804             p--;
1805         }
1806
1807         if (tracknumfix[0] != '\0')
1808         {
1809             id3.tracknum = atoi(tracknumfix);
1810             /* Set a flag to indicate track number has been generated. */
1811             entry.flag |= FLAG_TRKNUMGEN;
1812         }
1813         else
1814         {
1815             /* Unable to generate track number. */
1816             id3.tracknum = -1;
1817         }
1818     }
1819
1820     /* Numeric tags */
1821     entry.tag_offset[tag_year] = id3.year;
1822     entry.tag_offset[tag_discnumber] = id3.discnum;
1823     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1824     entry.tag_offset[tag_length] = id3.length;
1825     entry.tag_offset[tag_bitrate] = id3.bitrate;
1826     entry.tag_offset[tag_mtime] = mtime;
1827
1828     /* String tags. */
1829     has_albumartist = id3.albumartist != NULL
1830         && strlen(id3.albumartist) > 0;
1831     has_grouping = id3.grouping != NULL
1832         && strlen(id3.grouping) > 0;
1833
1834     ADD_TAG(entry, tag_filename, &path);
1835     ADD_TAG(entry, tag_title, &id3.title);
1836     ADD_TAG(entry, tag_artist, &id3.artist);
1837     ADD_TAG(entry, tag_album, &id3.album);
1838     ADD_TAG(entry, tag_genre, &id3.genre_string);
1839     ADD_TAG(entry, tag_composer, &id3.composer);
1840     ADD_TAG(entry, tag_comment, &id3.comment);
1841     if (has_albumartist)
1842     {
1843         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1844     }
1845     else
1846     {
1847         ADD_TAG(entry, tag_albumartist, &id3.artist);
1848     }
1849     if (has_grouping)
1850     {
1851         ADD_TAG(entry, tag_grouping, &id3.grouping);
1852     }
1853     else
1854     {
1855         ADD_TAG(entry, tag_grouping, &id3.title);
1856     }
1857     entry.data_length = offset;
1858
1859     /* Write the header */
1860     write(cachefd, &entry, sizeof(struct temp_file_entry));
1861
1862     /* And tags also... Correct order is critical */
1863     write_item(path);
1864     write_item(id3.title);
1865     write_item(id3.artist);
1866     write_item(id3.album);
1867     write_item(id3.genre_string);
1868     write_item(id3.composer);
1869     write_item(id3.comment);
1870     if (has_albumartist)
1871     {
1872         write_item(id3.albumartist);
1873     }
1874     else
1875     {
1876         write_item(id3.artist);
1877     }
1878     if (has_grouping)
1879     {
1880         write_item(id3.grouping);
1881     }
1882     else
1883     {
1884         write_item(id3.title);
1885     }
1886     total_entry_count++;
1887 }
1888
1889 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1890 {
1891     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1892     int len = strlen(str)+1;
1893     int i;
1894     unsigned crc32;
1895     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1896     char buf[TAG_MAXLEN+32];
1897
1898     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1899         buf[i] = tolower(str[i]);
1900     buf[i] = '\0';
1901
1902     crc32 = crc_32(buf, i, 0xffffffff);
1903
1904     if (unique)
1905     {
1906         /* Check if the crc does not exist -> entry does not exist for sure. */
1907         for (i = 0; i < tempbufidx; i++)
1908         {
1909             if (crcbuf[-i] != crc32)
1910                 continue;
1911
1912             if (!strcasecmp(str, index[i].str))
1913             {
1914                 if (id < 0 || id >= lookup_buffer_depth)
1915                 {
1916                     logf("lookup buf overf.: %d", id);
1917                     return false;
1918                 }
1919
1920                 lookup[id] = &index[i];
1921                 return true;
1922             }
1923         }
1924     }
1925
1926     /* Insert to CRC buffer. */
1927     crcbuf[-tempbufidx] = crc32;
1928     tempbuf_left -= 4;
1929
1930     /* Insert it to the buffer. */
1931     tempbuf_left -= len;
1932     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1933         return false;
1934
1935     if (id >= lookup_buffer_depth)
1936     {
1937         logf("lookup buf overf. #2: %d", id);
1938         return false;
1939     }
1940
1941     if (id >= 0)
1942     {
1943         lookup[id] = &index[tempbufidx];
1944         index[tempbufidx].idlist.id = id;
1945     }
1946     else
1947         index[tempbufidx].idlist.id = -1;
1948
1949     index[tempbufidx].idlist.next = NULL;
1950     index[tempbufidx].idx_id = idx_id;
1951     index[tempbufidx].seek = -1;
1952     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1953     memcpy(index[tempbufidx].str, str, len);
1954     tempbuf_pos += len;
1955     tempbufidx++;
1956
1957     return true;
1958 }
1959
1960 static int compare(const void *p1, const void *p2)
1961 {
1962     do_timed_yield();
1963
1964     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1965     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1966
1967     if (strcmp(e1->str, UNTAGGED) == 0)
1968     {
1969         if (strcmp(e2->str, UNTAGGED) == 0)
1970             return 0;
1971         return -1;
1972     }
1973     else if (strcmp(e2->str, UNTAGGED) == 0)
1974         return 1;
1975
1976     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1977 }
1978
1979 static int tempbuf_sort(int fd)
1980 {
1981     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1982     struct tagfile_entry fe;
1983     int i;
1984     int length;
1985
1986     /* Generate reverse lookup entries. */
1987     for (i = 0; i < lookup_buffer_depth; i++)
1988     {
1989         struct tempbuf_id_list *idlist;
1990
1991         if (!lookup[i])
1992             continue;
1993
1994         if (lookup[i]->idlist.id == i)
1995             continue;
1996
1997         idlist = &lookup[i]->idlist;
1998         while (idlist->next != NULL)
1999             idlist = idlist->next;
2000
2001         tempbuf_left -= sizeof(struct tempbuf_id_list);
2002         if (tempbuf_left - 4 < 0)
2003             return -1;
2004
2005         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2006         if (tempbuf_pos & 0x03)
2007         {
2008             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2009             tempbuf_left -= 3;
2010             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2011         }
2012         tempbuf_pos += sizeof(struct tempbuf_id_list);
2013
2014         idlist = idlist->next;
2015         idlist->id = i;
2016         idlist->next = NULL;
2017
2018         do_timed_yield();
2019     }
2020
2021     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2022     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2023
2024     for (i = 0; i < tempbufidx; i++)
2025     {
2026         struct tempbuf_id_list *idlist = &index[i].idlist;
2027
2028         /* Fix the lookup list. */
2029         while (idlist != NULL)
2030         {
2031             if (idlist->id >= 0)
2032                 lookup[idlist->id] = &index[i];
2033             idlist = idlist->next;
2034         }
2035
2036         index[i].seek = lseek(fd, 0, SEEK_CUR);
2037         length = strlen(index[i].str) + 1;
2038         fe.tag_length = length;
2039         fe.idx_id = index[i].idx_id;
2040
2041         /* Check the chunk alignment. */
2042         if ((fe.tag_length + sizeof(struct tagfile_entry))
2043             % TAGFILE_ENTRY_CHUNK_LENGTH)
2044         {
2045             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2046                 ((fe.tag_length + sizeof(struct tagfile_entry))
2047                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2048         }
2049
2050 #ifdef TAGCACHE_STRICT_ALIGN
2051         /* Make sure the entry is long aligned. */
2052         if (index[i].seek & 0x03)
2053         {
2054             logf("tempbuf_sort: alignment error!");
2055             return -3;
2056         }
2057 #endif
2058
2059         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2060             sizeof(struct tagfile_entry))
2061         {
2062             logf("tempbuf_sort: write error #1");
2063             return -1;
2064         }
2065
2066         if (write(fd, index[i].str, length) != length)
2067         {
2068             logf("tempbuf_sort: write error #2");
2069             return -2;
2070         }
2071
2072         /* Write some padding. */
2073         if (fe.tag_length - length > 0)
2074             write(fd, "XXXXXXXX", fe.tag_length - length);
2075     }
2076
2077     return i;
2078 }
2079
2080 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2081 {
2082     if (id < 0 || id >= lookup_buffer_depth)
2083         return NULL;
2084
2085     return lookup[id];
2086 }
2087
2088
2089 inline static int tempbuf_find_location(int id)
2090 {
2091     struct tempbuf_searchidx *entry;
2092
2093     entry = tempbuf_locate(id);
2094     if (entry == NULL)
2095         return -1;
2096
2097     return entry->seek;
2098 }
2099
2100 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2101 {
2102     struct master_header tcmh;
2103     struct index_entry idx;
2104     int masterfd;
2105     int masterfd_pos;
2106     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2107     int max_entries;
2108     int entries_processed = 0;
2109     int i, j;
2110     char buf[TAG_MAXLEN];
2111
2112     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2113
2114     logf("Building numeric indices...");
2115     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2116
2117     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2118         return false;
2119
2120     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2121                          SEEK_CUR);
2122     if (masterfd_pos == filesize(masterfd))
2123     {
2124         logf("we can't append!");
2125         close(masterfd);
2126         return false;
2127     }
2128
2129     while (entries_processed < h->entry_count)
2130     {
2131         int count = MIN(h->entry_count - entries_processed, max_entries);
2132
2133         /* Read in as many entries as possible. */
2134         for (i = 0; i < count; i++)
2135         {
2136             struct temp_file_entry *tfe = &entrybuf[i];
2137             int datastart;
2138
2139             /* Read in numeric data. */
2140             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2141                 sizeof(struct temp_file_entry))
2142             {
2143                 logf("read fail #1");
2144                 close(masterfd);
2145                 return false;
2146             }
2147
2148             datastart = lseek(tmpfd, 0, SEEK_CUR);
2149
2150             /**
2151              * Read string data from the following tags:
2152              * - tag_filename
2153              * - tag_artist
2154              * - tag_album
2155              * - tag_title
2156              *
2157              * A crc32 hash is calculated from the read data
2158              * and stored back to the data offset field kept in memory.
2159              */
2160 #define tmpdb_read_string_tag(tag) \
2161     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2162     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2163     { \
2164         logf("read fail: buffer overflow"); \
2165         close(masterfd); \
2166         return false; \
2167     } \
2168     \
2169     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2170         tfe->tag_length[tag]) \
2171     { \
2172         logf("read fail #2"); \
2173         close(masterfd); \
2174         return false; \
2175     } \
2176     \
2177     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2178     lseek(tmpfd, datastart, SEEK_SET)
2179
2180             tmpdb_read_string_tag(tag_filename);
2181             tmpdb_read_string_tag(tag_artist);
2182             tmpdb_read_string_tag(tag_album);
2183             tmpdb_read_string_tag(tag_title);
2184
2185             /* Seek to the end of the string data. */
2186             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2187         }
2188
2189         /* Backup the master index position. */
2190         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2191         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2192
2193         /* Check if we can resurrect some deleted runtime statistics data. */
2194         for (i = 0; i < tcmh.tch.entry_count; i++)
2195         {
2196             /* Read the index entry. */
2197             if (ecread_index_entry(masterfd, &idx)
2198                 != sizeof(struct index_entry))
2199             {
2200                 logf("read fail #3");
2201                 close(masterfd);
2202                 return false;
2203             }
2204
2205             /**
2206              * Skip unless the entry is marked as being deleted
2207              * or the data has already been resurrected.
2208              */
2209             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2210                 continue;
2211
2212             /* Now try to match the entry. */
2213             /**
2214              * To succesfully match a song, the following conditions
2215              * must apply:
2216              *
2217              * For numeric fields: tag_length
2218              * - Full identical match is required
2219              *
2220              * If tag_filename matches, no further checking necessary.
2221              *
2222              * For string hashes: tag_artist, tag_album, tag_title
2223              * - Two of these must match
2224              */
2225             for (j = 0; j < count; j++)
2226             {
2227                 struct temp_file_entry *tfe = &entrybuf[j];
2228
2229                 /* Try to match numeric fields first. */
2230                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2231                     continue;
2232
2233                 /* Now it's time to do the hash matching. */
2234                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2235                 {
2236                     int match_count = 0;
2237
2238                     /* No filename match, check if we can match two other tags. */
2239 #define tmpdb_match(tag) \
2240     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2241         match_count++
2242
2243                     tmpdb_match(tag_artist);
2244                     tmpdb_match(tag_album);
2245                     tmpdb_match(tag_title);
2246
2247                     if (match_count < 2)
2248                     {
2249                         /* Still no match found, give up. */
2250                         continue;
2251                     }
2252                 }
2253
2254                 /* A match found, now copy & resurrect the statistical data. */
2255 #define tmpdb_copy_tag(tag) \
2256     tfe->tag_offset[tag] = idx.tag_seek[tag]
2257
2258                 tmpdb_copy_tag(tag_playcount);
2259                 tmpdb_copy_tag(tag_rating);
2260                 tmpdb_copy_tag(tag_playtime);
2261                 tmpdb_copy_tag(tag_lastplayed);
2262                 tmpdb_copy_tag(tag_commitid);
2263
2264                 /* Avoid processing this entry again. */
2265                 idx.flag |= FLAG_RESURRECTED;
2266
2267                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2268                 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2269                 {
2270                     logf("masterfd writeback fail #1");
2271                     close(masterfd);
2272                     return false;
2273                 }
2274
2275                 logf("Entry resurrected");
2276             }
2277         }
2278
2279
2280         /* Restore the master index position. */
2281         lseek(masterfd, masterfd_pos, SEEK_SET);
2282
2283         /* Commit the data to the index. */
2284         for (i = 0; i < count; i++)
2285         {
2286             int loc = lseek(masterfd, 0, SEEK_CUR);
2287
2288             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2289             {
2290                 logf("read fail #3");
2291                 close(masterfd);
2292                 return false;
2293             }
2294
2295             for (j = 0; j < TAG_COUNT; j++)
2296             {
2297                 if (!TAGCACHE_IS_NUMERIC(j))
2298                     continue;
2299
2300                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2301             }
2302             idx.flag = entrybuf[i].flag;
2303
2304             if (idx.tag_seek[tag_commitid])
2305             {
2306                 /* Data has been resurrected. */
2307                 idx.flag |= FLAG_DIRTYNUM;
2308             }
2309             else if (tc_stat.ready && current_tcmh.commitid > 0)
2310             {
2311                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2312                 idx.flag |= FLAG_DIRTYNUM;
2313             }
2314
2315             /* Write back the updated index. */
2316             lseek(masterfd, loc, SEEK_SET);
2317             if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2318             {
2319                 logf("write fail");
2320                 close(masterfd);
2321                 return false;
2322             }
2323         }
2324
2325         entries_processed += count;
2326         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2327     }
2328
2329     close(masterfd);
2330
2331     return true;
2332 }
2333
2334 /**
2335  * Return values:
2336  *     > 0   success
2337  *    == 0   temporary failure
2338  *     < 0   fatal error
2339  */
2340 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2341 {
2342     int i;
2343     struct tagcache_header tch;
2344     struct master_header   tcmh;
2345     struct index_entry idxbuf[IDX_BUF_DEPTH];
2346     int idxbuf_pos;
2347     char buf[TAG_MAXLEN+32];
2348     int fd = -1, masterfd;
2349     bool error = false;
2350     int init;
2351     int masterfd_pos;
2352
2353     logf("Building index: %d", index_type);
2354
2355     /* Check the number of entries we need to allocate ram for. */
2356     commit_entry_count = h->entry_count + 1;
2357
2358     masterfd = open_master_fd(&tcmh, false);
2359     if (masterfd >= 0)
2360     {
2361         commit_entry_count += tcmh.tch.entry_count;
2362         close(masterfd);
2363     }
2364     else
2365         remove_files(); /* Just to be sure we are clean. */
2366
2367     /* Open the index file, which contains the tag names. */
2368     fd = open_tag_fd(&tch, index_type, true);
2369     if (fd >= 0)
2370     {
2371         logf("tch.datasize=%ld", tch.datasize);
2372         lookup_buffer_depth = 1 +
2373         /* First part */ commit_entry_count +
2374         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2375     }
2376     else
2377     {
2378         lookup_buffer_depth = 1 +
2379         /* First part */ commit_entry_count +
2380         /* Second part */ 0;
2381     }
2382
2383     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2384     logf("commit_entry_count=%ld", commit_entry_count);
2385
2386     /* Allocate buffer for all index entries from both old and new
2387      * tag files. */
2388     tempbufidx = 0;
2389     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2390
2391     /* Allocate lookup buffer. The first portion of commit_entry_count
2392      * contains the new tags in the temporary file and the second
2393      * part for locating entries already in the db.
2394      *
2395      *  New tags  Old tags
2396      * +---------+---------------------------+
2397      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2398      * +---------+---------------------------+
2399      *
2400      * Old tags are inserted to a temporary buffer with position:
2401      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2402      * And new tags with index:
2403      *     tempbuf_insert(idx, ...);
2404      *
2405      * The buffer is sorted and written into tag file:
2406      *     tempbuf_sort(...);
2407      * leaving master index locations messed up.
2408      *
2409      * That is fixed using the lookup buffer for old tags:
2410      *     new_seek = tempbuf_find_location(old_seek, ...);
2411      * and for new tags:
2412      *     new_seek = tempbuf_find_location(idx);
2413      */
2414     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2415     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2416     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2417
2418     /* And calculate the remaining data space used mainly for storing
2419      * tag data (strings). */
2420     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2421     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2422     {
2423         logf("Buffer way too small!");
2424         return 0;
2425     }
2426
2427     if (fd >= 0)
2428     {
2429         /**
2430          * If tag file contains unique tags (sorted index), we will load
2431          * it entirely into memory so we can resort it later for use with
2432          * chunked browsing.
2433          */
2434         if (TAGCACHE_IS_SORTED(index_type))
2435         {
2436             logf("loading tags...");
2437             for (i = 0; i < tch.entry_count; i++)
2438             {
2439                 struct tagfile_entry entry;
2440                 int loc = lseek(fd, 0, SEEK_CUR);
2441                 bool ret;
2442
2443                 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2444                 {
2445                     logf("read error #7");
2446                     close(fd);
2447                     return -2;
2448                 }
2449
2450                 if (entry.tag_length >= (int)sizeof(buf))
2451                 {
2452                     logf("too long tag #3");
2453                     close(fd);
2454                     return -2;
2455                 }
2456
2457                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2458                 {
2459                     logf("read error #8");
2460                     close(fd);
2461                     return -2;
2462                 }
2463
2464                 /* Skip deleted entries. */
2465                 if (buf[0] == '\0')
2466                     continue;
2467
2468                 /**
2469                  * Save the tag and tag id in the memory buffer. Tag id
2470                  * is saved so we can later reindex the master lookup
2471                  * table when the index gets resorted.
2472                  */
2473                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2474                                      + commit_entry_count, entry.idx_id,
2475                                      TAGCACHE_IS_UNIQUE(index_type));
2476                 if (!ret)
2477                 {
2478                     close(fd);
2479                     return -3;
2480                 }
2481                 do_timed_yield();
2482             }
2483             logf("done");
2484         }
2485         else
2486             tempbufidx = tch.entry_count;
2487     }
2488     else
2489     {
2490         /**
2491          * Creating new index file to store the tags. No need to preload
2492          * anything whether the index type is sorted or not.
2493          */
2494         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2495         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2496         if (fd < 0)
2497         {
2498             logf("%s open fail", buf);
2499             return -2;
2500         }
2501
2502         tch.magic = TAGCACHE_MAGIC;
2503         tch.entry_count = 0;
2504         tch.datasize = 0;
2505
2506         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2507             != sizeof(struct tagcache_header))
2508         {
2509             logf("header write failed");
2510             close(fd);
2511             return -2;
2512         }
2513     }
2514
2515     /* Loading the tag lookup file as "master file". */
2516     logf("Loading index file");
2517     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2518
2519     if (masterfd < 0)
2520     {
2521         logf("Creating new DB");
2522         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2523
2524         if (masterfd < 0)
2525         {
2526             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2527             close(fd);
2528             return -2;
2529         }
2530
2531         /* Write the header (write real values later). */
2532         memset(&tcmh, 0, sizeof(struct master_header));
2533         tcmh.tch = *h;
2534         tcmh.tch.entry_count = 0;
2535         tcmh.tch.datasize = 0;
2536         tcmh.dirty = true;
2537         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2538         init = true;
2539         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2540     }
2541     else
2542     {
2543         /**
2544          * Master file already exists so we need to process the current
2545          * file first.
2546          */
2547         init = false;
2548
2549         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2550             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2551         {
2552             logf("header error");
2553             close(fd);
2554             close(masterfd);
2555             return -2;
2556         }
2557
2558         /**
2559          * If we reach end of the master file, we need to expand it to
2560          * hold new tags. If the current index is not sorted, we can
2561          * simply append new data to end of the file.
2562          * However, if the index is sorted, we need to update all tag
2563          * pointers in the master file for the current index.
2564          */
2565         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2566             SEEK_CUR);
2567         if (masterfd_pos == filesize(masterfd))
2568         {
2569             logf("appending...");
2570             init = true;
2571         }
2572     }
2573
2574     /**
2575      * Load new unique tags in memory to be sorted later and added
2576      * to the master lookup file.
2577      */
2578     if (TAGCACHE_IS_SORTED(index_type))
2579     {
2580         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2581         /* h is the header of the temporary file containing new tags. */
2582         logf("inserting new tags...");
2583         for (i = 0; i < h->entry_count; i++)
2584         {
2585             struct temp_file_entry entry;
2586
2587             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2588                 sizeof(struct temp_file_entry))
2589             {
2590                 logf("read fail #3");
2591                 error = true;
2592                 goto error_exit;
2593             }
2594
2595             /* Read data. */
2596             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2597             {
2598                 logf("too long entry!");
2599                 error = true;
2600                 goto error_exit;
2601             }
2602
2603             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2604             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2605                 entry.tag_length[index_type])
2606             {
2607                 logf("read fail #4");
2608                 error = true;
2609                 goto error_exit;
2610             }
2611
2612             if (TAGCACHE_IS_UNIQUE(index_type))
2613                 error = !tempbuf_insert(buf, i, -1, true);
2614             else
2615                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2616
2617             if (error)
2618             {
2619                 logf("insert error");
2620                 goto error_exit;
2621             }
2622
2623             /* Skip to next. */
2624             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2625                     entry.tag_length[index_type], SEEK_CUR);
2626             do_timed_yield();
2627         }
2628         logf("done");
2629
2630         /* Sort the buffer data and write it to the index file. */
2631         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2632         /**
2633          * We need to truncate the index file now. There can be junk left
2634          * at the end of file (however, we _should_ always follow the
2635          * entry_count and don't crash with that).
2636          */
2637         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2638
2639         i = tempbuf_sort(fd);
2640         if (i < 0)
2641             goto error_exit;
2642         logf("sorted %d tags", i);
2643
2644         /**
2645          * Now update all indexes in the master lookup file.
2646          */
2647         logf("updating indices...");
2648         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2649         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2650         {
2651             int j;
2652             int loc = lseek(masterfd, 0, SEEK_CUR);
2653
2654             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2655
2656             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2657                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2658             {
2659                 logf("read fail #5");
2660                 error = true;
2661                 goto error_exit ;
2662             }
2663             lseek(masterfd, loc, SEEK_SET);
2664
2665             for (j = 0; j < idxbuf_pos; j++)
2666             {
2667                 if (idxbuf[j].flag & FLAG_DELETED)
2668                 {
2669                     /* We can just ignore deleted entries. */
2670                     // idxbuf[j].tag_seek[index_type] = 0;
2671                     continue;
2672                 }
2673
2674                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2675                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2676                     + commit_entry_count);
2677
2678                 if (idxbuf[j].tag_seek[index_type] < 0)
2679                 {
2680                     logf("update error: %ld/%d/%ld",
2681                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2682                     error = true;
2683                     goto error_exit;
2684                 }
2685
2686                 do_timed_yield();
2687             }
2688
2689             /* Write back the updated index. */
2690             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2691                         index_entry_ec, tc_stat.econ) !=
2692                 (int)sizeof(struct index_entry)*idxbuf_pos)
2693             {
2694                 logf("write fail");
2695                 error = true;
2696                 goto error_exit;
2697             }
2698         }
2699         logf("done");
2700     }
2701
2702     /**
2703      * Walk through the temporary file containing the new tags.
2704      */
2705     // build_normal_index(h, tmpfd, masterfd, idx);
2706     logf("updating new indices...");
2707     lseek(masterfd, masterfd_pos, SEEK_SET);
2708     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2709     lseek(fd, 0, SEEK_END);
2710     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2711     {
2712         int j;
2713
2714         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2715         if (init)
2716         {
2717             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2718         }
2719         else
2720         {
2721             int loc = lseek(masterfd, 0, SEEK_CUR);
2722
2723             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2724                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2725             {
2726                 logf("read fail #6");
2727                 error = true;
2728                 break ;
2729             }
2730             lseek(masterfd, loc, SEEK_SET);
2731         }
2732
2733         /* Read entry headers. */
2734         for (j = 0; j < idxbuf_pos; j++)
2735         {
2736             if (!TAGCACHE_IS_SORTED(index_type))
2737             {
2738                 struct temp_file_entry entry;
2739                 struct tagfile_entry fe;
2740
2741                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2742                     sizeof(struct temp_file_entry))
2743                 {
2744                     logf("read fail #7");
2745                     error = true;
2746                     break ;
2747                 }
2748
2749                 /* Read data. */
2750                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2751                 {
2752                     logf("too long entry!");
2753                     logf("length=%d", entry.tag_length[index_type]);
2754                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2755                     error = true;
2756                     break ;
2757                 }
2758
2759                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2760                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2761                     entry.tag_length[index_type])
2762                 {
2763                     logf("read fail #8");
2764                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2765                     logf("length=0x%02x", entry.tag_length[index_type]);
2766                     error = true;
2767                     break ;
2768                 }
2769
2770                 /* Write to index file. */
2771                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2772                 fe.tag_length = entry.tag_length[index_type];
2773                 fe.idx_id = tcmh.tch.entry_count + i + j;
2774                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2775                 write(fd, buf, fe.tag_length);
2776                 tempbufidx++;
2777
2778                 /* Skip to next. */
2779                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2780                       entry.tag_length[index_type], SEEK_CUR);
2781             }
2782             else
2783             {
2784                 /* Locate the correct entry from the sorted array. */
2785                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2786                 if (idxbuf[j].tag_seek[index_type] < 0)
2787                 {
2788                     logf("entry not found (%d)", j);
2789                     error = true;
2790                     break ;
2791                 }
2792             }
2793         }
2794
2795         /* Write index. */
2796         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2797                   index_entry_ec, tc_stat.econ) !=
2798             (int)sizeof(struct index_entry)*idxbuf_pos)
2799         {
2800             logf("tagcache: write fail #4");
2801             error = true;
2802             break ;
2803         }
2804
2805         do_timed_yield();
2806     }
2807     logf("done");
2808
2809     /* Finally write the header. */
2810     tch.magic = TAGCACHE_MAGIC;
2811     tch.entry_count = tempbufidx;
2812     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2813     lseek(fd, 0, SEEK_SET);
2814     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2815
2816     if (index_type != tag_filename)
2817         h->datasize += tch.datasize;
2818     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2819     error_exit:
2820
2821     close(fd);
2822     close(masterfd);
2823
2824     if (error)
2825         return -2;
2826
2827     return 1;
2828 }
2829
2830 static bool commit(void)
2831 {
2832     struct tagcache_header tch;
2833     struct master_header   tcmh;
2834     int i, len, rc;
2835     int tmpfd;
2836     int masterfd;
2837 #ifdef HAVE_DIRCACHE
2838     bool dircache_buffer_stolen = false;
2839 #endif
2840     bool local_allocation = false;
2841
2842     logf("committing tagcache");
2843
2844     while (write_lock)
2845         sleep(1);
2846
2847     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2848     if (tmpfd < 0)
2849     {
2850         logf("nothing to commit");
2851         return true;
2852     }
2853
2854
2855     /* Load the header. */
2856     len = sizeof(struct tagcache_header);
2857     rc = read(tmpfd, &tch, len);
2858
2859     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2860     {
2861         logf("incorrect header");
2862         close(tmpfd);
2863         remove(TAGCACHE_FILE_TEMP);
2864         return false;
2865     }
2866
2867     if (tch.entry_count == 0)
2868     {
2869         logf("nothing to commit");
2870         close(tmpfd);
2871         remove(TAGCACHE_FILE_TEMP);
2872         return true;
2873     }
2874
2875 #ifdef HAVE_EEPROM_SETTINGS
2876     remove(TAGCACHE_STATEFILE);
2877 #endif
2878
2879     /* At first be sure to unload the ramcache! */
2880 #ifdef HAVE_TC_RAMCACHE
2881     tc_stat.ramcache = false;
2882 #endif
2883
2884     read_lock++;
2885
2886     /* Try to steal every buffer we can :) */
2887     if (tempbuf_size == 0)
2888         local_allocation = true;
2889
2890 #ifdef HAVE_DIRCACHE
2891     if (tempbuf_size == 0)
2892     {
2893         /* Try to steal the dircache buffer. */
2894         tempbuf = dircache_steal_buffer(&tempbuf_size);
2895         tempbuf_size &= ~0x03;
2896
2897         if (tempbuf_size > 0)
2898         {
2899             dircache_buffer_stolen = true;
2900         }
2901     }
2902 #endif
2903
2904 #ifdef HAVE_TC_RAMCACHE
2905     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2906     {
2907         tempbuf = (char *)(hdr + 1);
2908         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2909         tempbuf_size &= ~0x03;
2910     }
2911 #endif
2912
2913     /* And finally fail if there are no buffers available. */
2914     if (tempbuf_size == 0)
2915     {
2916         logf("delaying commit until next boot");
2917         tc_stat.commit_delayed = true;
2918         close(tmpfd);
2919         read_lock--;
2920         return false;
2921     }
2922
2923     logf("commit %ld entries...", tch.entry_count);
2924
2925     /* Mark DB dirty so it will stay disabled if commit fails. */
2926     current_tcmh.dirty = true;
2927     update_master_header();
2928
2929     /* Now create the index files. */
2930     tc_stat.commit_step = 0;
2931     tch.datasize = 0;
2932     tc_stat.commit_delayed = false;
2933
2934     for (i = 0; i < TAG_COUNT; i++)
2935     {
2936         int ret;
2937
2938         if (TAGCACHE_IS_NUMERIC(i))
2939             continue;
2940
2941         tc_stat.commit_step++;
2942         ret = build_index(i, &tch, tmpfd);
2943         if (ret <= 0)
2944         {
2945             close(tmpfd);
2946             logf("tagcache failed init");
2947             if (ret == 0)
2948                 tc_stat.commit_delayed = true;
2949
2950             tc_stat.commit_step = 0;
2951             read_lock--;
2952             return false;
2953         }
2954     }
2955
2956     if (!build_numeric_indices(&tch, tmpfd))
2957     {
2958         logf("Failure to commit numeric indices");
2959         close(tmpfd);
2960         tc_stat.commit_step = 0;
2961         read_lock--;
2962         return false;
2963     }
2964
2965     close(tmpfd);
2966     remove(TAGCACHE_FILE_TEMP);
2967
2968     tc_stat.commit_step = 0;
2969
2970     /* Update the master index headers. */
2971     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2972     {
2973         read_lock--;
2974         return false;
2975     }
2976
2977     tcmh.tch.entry_count += tch.entry_count;
2978     tcmh.tch.datasize = sizeof(struct master_header)
2979         + sizeof(struct index_entry) * tcmh.tch.entry_count
2980         + tch.datasize;
2981     tcmh.dirty = false;
2982     tcmh.commitid++;
2983
2984     lseek(masterfd, 0, SEEK_SET);
2985     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2986     close(masterfd);
2987
2988     logf("tagcache committed");
2989     tc_stat.ready = check_all_headers();
2990     tc_stat.readyvalid = true;
2991
2992     if (local_allocation)
2993     {
2994         tempbuf = NULL;
2995         tempbuf_size = 0;
2996     }
2997
2998 #ifdef HAVE_DIRCACHE
2999     /* Rebuild the dircache, if we stole the buffer. */
3000     if (dircache_buffer_stolen)
3001         dircache_build(0);
3002 #endif
3003
3004 #ifdef HAVE_TC_RAMCACHE
3005     /* Reload tagcache. */
3006     if (tc_stat.ramcache_allocated > 0)
3007         tagcache_start_scan();
3008 #endif
3009
3010     read_lock--;
3011
3012     return true;
3013 }
3014
3015 static void allocate_tempbuf(void)
3016 {
3017     /* Yeah, malloc would be really nice now :) */
3018 #ifdef __PCTOOL__
3019     tempbuf_size = 32*1024*1024;
3020     tempbuf = malloc(tempbuf_size);
3021 #else
3022     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3023     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3024     audiobuf += tempbuf_size;
3025 #endif
3026 }
3027
3028 static void free_tempbuf(void)
3029 {
3030     if (tempbuf_size == 0)
3031         return ;
3032
3033 #ifdef __PCTOOL__
3034     free(tempbuf);
3035 #else
3036     audiobuf -= tempbuf_size;
3037 #endif
3038     tempbuf = NULL;
3039     tempbuf_size = 0;
3040 }
3041
3042 #ifndef __PCTOOL__
3043
3044 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3045 {
3046     struct index_entry idx;
3047
3048     if (!tc_stat.ready)
3049         return false;
3050
3051     if (!TAGCACHE_IS_NUMERIC(tag))
3052         return false;
3053
3054     if (!get_index(masterfd, idx_id, &idx, false))
3055         return false;
3056
3057     idx.tag_seek[tag] = data;
3058     idx.flag |= FLAG_DIRTYNUM;
3059
3060     return write_index(masterfd, idx_id, &idx);
3061 }
3062
3063 #if 0
3064 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3065                                    int tag, long data)
3066 {
3067     struct master_header myhdr;
3068
3069     if (tcs->masterfd < 0)
3070     {
3071         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3072             return false;
3073     }
3074
3075     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3076 }
3077 #endif
3078
3079 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3080
3081 static bool command_queue_is_full(void)
3082 {
3083     int next;
3084
3085     next = command_queue_widx + 1;
3086     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3087         next = 0;
3088
3089     return (next == command_queue_ridx);
3090 }
3091
3092 static void command_queue_sync_callback(void *data)
3093 {
3094     (void)data;
3095     struct master_header myhdr;
3096     int masterfd;
3097
3098     mutex_lock(&command_queue_mutex);
3099
3100     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3101         return;
3102
3103     while (command_queue_ridx != command_queue_widx)
3104     {
3105         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3106
3107         switch (ce->command)
3108         {
3109             case CMD_UPDATE_MASTER_HEADER:
3110             {
3111                 close(masterfd);
3112                 update_master_header();
3113
3114                 /* Re-open the masterfd. */
3115                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3116                     return;
3117
3118                 break;
3119             }
3120             case CMD_UPDATE_NUMERIC:
3121             {
3122                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3123                 break;
3124             }
3125         }
3126
3127         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3128             command_queue_ridx = 0;
3129     }
3130
3131     close(masterfd);
3132
3133     tc_stat.queue_length = 0;
3134     mutex_unlock(&command_queue_mutex);
3135 }
3136
3137 static void run_command_queue(bool force)
3138 {
3139     if (COMMAND_QUEUE_IS_EMPTY)
3140         return;
3141
3142     if (force || command_queue_is_full())
3143         command_queue_sync_callback(NULL);
3144     else
3145         register_storage_idle_func(command_queue_sync_callback);
3146 }
3147
3148 static void queue_command(int cmd, long idx_id, int tag, long data)
3149 {
3150     while (1)
3151     {
3152         int next;
3153
3154         mutex_lock(&command_queue_mutex);
3155         next = command_queue_widx + 1;
3156         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3157             next = 0;
3158
3159         /* Make sure queue is not full. */
3160         if (next != command_queue_ridx)
3161         {
3162             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3163
3164             ce->command = cmd;
3165             ce->idx_id = idx_id;
3166             ce->tag = tag;
3167             ce->data = data;
3168
3169             command_queue_widx = next;
3170
3171             tc_stat.queue_length++;
3172
3173             mutex_unlock(&command_queue_mutex);
3174             break;
3175         }
3176
3177         /* Queue is full, try again later... */
3178         mutex_unlock(&command_queue_mutex);
3179         sleep(1);
3180     }
3181 }
3182
3183 long tagcache_increase_serial(void)
3184 {
3185     long old;
3186
3187     if (!tc_stat.ready)
3188         return -2;
3189
3190     while (read_lock)
3191         sleep(1);
3192
3193     old = current_tcmh.serial++;
3194     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3195
3196     return old;
3197 }
3198
3199 void tagcache_update_numeric(int idx_id, int tag, long data)
3200 {
3201     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3202 }
3203 #endif /* !__PCTOOL__ */
3204
3205 long tagcache_get_serial(void)
3206 {
3207     return current_tcmh.serial;
3208 }
3209
3210 long tagcache_get_commitid(void)
3211 {
3212     return current_tcmh.commitid;
3213 }
3214
3215 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3216 {
3217     char buf[512];
3218     int i;
3219
3220     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3221     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3222     {
3223         if (*datastr == '\0')
3224             break;
3225
3226         if (*datastr == '"' || *datastr == '\\')
3227             buf[i++] = '\\';
3228
3229         buf[i] = *(datastr++);
3230     }
3231
3232     strcpy(&buf[i], "\" ");
3233
3234     write(fd, buf, i + 2);
3235
3236     return true;
3237 }
3238
3239 #ifndef __PCTOOL__
3240
3241 static bool read_tag(char *dest, long size,
3242                      const char *src, const char *tagstr)
3243 {
3244     int pos;
3245     char current_tag[32];
3246
3247     while (*src != '\0')
3248     {
3249         /* Skip all whitespace */
3250         while (*src == ' ')
3251             src++;
3252
3253         if (*src == '\0')
3254             break;
3255
3256         pos = 0;
3257         /* Read in tag name */
3258         while (*src != '=' && *src != ' ')
3259         {
3260             current_tag[pos] = *src;
3261             src++;
3262             pos++;
3263
3264             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3265                 return false;
3266         }
3267         current_tag[pos] = '\0';
3268
3269         /* Read in tag data */
3270
3271         /* Find the start. */
3272         while (*src != '"' && *src != '\0')
3273             src++;
3274
3275         if (*src == '\0' || *(++src) == '\0')
3276             return false;
3277
3278         /* Read the data. */
3279         for (pos = 0; pos < size; pos++)
3280         {
3281             if (*src == '\0')
3282                 break;
3283
3284             if (*src == '\\')
3285             {
3286                 dest[pos] = *(src+1);
3287                 src += 2;
3288                 continue;
3289             }
3290
3291             dest[pos] = *src;
3292
3293             if (*src == '"')
3294             {
3295                 src++;
3296                 break;
3297             }
3298
3299             if (*src == '\0')
3300                 break;
3301
3302             src++;
3303         }
3304         dest[pos] = '\0';
3305
3306         if (!strcasecmp(tagstr, current_tag))
3307             return true;
3308     }
3309
3310     return false;
3311 }
3312
3313 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3314 {
3315     struct index_entry idx;
3316     char tag_data[TAG_MAXLEN+32];
3317     int idx_id;
3318     long masterfd = (long)parameters;
3319     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3320         tag_commitid };
3321     int i;
3322     (void)line_n;
3323
3324     if (*buf == '#')
3325         return 0;
3326
3327     logf("%d/%s", line_n, buf);
3328     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3329     {
3330         logf("filename missing");
3331         logf("-> %s", buf);
3332         return 0;
3333     }
3334
3335     idx_id = find_index(tag_data);
3336     if (idx_id < 0)
3337     {
3338         logf("entry not found");
3339         return 0;
3340     }
3341
3342     if (!get_index(masterfd, idx_id, &idx, false))
3343     {
3344         logf("failed to retrieve index entry");
3345         return 0;
3346     }
3347
3348     /* Stop if tag has already been modified. */
3349     if (idx.flag & FLAG_DIRTYNUM)
3350         return 0;
3351
3352     logf("import: %s", tag_data);
3353
3354     idx.flag |= FLAG_DIRTYNUM;
3355     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3356     {
3357         int data;
3358
3359         if (!read_tag(tag_data, sizeof tag_data, buf,
3360                       tagcache_tag_to_str(import_tags[i])))
3361         {
3362             continue;
3363         }
3364
3365         data = atoi(tag_data);
3366         if (data < 0)
3367             continue;
3368
3369         idx.tag_seek[import_tags[i]] = data;
3370
3371         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3372             current_tcmh.serial = data + 1;
3373         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3374             current_tcmh.commitid = data + 1;
3375     }
3376
3377     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3378 }
3379
3380 bool tagcache_import_changelog(void)
3381 {
3382     struct master_header myhdr;
3383     struct tagcache_header tch;
3384     int clfd;
3385     long masterfd;
3386     char buf[2048];
3387
3388     if (!tc_stat.ready)
3389         return false;
3390
3391     while (read_lock)
3392         sleep(1);
3393
3394     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3395     if (clfd < 0)
3396     {
3397         logf("failure to open changelog");
3398         return false;
3399     }
3400
3401     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3402     {
3403         close(clfd);
3404         return false;
3405     }
3406
3407     write_lock++;
3408
3409     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3410
3411     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3412                   parse_changelog_line);
3413
3414     close(clfd);
3415     close(masterfd);
3416
3417     if (filenametag_fd >= 0)
3418     {
3419         close(filenametag_fd);
3420         filenametag_fd = -1;
3421     }
3422
3423     write_lock--;
3424
3425     update_master_header();
3426
3427     return true;
3428 }
3429
3430 #endif /* !__PCTOOL__ */
3431
3432 bool tagcache_create_changelog(struct tagcache_search *tcs)
3433 {
3434     struct master_header myhdr;
3435     struct index_entry idx;
3436     char buf[TAG_MAXLEN+32];
3437     char temp[32];
3438     int clfd;
3439     int i, j;
3440
3441     if (!tc_stat.ready)
3442         return false;
3443
3444     if (!tagcache_search(tcs, tag_filename))
3445         return false;
3446
3447     /* Initialize the changelog */
3448     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3449     if (clfd < 0)
3450     {
3451         logf("failure to open changelog");
3452         return false;
3453     }
3454
3455     if (tcs->masterfd < 0)
3456     {
3457         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3458             return false;
3459     }
3460     else
3461     {
3462         lseek(tcs->masterfd, 0, SEEK_SET);
3463         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3464     }
3465
3466     write(clfd, "## Changelog version 1\n", 23);
3467
3468     for (i = 0; i < myhdr.tch.entry_count; i++)
3469     {
3470         if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3471         {
3472             logf("read error #9");
3473             tagcache_search_finish(tcs);
3474             close(clfd);
3475             return false;
3476         }
3477
3478         /* Skip until the entry found has been modified. */
3479         if (! (idx.flag & FLAG_DIRTYNUM) )
3480             continue;
3481
3482         /* Skip deleted entries too. */
3483         if (idx.flag & FLAG_DELETED)
3484             continue;
3485
3486         /* Now retrieve all tags. */
3487         for (j = 0; j < TAG_COUNT; j++)
3488         {
3489             if (TAGCACHE_IS_NUMERIC(j))
3490             {
3491                 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3492                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3493                 continue;
3494             }
3495
3496             tcs->type = j;
3497             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3498             write_tag(clfd, tagcache_tag_to_str(j), buf);
3499         }
3500
3501         write(clfd, "\n", 1);
3502         do_timed_yield();
3503     }
3504
3505     close(clfd);
3506
3507     tagcache_search_finish(tcs);
3508
3509     return true;
3510 }
3511
3512 static bool delete_entry(long idx_id)
3513 {
3514     int fd = -1;
3515     int masterfd = -1;
3516     int tag, i;
3517     struct index_entry idx, myidx;
3518     struct master_header myhdr;
3519     char buf[TAG_MAXLEN+32];
3520     int in_use[TAG_COUNT];
3521
3522     logf("delete_entry(): %ld", idx_id);
3523
3524 #ifdef HAVE_TC_RAMCACHE
3525     /* At first mark the entry removed from ram cache. */
3526     if (tc_stat.ramcache)
3527         hdr->indices[idx_id].flag |= FLAG_DELETED;
3528 #endif
3529
3530     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3531         return false;
3532
3533     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3534     if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3535     {
3536         logf("delete_entry(): read error");
3537         goto cleanup;
3538     }
3539
3540     if (myidx.flag & FLAG_DELETED)
3541     {
3542         logf("delete_entry(): already deleted!");
3543         goto cleanup;
3544     }
3545
3546     myidx.flag |= FLAG_DELETED;
3547     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3548     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3549     {
3550         logf("delete_entry(): write_error #1");
3551         goto cleanup;
3552     }
3553
3554     /* Now check which tags are no longer in use (if any) */
3555     for (tag = 0; tag < TAG_COUNT; tag++)
3556         in_use[tag] = 0;
3557
3558     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3559     for (i = 0; i < myhdr.tch.entry_count; i++)
3560     {
3561         struct index_entry *idxp;
3562
3563 #ifdef HAVE_TC_RAMCACHE
3564         /* Use RAM DB if available for greater speed */
3565         if (tc_stat.ramcache)
3566             idxp = &hdr->indices[i];
3567         else
3568 #endif
3569         {
3570             if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3571             {
3572                 logf("delete_entry(): read error #2");
3573                 goto cleanup;
3574             }
3575             idxp = &idx;
3576         }
3577
3578         if (idxp->flag & FLAG_DELETED)
3579             continue;
3580
3581         for (tag = 0; tag < TAG_COUNT; tag++)
3582         {
3583             if (TAGCACHE_IS_NUMERIC(tag))
3584                 continue;
3585
3586             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3587                 in_use[tag]++;
3588         }
3589     }
3590
3591     /* Now delete all tags no longer in use. */
3592     for (tag = 0; tag < TAG_COUNT; tag++)
3593     {
3594         struct tagcache_header tch;
3595         int oldseek = myidx.tag_seek[tag];
3596
3597         if (TAGCACHE_IS_NUMERIC(tag))
3598             continue;
3599
3600         /**
3601          * Replace tag seek with a hash value of the field string data.
3602          * That way runtime statistics of moved or altered files can be
3603          * resurrected.
3604          */
3605 #ifdef HAVE_TC_RAMCACHE
3606         if (tc_stat.ramcache && tag != tag_filename)
3607         {
3608             struct tagfile_entry *tfe;
3609             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3610
3611             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3612             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3613             myidx.tag_seek[tag] = *seek;
3614         }
3615         else
3616 #endif
3617         {
3618             struct tagfile_entry tfe;
3619
3620             /* Open the index file, which contains the tag names. */
3621             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3622                 goto cleanup;
3623
3624             /* Skip the header block */
3625             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3626             if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3627             {
3628                 logf("delete_entry(): read error #3");
3629                 goto cleanup;
3630             }
3631
3632             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3633             {
3634                 logf("delete_entry(): read error #4");
3635                 goto cleanup;
3636             }
3637
3638             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3639         }
3640
3641         if (in_use[tag])
3642         {
3643             logf("in use: %d/%d", tag, in_use[tag]);
3644             if (fd >= 0)
3645             {
3646                 close(fd);
3647                 fd = -1;
3648             }
3649             continue;
3650         }
3651
3652 #ifdef HAVE_TC_RAMCACHE
3653         /* Delete from ram. */
3654         if (tc_stat.ramcache && tag != tag_filename)
3655         {
3656             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3657             tagentry->tag_data[0] = '\0';
3658         }
3659 #endif
3660
3661         /* Open the index file, which contains the tag names. */
3662         if (fd < 0)
3663         {
3664             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3665                 goto cleanup;
3666         }
3667
3668         /* Skip the header block */
3669         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3670
3671         /* Debug, print 10 first characters of the tag
3672         read(fd, buf, 10);
3673         buf[10]='\0';
3674         logf("TAG:%s", buf);
3675         lseek(fd, -10, SEEK_CUR);
3676         */
3677
3678         /* Write first data byte in tag as \0 */
3679         write(fd, "", 1);
3680
3681         /* Now tag data has been removed */
3682         close(fd);
3683         fd = -1;
3684     }
3685
3686     /* Write index entry back into master index. */
3687     lseek(masterfd, sizeof(struct master_header) +
3688           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3689     if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3690     {
3691         logf("delete_entry(): write_error #2");
3692         goto cleanup;
3693     }
3694
3695     close(masterfd);
3696
3697     return true;
3698
3699     cleanup:
3700     if (fd >= 0)
3701         close(fd);
3702     if (masterfd >= 0)
3703         close(masterfd);
3704
3705     return false;
3706 }
3707
3708 #ifndef __PCTOOL__
3709 /**
3710  * Returns true if there is an event waiting in the queue
3711  * that requires the current operation to be aborted.
3712  */
3713 static bool check_event_queue(void)
3714 {
3715     struct queue_event ev;
3716
3717     if(!queue_peek(&tagcache_queue, &ev))
3718         return false;
3719
3720     switch (ev.id)
3721     {
3722         case Q_STOP_SCAN:
3723         case SYS_POWEROFF:
3724         case SYS_USB_CONNECTED:
3725             return true;
3726     }
3727
3728     return false;
3729 }
3730 #endif
3731
3732 #ifdef HAVE_TC_RAMCACHE
3733 static bool allocate_tagcache(void)
3734 {
3735     struct master_header tcmh;
3736     int fd;
3737
3738     /* Load the header. */
3739     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3740     {
3741         hdr = NULL;
3742         return false;
3743     }
3744
3745     close(fd);
3746
3747     /**
3748      * Now calculate the required cache size plus
3749      * some extra space for alignment fixes.
3750      */
3751     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3752         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3753     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3754     memset(hdr, 0, sizeof(struct ramcache_header));
3755     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3756     hdr->indices = (struct index_entry *)(hdr + 1);
3757     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3758
3759     return true;
3760 }
3761
3762 # ifdef HAVE_EEPROM_SETTINGS
3763 static bool tagcache_dumpload(void)
3764 {
3765     struct statefile_header shdr;
3766     int fd, rc;
3767     long offpos;
3768     int i;
3769
3770     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3771     if (fd < 0)
3772     {
3773         logf("no tagcache statedump");
3774         return false;
3775     }
3776
3777     /* Check the statefile memory placement */
3778     hdr = buffer_alloc(0);
3779     rc = read(fd, &shdr, sizeof(struct statefile_header));
3780     if (rc != sizeof(struct statefile_header)
3781         /* || (long)hdr != (long)shdr.hdr */)
3782     {
3783         logf("incorrect statefile");
3784         hdr = NULL;
3785         close(fd);
3786         return false;
3787     }
3788
3789     offpos = (long)hdr - (long)shdr.hdr;
3790
3791     /* Lets allocate real memory and load it */
3792     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3793     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3794     close(fd);
3795
3796     if (rc != shdr.tc_stat.ramcache_allocated)
3797     {
3798         logf("read failure!");
3799         hdr = NULL;
3800         return false;
3801     }
3802
3803     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3804
3805     /* Now fix the pointers */
3806     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3807     for (i = 0; i < TAG_COUNT; i++)
3808         hdr->tags[i] += offpos;
3809
3810     return true;
3811 }
3812
3813 static bool tagcache_dumpsave(void)
3814 {
3815     struct statefile_header shdr;
3816     int fd;
3817
3818     if (!tc_stat.ramcache)
3819         return false;
3820
3821     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3822     if (fd < 0)
3823     {
3824         logf("failed to create a statedump");
3825         return false;
3826     }
3827
3828     /* Create the header */
3829     shdr.hdr = hdr;
3830     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3831     write(fd, &shdr, sizeof(struct statefile_header));
3832
3833     /* And dump the data too */
3834     write(fd, hdr, tc_stat.ramcache_allocated);
3835     close(fd);
3836
3837     return true;
3838 }
3839 # endif
3840
3841 static bool load_tagcache(void)
3842 {
3843     struct tagcache_header *tch;
3844     long bytesleft = tc_stat.ramcache_allocated;
3845     struct index_entry *idx;
3846     int rc, fd;
3847     char *p;
3848     int i, tag;
3849
3850 # ifdef HAVE_DIRCACHE
3851     while (dircache_is_initializing())
3852         sleep(1);
3853
3854     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3855 # endif
3856
3857     logf("loading tagcache to ram...");
3858
3859     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3860     if (fd < 0)
3861     {
3862         logf("tagcache open failed");
3863         return false;
3864     }
3865
3866     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3867         != sizeof(struct master_header)
3868         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3869     {
3870         logf("incorrect header");
3871         return false;
3872     }
3873
3874     idx = hdr->indices;
3875
3876     /* Load the master index table. */
3877     for (i = 0; i < hdr->h.tch.entry_count; i++)
3878     {
3879         rc = ecread_index_entry(fd, idx);
3880         if (rc != sizeof(struct index_entry))
3881         {
3882             logf("read error #10");
3883             close(fd);
3884             return false;
3885         }
3886
3887         bytesleft -= sizeof(struct index_entry);
3888         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3889         {
3890             logf("too big tagcache.");
3891             close(fd);
3892             return false;
3893         }
3894
3895         idx++;
3896     }
3897
3898     close(fd);
3899
3900     /* Load the tags. */
3901     p = (char *)idx;
3902     for (tag = 0; tag < TAG_COUNT; tag++)
3903     {
3904         struct tagfile_entry *fe;
3905         char buf[TAG_MAXLEN+32];
3906
3907         if (TAGCACHE_IS_NUMERIC(tag))
3908             continue ;
3909
3910         //p = ((void *)p+1);
3911         p = (char *)((long)p & ~0x03) + 0x04;
3912         hdr->tags[tag] = p;
3913
3914         /* Check the header. */
3915         tch = (struct tagcache_header *)p;
3916         p += sizeof(struct tagcache_header);
3917
3918         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3919             return false;
3920
3921         for (hdr->entry_count[tag] = 0;
3922              hdr->entry_count[tag] < tch->entry_count;
3923              hdr->entry_count[tag]++)
3924         {
3925             long pos;
3926
3927             if (do_timed_yield())
3928             {
3929                 /* Abort if we got a critical event in queue */
3930                 if (check_event_queue())
3931                     return false;
3932             }
3933
3934             fe = (struct tagfile_entry *)p;
3935             pos = lseek(fd, 0, SEEK_CUR);
3936             rc = ecread_tagfile_entry(fd, fe);
3937             if (rc != sizeof(struct tagfile_entry))
3938             {
3939                 /* End of lookup table. */
3940                 logf("read error #11");
3941                 close(fd);
3942                 return false;
3943             }
3944
3945             /* We have a special handling for the filename tags. */
3946             if (tag == tag_filename)
3947             {
3948 # ifdef HAVE_DIRCACHE
3949                 const struct dircache_entry *dc;
3950 # endif
3951
3952                 // FIXME: This is wrong!
3953                 // idx = &hdr->indices[hdr->entry_count[i]];
3954                 idx = &hdr->indices[fe->idx_id];
3955
3956                 if (fe->tag_length >= (long)sizeof(buf)-1)
3957                 {
3958                     read(fd, buf, 10);
3959                     buf[10] = '\0';
3960                     logf("TAG:%s", buf);
3961                     logf("too long filename");
3962                     close(fd);
3963                     return false;
3964                 }
3965
3966                 rc = read(fd, buf, fe->tag_length);
3967                 if (rc != fe->tag_length)
3968                 {
3969                     logf("read error #12");
3970                     close(fd);
3971                     return false;
3972                 }
3973
3974                 /* Check if the entry has already been removed */
3975                 if (idx->flag & FLAG_DELETED)
3976                     continue;
3977
3978                 /* This flag must not be used yet. */
3979                 if (idx->flag & FLAG_DIRCACHE)
3980                 {
3981                     logf("internal error!");
3982                     close(fd);
3983                     return false;
3984                 }
3985
3986                 if (idx->tag_seek[tag] != pos)
3987                 {
3988                     logf("corrupt data structures!");
3989                     close(fd);
3990                     return false;
3991                 }
3992
3993 # ifdef HAVE_DIRCACHE
3994                 if (dircache_is_enabled())
3995                 {
3996                     dc = dircache_get_entry_ptr(buf);
3997                     if (dc == NULL)
3998                     {
3999                         logf("Entry no longer valid.");
4000                         logf("-> %s", buf);
4001                         if (global_settings.tagcache_autoupdate)
4002                             delete_entry(fe->idx_id);
4003                         continue ;
4004                     }
4005
4006                     idx->flag |= FLAG_DIRCACHE;
4007                     idx->tag_seek[tag_filename] = (long)dc;
4008                 }
4009                 else
4010 # endif
4011                 {
4012                     /* This will be very slow unless dircache is enabled
4013                        or target is flash based, but do it anyway for
4014                        consistency. */
4015                     /* Check if entry has been removed. */
4016                     if (global_settings.tagcache_autoupdate)
4017                     {
4018                         if (!file_exists(buf))
4019                         {
4020                             logf("Entry no longer valid.");
4021                             logf("-> %s", buf);
4022                             delete_entry(fe->idx_id);
4023                             continue;
4024                         }
4025                     }
4026                 }
4027
4028                 continue ;
4029             }
4030
4031             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4032             if (bytesleft < 0)
4033             {
4034                 logf("too big tagcache #2");
4035                 logf("tl: %ld", fe->tag_length);
4036                 logf("bl: %ld", bytesleft);
4037                 close(fd);
4038                 return false;
4039             }
4040
4041             p = fe->tag_data;
4042             rc = read(fd, fe->tag_data, fe->tag_length);
4043             p += rc;
4044
4045             if (rc != fe->tag_length)
4046             {
4047                 logf("read error #13");
4048                 logf("rc=0x%04x", rc); // 0x431
4049                 logf("len=0x%04lx", fe->tag_length); // 0x4000
4050                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4051                 logf("tag=0x%02x", tag); // 0x00
4052                 close(fd);
4053                 return false;
4054             }
4055         }
4056         close(fd);
4057     }
4058
4059     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4060     logf("tagcache loaded into ram!");
4061
4062     return true;
4063 }
4064 #endif /* HAVE_TC_RAMCACHE */
4065
4066 static bool check_deleted_files(void)
4067 {
4068     int fd;
4069     char buf[TAG_MAXLEN+32];
4070     struct tagfile_entry tfe;
4071
4072     logf("reverse scan...");
4073     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4074     fd = open(buf, O_RDONLY);
4075
4076     if (fd < 0)
4077     {
4078         logf("%s open fail", buf);
4079         return false;
4080     }
4081
4082     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4083     while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4084 #ifndef __PCTOOL__
4085            && !check_event_queue()
4086 #endif
4087            )
4088     {
4089         if (tfe.tag_length >= (long)sizeof(buf)-1)
4090         {
4091             logf("too long tag");
4092             close(fd);
4093             return false;
4094         }
4095
4096         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4097         {
4098             logf("read error #14");
4099             close(fd);
4100             return false;
4101         }
4102
4103         /* Check if the file has already deleted from the db. */
4104         if (*buf == '\0')
4105             continue;
4106
4107         /* Now check if the file exists. */
4108         if (!file_exists(buf))
4109         {
4110             logf("Entry no longer valid.");
4111             logf("-> %s / %ld", buf, tfe.tag_length);
4112             delete_entry(tfe.idx_id);
4113         }
4114     }
4115
4116     close(fd);
4117
4118     logf("done");
4119
4120     return true;
4121 }
4122
4123
4124 /* Note that this function must not be inlined, otherwise the whole point
4125  * of having the code in a separate function is lost.
4126  */
4127 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4128     int *ignore, int *unignore)
4129 {
4130     char newpath[MAX_PATH];
4131
4132     /* check for a database.ignore file */
4133     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4134     *ignore = file_exists(newpath);
4135     /* check for a database.unignore file */
4136     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4137     *unignore = file_exists(newpath);
4138 }
4139
4140
4141 static bool check_dir(const char *dirname, int add_files)
4142 {
4143     DIR *dir;
4144     int len;
4145     int success = false;
4146     int ignore, unignore;
4147
4148     dir = opendir(dirname);
4149     if (!dir)
4150     {
4151         logf("tagcache: opendir(%s) failed", dirname);
4152         return false;
4153     }
4154
4155     /* check for a database.ignore and database.unignore */
4156     check_ignore(dirname, &ignore, &unignore);
4157
4158     /* don't do anything if both ignore and unignore are there */
4159     if (ignore != unignore)
4160         add_files = unignore;
4161
4162     /* Recursively scan the dir. */
4163 #ifdef __PCTOOL__
4164     while (1)
4165 #else
4166     while (!check_event_queue())
4167 #endif
4168     {
4169         struct dirent *entry;
4170
4171         entry = readdir(dir);
4172
4173         if (entry == NULL)
4174         {
4175             success = true;
4176             break ;
4177         }
4178
4179         if (!strcmp((char *)entry->d_name, ".") ||
4180             !strcmp((char *)entry->d_name, ".."))
4181             continue;
4182
4183         yield();
4184
4185         len = strlen(curpath);
4186         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4187                  entry->d_name);
4188
4189         processed_dir_count++;
4190         if (entry->attribute & ATTR_DIRECTORY)
4191             check_dir(curpath, add_files);
4192         else if (add_files)
4193         {
4194             tc_stat.curentry = curpath;
4195
4196             /* Add a new entry to the temporary db file. */
4197             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4198 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4199                          , dir->internal_entry
4200 #endif
4201                          );
4202
4203             /* Wait until current path for debug screen is read and unset. */
4204             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4205                 yield();
4206
4207             tc_stat.curentry = NULL;
4208         }
4209
4210         curpath[len] = '\0';
4211     }
4212
4213     closedir(dir);
4214
4215     return success;
4216 }
4217
4218 void tagcache_screensync_event(void)
4219 {
4220     tc_stat.curentry = NULL;
4221 }
4222
4223 void tagcache_screensync_enable(bool state)
4224 {
4225     tc_stat.syncscreen = state;
4226 }
4227
4228 void tagcache_build(const char *path)
4229 {
4230     struct tagcache_header header;
4231     bool ret;
4232
4233     curpath[0] = '\0';
4234     data_size = 0;
4235     total_entry_count = 0;
4236     processed_dir_count = 0;
4237
4238 #ifdef HAVE_DIRCACHE
4239     while (dircache_is_initializing())
4240         sleep(1);
4241 #endif
4242
4243     logf("updating tagcache");
4244
4245     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4246     if (cachefd >= 0)
4247     {
4248         logf("skipping, cache already waiting for commit");
4249         close(cachefd);
4250         return ;
4251     }
4252
4253     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4254     if (cachefd < 0)
4255     {
4256         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4257         return ;
4258     }
4259
4260     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4261
4262     cpu_boost(true);
4263
4264     logf("Scanning files...");
4265     /* Scan for new files. */
4266     memset(&header, 0, sizeof(struct tagcache_header));
4267     write(cachefd, &header, sizeof(struct tagcache_header));
4268
4269     if (strcmp("/", path) != 0)
4270         strcpy(curpath, path);
4271     ret = check_dir(path, true);
4272
4273     /* Write the header. */
4274     header.magic = TAGCACHE_MAGIC;
4275     header.datasize = data_size;
4276     header.entry_count = total_entry_count;
4277     lseek(cachefd, 0, SEEK_SET);
4278     write(cachefd, &header, sizeof(struct tagcache_header));
4279     close(cachefd);
4280
4281     if (filenametag_fd >= 0)
4282     {
4283         close(filenametag_fd);
4284         filenametag_fd = -1;
4285     }
4286
4287     if (!ret)
4288     {
4289         logf("Aborted.");
4290         cpu_boost(false);
4291         return ;
4292     }
4293
4294     /* Commit changes to the database. */
4295 #ifdef __PCTOOL__
4296     allocate_tempbuf();
4297 #endif
4298     if (commit())
4299     {
4300         remove(TAGCACHE_FILE_TEMP);
4301         logf("tagcache built!");
4302     }
4303 #ifdef __PCTOOL__
4304     free_tempbuf();
4305 #endif
4306
4307 #ifdef HAVE_TC_RAMCACHE
4308     if (hdr)
4309     {
4310         /* Import runtime statistics if we just initialized the db. */
4311         if (hdr->h.serial == 0)
4312             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4313     }
4314 #endif
4315
4316     cpu_boost(false);
4317 }
4318
4319 #ifdef HAVE_TC_RAMCACHE
4320 static void load_ramcache(void)
4321 {
4322     if (!hdr)
4323         return ;
4324
4325     cpu_boost(true);
4326
4327     /* At first we should load the cache (if exists). */
4328     tc_stat.ramcache = load_tagcache();
4329
4330     if (!tc_stat.ramcache)
4331     {
4332         /* If loading failed, it must indicate some problem with the db
4333          * so disable it entirely to prevent further issues. */
4334         tc_stat.ready = false;
4335         hdr = NULL;
4336     }
4337
4338     cpu_boost(false);
4339 }
4340
4341 void tagcache_unload_ramcache(void)
4342 {
4343     tc_stat.ramcache = false;
4344     /* Just to make sure there is no statefile present. */
4345     // remove(TAGCACHE_STATEFILE);
4346 }
4347 #endif
4348
4349 #ifndef __PCTOOL__
4350 static void tagcache_thread(void)
4351 {
4352     struct queue_event ev;
4353     bool check_done = false;
4354
4355     /* If the previous cache build/update was interrupted, commit
4356      * the changes first in foreground. */
4357     cpu_boost(true);
4358     allocate_tempbuf();
4359     commit();
4360     free_tempbuf();
4361
4362 #ifdef HAVE_TC_RAMCACHE
4363 # ifdef HAVE_EEPROM_SETTINGS
4364     if (firmware_settings.initialized && firmware_settings.disk_clean)
4365         check_done = tagcache_dumpload();
4366
4367     remove(TAGCACHE_STATEFILE);
4368 # endif
4369
4370     /* Allocate space for the tagcache if found on disk. */
4371     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4372         allocate_tagcache();
4373 #endif
4374
4375     cpu_boost(false);
4376     tc_stat.initialized = true;
4377
4378     /* Don't delay bootup with the header check but do it on background. */
4379     sleep(HZ);
4380     tc_stat.ready = check_all_headers();
4381     tc_stat.readyvalid = true;
4382
4383     while (1)
4384     {
4385         run_command_queue(false);
4386
4387         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4388
4389         switch (ev.id)
4390         {
4391             case Q_IMPORT_CHANGELOG:
4392                 tagcache_import_changelog();
4393                 break;
4394
4395             case Q_REBUILD:
4396                 remove_files();
4397                 remove(TAGCACHE_FILE_TEMP);
4398                 tagcache_build("/");
4399                 break;
4400
4401             case Q_UPDATE:
4402                 tagcache_build("/");
4403 #ifdef HAVE_TC_RAMCACHE
4404                 load_ramcache();
4405 #endif
4406                 check_deleted_files();
4407                 break ;
4408
4409             case Q_START_SCAN:
4410                 check_done = false;
4411             case SYS_TIMEOUT:
4412                 if (check_done || !tc_stat.ready)
4413                     break ;
4414
4415 #ifdef HAVE_TC_RAMCACHE
4416                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4417                 {
4418                     load_ramcache();
4419                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4420                         tagcache_build("/");
4421                 }
4422                 else
4423 #endif
4424                 if (global_settings.tagcache_autoupdate)
4425                 {
4426                     tagcache_build("/");
4427
4428                     /* This will be very slow unless dircache is enabled
4429                        or target is flash based, but do it anyway for
4430                        consistency. */
4431                     check_deleted_files();
4432                 }
4433
4434                 logf("tagcache check done");
4435
4436                 check_done = true;
4437                 break ;
4438
4439             case Q_STOP_SCAN:
4440                 break ;
4441
4442             case SYS_POWEROFF:
4443                 break ;
4444
4445 #ifndef SIMULATOR
4446             case SYS_USB_CONNECTED:
4447                 logf("USB: TagCache");
4448                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4449                 usb_wait_for_disconnect(&tagcache_queue);
4450                 break ;
4451 #endif
4452         }
4453     }
4454 }
4455
4456 bool tagcache_prepare_shutdown(void)
4457 {
4458     if (tagcache_get_commit_step() > 0)
4459         return false;
4460
4461     tagcache_stop_scan();
4462     while (read_lock || write_lock)
4463         sleep(1);
4464
4465     return true;
4466 }
4467
4468 void tagcache_shutdown(void)
4469 {
4470     /* Flush the command queue. */
4471     run_command_queue(true);
4472
4473 #ifdef HAVE_EEPROM_SETTINGS
4474     if (tc_stat.ramcache)
4475         tagcache_dumpsave();
4476 #endif
4477 }
4478
4479 static int get_progress(void)
4480 {
4481     int total_count = -1;
4482
4483 #ifdef HAVE_DIRCACHE
4484     if (dircache_is_enabled())
4485     {
4486         total_count = dircache_get_entry_count();
4487     }
4488     else
4489 #endif
4490 #ifdef HAVE_TC_RAMCACHE
4491     {
4492         if (hdr && tc_stat.ramcache)
4493             total_count = hdr->h.tch.entry_count;
4494     }
4495 #endif
4496
4497     if (total_count < 0)
4498         return -1;
4499
4500     return processed_dir_count * 100 / total_count;
4501 }
4502
4503 struct tagcache_stat* tagcache_get_stat(void)
4504 {
4505     tc_stat.progress = get_progress();
4506     tc_stat.processed_entries = processed_dir_count;
4507
4508     return &tc_stat;
4509 }
4510
4511 void tagcache_start_scan(void)
4512 {
4513     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4514 }
4515
4516 bool tagcache_update(void)
4517 {
4518     if (!tc_stat.ready)
4519         return false;
4520
4521     queue_post(&tagcache_queue, Q_UPDATE, 0);
4522     return false;
4523 }
4524
4525 bool tagcache_rebuild()
4526 {
4527     queue_post(&tagcache_queue, Q_REBUILD, 0);
4528     return false;
4529 }
4530
4531 void tagcache_stop_scan(void)
4532 {
4533     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4534 }
4535
4536 #ifdef HAVE_TC_RAMCACHE
4537 bool tagcache_is_ramcache(void)
4538 {
4539     return tc_stat.ramcache;
4540 }
4541 #endif
4542
4543 #endif /* !__PCTOOL__ */
4544
4545
4546 void tagcache_init(void)
4547 {
4548     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4549     memset(&current_tcmh, 0, sizeof(struct master_header));
4550     filenametag_fd = -1;
4551     write_lock = read_lock = 0;
4552
4553 #ifndef __PCTOOL__
4554     mutex_init(&command_queue_mutex);
4555     queue_init(&tagcache_queue, true);
4556     create_thread(tagcache_thread, tagcache_stack,
4557                   sizeof(tagcache_stack), 0, tagcache_thread_name
4558                   IF_PRIO(, PRIORITY_BACKGROUND)
4559                   IF_COP(, CPU));
4560 #else
4561     tc_stat.initialized = true;
4562     allocate_tempbuf();
4563     commit();
4564     free_tempbuf();
4565     tc_stat.ready = check_all_headers();
4566 #endif
4567 }
4568
4569 #ifdef __PCTOOL__
4570 void tagcache_reverse_scan(void)
4571 {
4572     logf("Checking for deleted files");
4573     check_deleted_files();
4574 }
4575 #endif
4576
4577 bool tagcache_is_initialized(void)
4578 {
4579     return tc_stat.initialized;
4580 }
4581 bool tagcache_is_usable(void)
4582 {
4583     return tc_stat.initialized && tc_stat.ready;
4584 }
4585 int tagcache_get_commit_step(void)
4586 {
4587     return tc_stat.commit_step;
4588 }
4589 int tagcache_get_max_commit_step(void)
4590 {
4591     return (int)(SORTED_TAGS_COUNT)+1;
4592 }
4593