apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 #define UNTAGGED "<Untagged>"
  99
 100 /* Previous path when scanning directory tree recursively. */
 101 static char curpath[TAG_MAXLEN+32];
 102
 103 /* Used when removing duplicates. */
 104 static char *tempbuf;     /* Allocated when needed. */
 105 static long tempbufidx;   /* Current location in buffer. */
 106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 107 static long tempbuf_left; /* Buffer space left. */
 108 static long tempbuf_pos;
 109
 110 #define SORTED_TAGS_COUNT 8
 111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 114     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 115 /* Tags we want to get sorted (loaded to the tempbuf). */
 116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 117     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 118     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 119
 120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 122     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 123     (1LU << tag_albumartist) | (1LU << tag_grouping))
 124
 125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 126 static const char *tags_str[] = { "artist", "album", "genre", "title",
 127     "filename", "composer", "comment", "albumartist", "grouping", "year",
 128     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 129     "playtime", "lastplayed", "commitid", "mtime" };
 130
 131 /* Status information of the tagcache. */
 132 static struct tagcache_stat tc_stat;
 133
 134 /* Queue commands. */
 135 enum tagcache_queue {
 136     Q_STOP_SCAN = 0,
 137     Q_START_SCAN,
 138     Q_IMPORT_CHANGELOG,
 139     Q_UPDATE,
 140     Q_REBUILD,
 141
 142     /* Internal tagcache command queue. */
 143     CMD_UPDATE_MASTER_HEADER,
 144     CMD_UPDATE_NUMERIC,
 145 };
 146
 147 struct tagcache_command_entry {
 148     int32_t command;
 149     int32_t idx_id;
 150     int32_t tag;
 151     int32_t data;
 152 };
 153
 154 #ifndef __PCTOOL__
 155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 156 static volatile int command_queue_widx = 0;
 157 static volatile int command_queue_ridx = 0;
 158 static struct mutex command_queue_mutex;
 159 #endif
 160
 161 /* Tag database structures. */
 162
 163 /* Variable-length tag entry in tag files. */
 164 struct tagfile_entry {
 165     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 166     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 167     char tag_data[0];  /* Begin of the tag data */
 168 };
 169
 170 /* Fixed-size tag entry in master db index. */
 171 struct index_entry {
 172     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 173     int32_t flag;                /* Status flags */
 174 };
 175
 176 /* Header is the same in every file. */
 177 struct tagcache_header {
 178     int32_t magic;       /* Header version number */
 179     int32_t datasize;    /* Data size in bytes */
 180     int32_t entry_count; /* Number of entries in this file */
 181 };
 182
 183 struct master_header {
 184     struct tagcache_header tch;
 185     int32_t serial; /* Increasing counting number */
 186     int32_t commitid; /* Number of commits so far */
 187     int32_t dirty;
 188 };
 189
 190 /* For the endianess correction */
 191 static const char *tagfile_entry_ec   = "ll";
 192 /**
 193  Note: This should be (1 + TAG_COUNT) amount of l's.
 194  */
 195 static const char *index_entry_ec     = "lllllllllllllllllllll";
 196
 197 static const char *tagcache_header_ec = "lll";
 198 static const char *master_header_ec   = "llllll";
 199
 200 static struct master_header current_tcmh;
 201
 202 #ifdef HAVE_TC_RAMCACHE
 203 /* Header is created when loading database to ram. */
 204 struct ramcache_header {
 205     struct master_header h;      /* Header from the master index */
 206     struct index_entry *indices; /* Master index file content */
 207     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 208     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 209 };
 210
 211 # ifdef HAVE_EEPROM_SETTINGS
 212 struct statefile_header {
 213     struct ramcache_header *hdr;
 214     struct tagcache_stat tc_stat;
 215 };
 216 # endif
 217
 218 /* Pointer to allocated ramcache_header */
 219 static struct ramcache_header *hdr;
 220 #endif
 221
 222 /**
 223  * Full tag entries stored in a temporary file waiting
 224  * for commit to the cache. */
 225 struct temp_file_entry {
 226     long tag_offset[TAG_COUNT];
 227     short tag_length[TAG_COUNT];
 228     long flag;
 229
 230     long data_length;
 231 };
 232
 233 struct tempbuf_id_list {
 234     long id;
 235     struct tempbuf_id_list *next;
 236 };
 237
 238 struct tempbuf_searchidx {
 239     long idx_id;
 240     char *str;
 241     int seek;
 242     struct tempbuf_id_list idlist;
 243 };
 244
 245 /* Lookup buffer for fixing messed up index while after sorting. */
 246 static long commit_entry_count;
 247 static long lookup_buffer_depth;
 248 static struct tempbuf_searchidx **lookup;
 249
 250 /* Used when building the temporary file. */
 251 static int cachefd = -1, filenametag_fd;
 252 static int total_entry_count = 0;
 253 static int data_size = 0;
 254 static int processed_dir_count;
 255
 256 /* Thread safe locking */
 257 static volatile int write_lock;
 258 static volatile int read_lock;
 259
 260 static bool delete_entry(long idx_id);
 261
 262 const char* tagcache_tag_to_str(int tag)
 263 {
 264     return tags_str[tag];
 265 }
 266
 267 #ifdef HAVE_DIRCACHE
 268 /**
 269  * Returns true if specified flag is still present, i.e., dircache
 270  * has not been reloaded.
 271  */
 272 static bool is_dircache_intact(void)
 273 {
 274     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 275 }
 276 #endif
 277
 278 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 279 {
 280     int fd;
 281     char buf[MAX_PATH];
 282     int rc;
 283
 284     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 285         return -1;
 286
 287     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 288
 289     fd = open(buf, write ? O_RDWR : O_RDONLY);
 290     if (fd < 0)
 291     {
 292         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 293         tc_stat.ready = false;
 294         return fd;
 295     }
 296
 297     /* Check the header. */
 298     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 299     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 300     {
 301         logf("header error");
 302         tc_stat.ready = false;
 303         close(fd);
 304         return -2;
 305     }
 306
 307     return fd;
 308 }
 309
 310 static int open_master_fd(struct master_header *hdr, bool write)
 311 {
 312     int fd;
 313     int rc;
 314
 315     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 316     if (fd < 0)
 317     {
 318         logf("master file open failed for R/W");
 319         tc_stat.ready = false;
 320         return fd;
 321     }
 322
 323     tc_stat.econ = false;
 324
 325     /* Check the header. */
 326     rc = read(fd, hdr, sizeof(struct master_header));
 327     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 328     {
 329         /* Success. */
 330         return fd;
 331     }
 332
 333     /* Trying to read again, this time with endianess correction enabled. */
 334     lseek(fd, 0, SEEK_SET);
 335
 336     rc = ecread(fd, hdr, 1, master_header_ec, true);
 337     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 338     {
 339         logf("header error");
 340         tc_stat.ready = false;
 341         close(fd);
 342         return -2;
 343     }
 344
 345     tc_stat.econ = true;
 346
 347     return fd;
 348 }
 349
 350 #ifndef __PCTOOL__
 351 static bool do_timed_yield(void)
 352 {
 353     /* Sorting can lock up for quite a while, so yield occasionally */
 354     static long wakeup_tick = 0;
 355     if (TIME_AFTER(current_tick, wakeup_tick))
 356     {
 357         wakeup_tick = current_tick + (HZ/4);
 358         yield();
 359         return true;
 360     }
 361     return false;
 362 }
 363 #endif
 364
 365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 366 static long find_entry_ram(const char *filename,
 367                            const struct dirent *dc)
 368 {
 369     static long last_pos = 0;
 370     int i;
 371
 372     /* Check if we tagcache is loaded into ram. */
 373     if (!tc_stat.ramcache)
 374         return -1;
 375
 376     if (dc == NULL)
 377         dc = dircache_get_entry_ptr(filename);
 378
 379     if (dc == NULL)
 380     {
 381         logf("tagcache: file not found.");
 382         return -1;
 383     }
 384
 385     try_again:
 386
 387     if (last_pos > 0)
 388         i = last_pos;
 389     else
 390         i = 0;
 391
 392     for (; i < hdr->h.tch.entry_count; i++)
 393     {
 394         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 395         {
 396             last_pos = MAX(0, i - 3);
 397             return i;
 398         }
 399
 400         do_timed_yield();
 401     }
 402
 403     if (last_pos > 0)
 404     {
 405         last_pos = 0;
 406         goto try_again;
 407     }
 408
 409     return -1;
 410 }
 411 #endif
 412
 413 static long find_entry_disk(const char *filename, bool localfd)
 414 {
 415     struct tagcache_header tch;
 416     static long last_pos = -1;
 417     long pos_history[POS_HISTORY_COUNT];
 418     long pos_history_idx = 0;
 419     bool found = false;
 420     struct tagfile_entry tfe;
 421     int fd;
 422     char buf[TAG_MAXLEN+32];
 423     int i;
 424     int pos = -1;
 425
 426     if (!tc_stat.ready)
 427         return -2;
 428
 429     fd = filenametag_fd;
 430     if (fd < 0 || localfd)
 431     {
 432         last_pos = -1;
 433         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 434             return -1;
 435     }
 436
 437     check_again:
 438
 439     if (last_pos > 0)
 440         lseek(fd, last_pos, SEEK_SET);
 441     else
 442         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 443
 444     while (true)
 445     {
 446         pos = lseek(fd, 0, SEEK_CUR);
 447         for (i = pos_history_idx-1; i >= 0; i--)
 448             pos_history[i+1] = pos_history[i];
 449         pos_history[0] = pos;
 450
 451         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 452             != sizeof(struct tagfile_entry))
 453         {
 454             break ;
 455         }
 456
 457         if (tfe.tag_length >= (long)sizeof(buf))
 458         {
 459             logf("too long tag #1");
 460             close(fd);
 461             if (!localfd)
 462                 filenametag_fd = -1;
 463             last_pos = -1;
 464             return -2;
 465         }
 466
 467         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 468         {
 469             logf("read error #2");
 470             close(fd);
 471             if (!localfd)
 472                 filenametag_fd = -1;
 473             last_pos = -1;
 474             return -3;
 475         }
 476
 477         if (!strcasecmp(filename, buf))
 478         {
 479             last_pos = pos_history[pos_history_idx];
 480             found = true;
 481             break ;
 482         }
 483
 484         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 485             pos_history_idx++;
 486     }
 487
 488     /* Not found? */
 489     if (!found)
 490     {
 491         if (last_pos > 0)
 492         {
 493             last_pos = -1;
 494             logf("seek again");
 495             goto check_again;
 496         }
 497
 498         if (fd != filenametag_fd || localfd)
 499             close(fd);
 500         return -4;
 501     }
 502
 503     if (fd != filenametag_fd || localfd)
 504         close(fd);
 505
 506     return tfe.idx_id;
 507 }
 508
 509 static int find_index(const char *filename)
 510 {
 511     long idx_id = -1;
 512
 513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 514     if (tc_stat.ramcache && is_dircache_intact())
 515         idx_id = find_entry_ram(filename, NULL);
 516 #endif
 517
 518     if (idx_id < 0)
 519         idx_id = find_entry_disk(filename, true);
 520
 521     return idx_id;
 522 }
 523
 524 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 525 {
 526     int idx_id;
 527
 528     if (!tc_stat.ready)
 529         return false;
 530
 531     idx_id = find_index(filename);
 532     if (idx_id < 0)
 533         return false;
 534
 535     if (!tagcache_search(tcs, tag_filename))
 536         return false;
 537
 538     tcs->entry_count = 0;
 539     tcs->idx_id = idx_id;
 540
 541     return true;
 542 }
 543
 544 static bool get_index(int masterfd, int idxid,
 545                       struct index_entry *idx, bool use_ram)
 546 {
 547     bool localfd = false;
 548
 549     if (idxid < 0)
 550     {
 551         logf("Incorrect idxid: %d", idxid);
 552         return false;
 553     }
 554
 555 #ifdef HAVE_TC_RAMCACHE
 556     if (tc_stat.ramcache && use_ram)
 557     {
 558         if (hdr->indices[idxid].flag & FLAG_DELETED)
 559             return false;
 560
 561 # ifdef HAVE_DIRCACHE
 562         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 563             || is_dircache_intact())
 564 #endif
 565         {
 566             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 567             return true;
 568         }
 569     }
 570 #else
 571     (void)use_ram;
 572 #endif
 573
 574     if (masterfd < 0)
 575     {
 576         struct master_header tcmh;
 577
 578         localfd = true;
 579         masterfd = open_master_fd(&tcmh, false);
 580         if (masterfd < 0)
 581             return false;
 582     }
 583
 584     lseek(masterfd, idxid * sizeof(struct index_entry)
 585           + sizeof(struct master_header), SEEK_SET);
 586     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 587         != sizeof(struct index_entry))
 588     {
 589         logf("read error #3");
 590         if (localfd)
 591             close(masterfd);
 592
 593         return false;
 594     }
 595
 596     if (localfd)
 597         close(masterfd);
 598
 599     if (idx->flag & FLAG_DELETED)
 600         return false;
 601
 602     return true;
 603 }
 604
 605 #ifndef __PCTOOL__
 606
 607 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 608 {
 609     /* We need to exclude all memory only flags & tags when writing to disk. */
 610     if (idx->flag & FLAG_DIRCACHE)
 611     {
 612         logf("memory only flags!");
 613         return false;
 614     }
 615
 616 #ifdef HAVE_TC_RAMCACHE
 617     /* Only update numeric data. Writing the whole index to RAM by memcpy
 618      * destroys dircache pointers!
 619      */
 620     if (tc_stat.ramcache)
 621     {
 622         int tag;
 623         struct index_entry *idx_ram = &hdr->indices[idxid];
 624
 625         for (tag = 0; tag < TAG_COUNT; tag++)
 626         {
 627             if (TAGCACHE_IS_NUMERIC(tag))
 628             {
 629                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 630             }
 631         }
 632
 633         /* Don't touch the dircache flag or attributes. */
 634         idx_ram->flag = (idx->flag & 0x0000ffff)
 635             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 636     }
 637 #endif
 638
 639     lseek(masterfd, idxid * sizeof(struct index_entry)
 640           + sizeof(struct master_header), SEEK_SET);
 641     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 642         != sizeof(struct index_entry))
 643     {
 644         logf("write error #3");
 645         logf("idxid: %d", idxid);
 646         return false;
 647     }
 648
 649     return true;
 650 }
 651
 652 #endif /* !__PCTOOL__ */
 653
 654 static bool open_files(struct tagcache_search *tcs, int tag)
 655 {
 656     if (tcs->idxfd[tag] < 0)
 657     {
 658         char fn[MAX_PATH];
 659
 660         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 661         tcs->idxfd[tag] = open(fn, O_RDONLY);
 662     }
 663
 664     if (tcs->idxfd[tag] < 0)
 665     {
 666         logf("File not open!");
 667         return false;
 668     }
 669
 670     return true;
 671 }
 672
 673 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 674                      int tag, char *buf, long size)
 675 {
 676     struct tagfile_entry tfe;
 677     long seek;
 678
 679     *buf = '\0';
 680
 681     if (TAGCACHE_IS_NUMERIC(tag))
 682         return false;
 683
 684     seek = idx->tag_seek[tag];
 685     if (seek < 0)
 686     {
 687         logf("Retrieve failed");
 688         return false;
 689     }
 690
 691 #ifdef HAVE_TC_RAMCACHE
 692     if (tcs->ramsearch)
 693     {
 694         struct tagfile_entry *ep;
 695
 696 # ifdef HAVE_DIRCACHE
 697         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 698             && is_dircache_intact())
 699         {
 700             dircache_copy_path((struct dirent *)seek,
 701                                buf, size);
 702             return true;
 703         }
 704         else
 705 # endif
 706         if (tag != tag_filename)
 707         {
 708             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 709             strlcpy(buf, ep->tag_data, size);
 710
 711             return true;
 712         }
 713     }
 714 #endif
 715
 716     if (!open_files(tcs, tag))
 717         return false;
 718
 719     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 720     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 721         != sizeof(struct tagfile_entry))
 722     {
 723         logf("read error #5");
 724         return false;
 725     }
 726
 727     if (tfe.tag_length >= size)
 728     {
 729         logf("too small buffer");
 730         return false;
 731     }
 732
 733     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 734         tfe.tag_length)
 735     {
 736         logf("read error #6");
 737         return false;
 738     }
 739
 740     buf[tfe.tag_length] = '\0';
 741
 742     return true;
 743 }
 744
 745 static long check_virtual_tags(int tag, const struct index_entry *idx)
 746 {
 747     long data = 0;
 748
 749     switch (tag)
 750     {
 751         case tag_virt_length_sec:
 752             data = (idx->tag_seek[tag_length]/1000) % 60;
 753             break;
 754
 755         case tag_virt_length_min:
 756             data = (idx->tag_seek[tag_length]/1000) / 60;
 757             break;
 758
 759         case tag_virt_playtime_sec:
 760             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 761             break;
 762
 763         case tag_virt_playtime_min:
 764             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 765             break;
 766
 767         case tag_virt_autoscore:
 768             if (idx->tag_seek[tag_length] == 0
 769                 || idx->tag_seek[tag_playcount] == 0)
 770             {
 771                 data = 0;
 772             }
 773             else
 774             {
 775                 /* A straight calculus gives:
 776                      autoscore = 100 * playtime / length / playcout (1)
 777                    Now, consider the euclidian division of playtime by length:
 778                      playtime = alpha * length + beta
 779                    With:
 780                      0 <= beta < length
 781                    Now, (1) becomes:
 782                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 783                    Both terms should be small enough to avoid any overflow
 784                 */
 785                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 786                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 787                 data /=  idx->tag_seek[tag_playcount];
 788             }
 789             break;
 790
 791         /* How many commits before the file has been added to the DB. */
 792         case tag_virt_entryage:
 793             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 794             break;
 795
 796         default:
 797             data = idx->tag_seek[tag];
 798     }
 799
 800     return data;
 801 }
 802
 803 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 804 {
 805     struct index_entry idx;
 806
 807     if (!tc_stat.ready)
 808         return false;
 809
 810     if (!TAGCACHE_IS_NUMERIC(tag))
 811         return -1;
 812
 813     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 814         return -2;
 815
 816     return check_virtual_tags(tag, &idx);
 817 }
 818
 819 inline static bool str_ends_with(const char *str1, const char *str2)
 820 {
 821     int str_len = strlen(str1);
 822     int clause_len = strlen(str2);
 823
 824     if (clause_len > str_len)
 825         return false;
 826
 827     return !strcasecmp(&str1[str_len - clause_len], str2);
 828 }
 829
 830 inline static bool str_oneof(const char *str, const char *list)
 831 {
 832     const char *sep;
 833     int l, len = strlen(str);
 834
 835     while (*list)
 836     {
 837         sep = strchr(list, '|');
 838         l = sep ? (long)sep - (long)list : (int)strlen(list);
 839         if ((l==len) && !strncasecmp(str, list, len))
 840             return true;
 841         list += sep ? l + 1 : l;
 842     }
 843
 844     return false;
 845 }
 846
 847 static bool check_against_clause(long numeric, const char *str,
 848                                  const struct tagcache_search_clause *clause)
 849 {
 850     if (clause->numeric)
 851     {
 852         switch (clause->type)
 853         {
 854             case clause_is:
 855                 return numeric == clause->numeric_data;
 856             case clause_is_not:
 857                 return numeric != clause->numeric_data;
 858             case clause_gt:
 859                 return numeric > clause->numeric_data;
 860             case clause_gteq:
 861                 return numeric >= clause->numeric_data;
 862             case clause_lt:
 863                 return numeric < clause->numeric_data;
 864             case clause_lteq:
 865                 return numeric <= clause->numeric_data;
 866             default:
 867                 logf("Incorrect numeric tag: %d", clause->type);
 868         }
 869     }
 870     else
 871     {
 872         switch (clause->type)
 873         {
 874             case clause_is:
 875                 return !strcasecmp(clause->str, str);
 876             case clause_is_not:
 877                 return strcasecmp(clause->str, str);
 878             case clause_gt:
 879                 return 0>strcasecmp(clause->str, str);
 880             case clause_gteq:
 881                 return 0>=strcasecmp(clause->str, str);
 882             case clause_lt:
 883                 return 0<strcasecmp(clause->str, str);
 884             case clause_lteq:
 885                 return 0<=strcasecmp(clause->str, str);
 886             case clause_contains:
 887                 return (strcasestr(str, clause->str) != NULL);
 888             case clause_not_contains:
 889                 return (strcasestr(str, clause->str) == NULL);
 890             case clause_begins_with:
 891                 return (strcasestr(str, clause->str) == str);
 892             case clause_not_begins_with:
 893                 return (strcasestr(str, clause->str) != str);
 894             case clause_ends_with:
 895                 return str_ends_with(str, clause->str);
 896             case clause_not_ends_with:
 897                 return !str_ends_with(str, clause->str);
 898             case clause_oneof:
 899                 return str_oneof(str, clause->str);
 900
 901             default:
 902                 logf("Incorrect tag: %d", clause->type);
 903         }
 904     }
 905
 906     return false;
 907 }
 908
 909 static bool check_clauses(struct tagcache_search *tcs,
 910                           struct index_entry *idx,
 911                           struct tagcache_search_clause **clause, int count)
 912 {
 913     int i;
 914
 915 #ifdef HAVE_TC_RAMCACHE
 916     if (tcs->ramsearch)
 917     {
 918         /* Go through all conditional clauses. */
 919         for (i = 0; i < count; i++)
 920         {
 921             struct tagfile_entry *tfe;
 922             int seek;
 923             char buf[256];
 924             char *str = NULL;
 925
 926             seek = check_virtual_tags(clause[i]->tag, idx);
 927
 928             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 929             {
 930                 if (clause[i]->tag == tag_filename)
 931                 {
 932                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 933                     str = buf;
 934                 }
 935                 else
 936                 {
 937                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 938                     str = tfe->tag_data;
 939                 }
 940             }
 941
 942             if (!check_against_clause(seek, str, clause[i]))
 943                 return false;
 944         }
 945     }
 946     else
 947 #endif
 948     {
 949         /* Check for conditions. */
 950         for (i = 0; i < count; i++)
 951         {
 952             struct tagfile_entry tfe;
 953             int seek;
 954             char str[256];
 955
 956             seek = check_virtual_tags(clause[i]->tag, idx);
 957
 958             memset(str, 0, sizeof str);
 959             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 960             {
 961                 int fd = tcs->idxfd[clause[i]->tag];
 962                 lseek(fd, seek, SEEK_SET);
 963                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 964                 if (tfe.tag_length >= (int)sizeof(str))
 965                 {
 966                     logf("Too long tag read!");
 967                     break ;
 968                 }
 969
 970                 read(fd, str, tfe.tag_length);
 971
 972                 /* Check if entry has been deleted. */
 973                 if (str[0] == '\0')
 974                     break;
 975             }
 976
 977             if (!check_against_clause(seek, str, clause[i]))
 978                 return false;
 979         }
 980     }
 981
 982     return true;
 983 }
 984
 985 bool tagcache_check_clauses(struct tagcache_search *tcs,
 986                             struct tagcache_search_clause **clause, int count)
 987 {
 988     struct index_entry idx;
 989
 990     if (count == 0)
 991         return true;
 992
 993     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 994         return false;
 995
 996     return check_clauses(tcs, &idx, clause, count);
 997 }
 998
 999 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1000 {
1001     int i;
1002
1003     /* If uniq buffer is not defined we must return true for search to work. */
1004     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1005         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1006     {
1007         return true;
1008     }
1009
1010     for (i = 0; i < tcs->unique_list_count; i++)
1011     {
1012         /* Return false if entry is found. */
1013         if (tcs->unique_list[i] == id)
1014             return false;
1015     }
1016
1017     if (tcs->unique_list_count < tcs->unique_list_capacity)
1018     {
1019         tcs->unique_list[i] = id;
1020         tcs->unique_list_count++;
1021     }
1022
1023     return true;
1024 }
1025
1026 static bool build_lookup_list(struct tagcache_search *tcs)
1027 {
1028     struct index_entry entry;
1029     int i, j;
1030
1031     tcs->seek_list_count = 0;
1032
1033 #ifdef HAVE_TC_RAMCACHE
1034     if (tcs->ramsearch
1035 # ifdef HAVE_DIRCACHE
1036         && (tcs->type != tag_filename || is_dircache_intact())
1037 # endif
1038         )
1039     {
1040         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1041         {
1042             struct tagcache_seeklist_entry *seeklist;
1043             struct index_entry *idx = &hdr->indices[i];
1044             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1045                 break ;
1046
1047             /* Skip deleted files. */
1048             if (idx->flag & FLAG_DELETED)
1049                 continue;
1050
1051             /* Go through all filters.. */
1052             for (j = 0; j < tcs->filter_count; j++)
1053             {
1054                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1055                 {
1056                     break ;
1057                 }
1058             }
1059
1060             if (j < tcs->filter_count)
1061                 continue ;
1062
1063             /* Check for conditions. */
1064             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1065                 continue;
1066
1067             /* Add to the seek list if not already in uniq buffer. */
1068             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1069                 continue;
1070
1071             /* Lets add it. */
1072             seeklist = &tcs->seeklist[tcs->seek_list_count];
1073             seeklist->seek = idx->tag_seek[tcs->type];
1074             seeklist->flag = idx->flag;
1075             seeklist->idx_id = i;
1076             tcs->seek_list_count++;
1077         }
1078
1079         tcs->seek_pos = i;
1080
1081         return tcs->seek_list_count > 0;
1082     }
1083 #endif
1084
1085     if (tcs->masterfd < 0)
1086     {
1087         struct master_header tcmh;
1088         tcs->masterfd = open_master_fd(&tcmh, false);
1089     }
1090
1091     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1092             sizeof(struct master_header), SEEK_SET);
1093
1094     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1095            == sizeof(struct index_entry))
1096     {
1097         struct tagcache_seeklist_entry *seeklist;
1098
1099         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1100             break ;
1101
1102         i = tcs->seek_pos;
1103         tcs->seek_pos++;
1104
1105         /* Check if entry has been deleted. */
1106         if (entry.flag & FLAG_DELETED)
1107             continue;
1108
1109         /* Go through all filters.. */
1110         for (j = 0; j < tcs->filter_count; j++)
1111         {
1112             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1113                 break ;
1114         }
1115
1116         if (j < tcs->filter_count)
1117             continue ;
1118
1119         /* Check for conditions. */
1120         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1121             continue;
1122
1123         /* Add to the seek list if not already in uniq buffer. */
1124         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1125             continue;
1126
1127         /* Lets add it. */
1128         seeklist = &tcs->seeklist[tcs->seek_list_count];
1129         seeklist->seek = entry.tag_seek[tcs->type];
1130         seeklist->flag = entry.flag;
1131         seeklist->idx_id = i;
1132         tcs->seek_list_count++;
1133
1134         yield();
1135     }
1136
1137     return tcs->seek_list_count > 0;
1138 }
1139
1140
1141 static void remove_files(void)
1142 {
1143     int i;
1144     char buf[MAX_PATH];
1145
1146     tc_stat.ready = false;
1147     tc_stat.ramcache = false;
1148     tc_stat.econ = false;
1149     remove(TAGCACHE_FILE_MASTER);
1150     for (i = 0; i < TAG_COUNT; i++)
1151     {
1152         if (TAGCACHE_IS_NUMERIC(i))
1153             continue;
1154
1155         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1156         remove(buf);
1157     }
1158 }
1159
1160
1161 static bool check_all_headers(void)
1162 {
1163     struct master_header myhdr;
1164     struct tagcache_header tch;
1165     int tag;
1166     int fd;
1167
1168     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1169         return false;
1170
1171     close(fd);
1172     if (myhdr.dirty)
1173     {
1174         logf("tagcache is dirty!");
1175         return false;
1176     }
1177
1178     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1179
1180     for (tag = 0; tag < TAG_COUNT; tag++)
1181     {
1182         if (TAGCACHE_IS_NUMERIC(tag))
1183             continue;
1184
1185         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1186             return false;
1187
1188         close(fd);
1189     }
1190
1191     return true;
1192 }
1193
1194 bool tagcache_is_busy(void)
1195 {
1196     return read_lock || write_lock;
1197 }
1198
1199 bool tagcache_search(struct tagcache_search *tcs, int tag)
1200 {
1201     struct tagcache_header tag_hdr;
1202     struct master_header   master_hdr;
1203     int i;
1204
1205     while (read_lock)
1206         sleep(1);
1207
1208     memset(tcs, 0, sizeof(struct tagcache_search));
1209     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1210         return false;
1211
1212     tcs->position = sizeof(struct tagcache_header);
1213     tcs->type = tag;
1214     tcs->seek_pos = 0;
1215     tcs->list_position = 0;
1216     tcs->seek_list_count = 0;
1217     tcs->filter_count = 0;
1218     tcs->masterfd = -1;
1219
1220     for (i = 0; i < TAG_COUNT; i++)
1221         tcs->idxfd[i] = -1;
1222
1223 #ifndef HAVE_TC_RAMCACHE
1224     tcs->ramsearch = false;
1225 #else
1226     tcs->ramsearch = tc_stat.ramcache;
1227     if (tcs->ramsearch)
1228     {
1229         tcs->entry_count = hdr->entry_count[tcs->type];
1230     }
1231     else
1232 #endif
1233     {
1234         /* Always open as R/W so we can pass tcs to functions that modify data also
1235          * without failing. */
1236         tcs->masterfd = open_master_fd(&master_hdr, true);
1237         if (tcs->masterfd < 0)
1238             return false;
1239
1240         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1241         {
1242             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1243             if (tcs->idxfd[tcs->type] < 0)
1244                 return false;
1245
1246             tcs->entry_count = tag_hdr.entry_count;
1247         }
1248         else
1249         {
1250             tcs->entry_count = master_hdr.tch.entry_count;
1251         }
1252     }
1253
1254     tcs->valid = true;
1255     tcs->initialized = true;
1256     write_lock++;
1257
1258     return true;
1259 }
1260
1261 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1262                                  void *buffer, long length)
1263 {
1264     tcs->unique_list = (unsigned long *)buffer;
1265     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1266     tcs->unique_list_count = 0;
1267 }
1268
1269 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1270                                 int tag, int seek)
1271 {
1272     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1273         return false;
1274
1275     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1276         return false;
1277
1278     tcs->filter_tag[tcs->filter_count] = tag;
1279     tcs->filter_seek[tcs->filter_count] = seek;
1280     tcs->filter_count++;
1281
1282     return true;
1283 }
1284
1285 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1286                                 struct tagcache_search_clause *clause)
1287 {
1288     int i;
1289
1290     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1291     {
1292         logf("Too many clauses");
1293         return false;
1294     }
1295
1296     /* Check if there is already a similar filter in present (filters are
1297      * much faster than clauses).
1298      */
1299     for (i = 0; i < tcs->filter_count; i++)
1300     {
1301         if (tcs->filter_tag[i] == clause->tag)
1302             return true;
1303     }
1304
1305     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1306     {
1307         char buf[MAX_PATH];
1308
1309         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1310         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1311     }
1312
1313     tcs->clause[tcs->clause_count] = clause;
1314     tcs->clause_count++;
1315
1316     return true;
1317 }
1318
1319 static bool get_next(struct tagcache_search *tcs)
1320 {
1321     static char buf[TAG_MAXLEN+32];
1322     struct tagfile_entry entry;
1323     long flag = 0;
1324
1325     if (!tcs->valid || !tc_stat.ready)
1326         return false;
1327
1328     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1329 #ifdef HAVE_TC_RAMCACHE
1330         && !tcs->ramsearch
1331 #endif
1332         )
1333         return false;
1334
1335     /* Relative fetch. */
1336     if (tcs->filter_count > 0 || tcs->clause_count > 0
1337         || TAGCACHE_IS_NUMERIC(tcs->type)
1338 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1339         /* We need to retrieve flag status for dircache. */
1340         || (tcs->ramsearch && tcs->type == tag_filename)
1341 #endif
1342         )
1343     {
1344         struct tagcache_seeklist_entry *seeklist;
1345
1346         /* Check for end of list. */
1347         if (tcs->list_position == tcs->seek_list_count)
1348         {
1349             tcs->list_position = 0;
1350
1351             /* Try to fetch more. */
1352             if (!build_lookup_list(tcs))
1353             {
1354                 tcs->valid = false;
1355                 return false;
1356             }
1357         }
1358
1359         seeklist = &tcs->seeklist[tcs->list_position];
1360         flag = seeklist->flag;
1361         tcs->position = seeklist->seek;
1362         tcs->idx_id = seeklist->idx_id;
1363         tcs->list_position++;
1364     }
1365     else
1366     {
1367         if (tcs->entry_count == 0)
1368         {
1369             tcs->valid = false;
1370             return false;
1371         }
1372
1373         tcs->entry_count--;
1374     }
1375
1376     tcs->result_seek = tcs->position;
1377
1378     if (TAGCACHE_IS_NUMERIC(tcs->type))
1379     {
1380         snprintf(buf, sizeof(buf), "%d", tcs->position);
1381         tcs->result = buf;
1382         tcs->result_len = strlen(buf) + 1;
1383         return true;
1384     }
1385
1386     /* Direct fetch. */
1387 #ifdef HAVE_TC_RAMCACHE
1388     if (tcs->ramsearch)
1389     {
1390
1391 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1392         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1393             && is_dircache_intact())
1394         {
1395             dircache_copy_path((struct dirent *)tcs->position,
1396                                buf, sizeof buf);
1397             tcs->result = buf;
1398             tcs->result_len = strlen(buf) + 1;
1399             tcs->ramresult = false;
1400
1401             return true;
1402         }
1403         else
1404 #endif
1405         if (tcs->type != tag_filename)
1406         {
1407             struct tagfile_entry *ep;
1408
1409             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1410             tcs->result = ep->tag_data;
1411             tcs->result_len = strlen(tcs->result) + 1;
1412             tcs->idx_id = ep->idx_id;
1413             tcs->ramresult = true;
1414
1415             /* Increase position for the next run. This may get overwritten. */
1416             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1417
1418             return true;
1419         }
1420     }
1421 #endif
1422
1423     if (!open_files(tcs, tcs->type))
1424     {
1425         tcs->valid = false;
1426         return false;
1427     }
1428
1429     /* Seek stream to the correct position and continue to direct fetch. */
1430     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1431
1432     if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1433                tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1434     {
1435         logf("read error #5");
1436         tcs->valid = false;
1437         return false;
1438     }
1439
1440     if (entry.tag_length > (long)sizeof(buf))
1441     {
1442         tcs->valid = false;
1443         logf("too long tag #2");
1444         logf("P:%X/%X", tcs->position, entry.tag_length);
1445         return false;
1446     }
1447
1448     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1449     {
1450         tcs->valid = false;
1451         logf("read error #4");
1452         return false;
1453     }
1454
1455     /**
1456      Update the position for the next read (this may be overridden
1457      if filters or clauses are being used).
1458      */
1459     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1460     tcs->result = buf;
1461     tcs->result_len = strlen(tcs->result) + 1;
1462     tcs->idx_id = entry.idx_id;
1463     tcs->ramresult = false;
1464
1465     return true;
1466 }
1467
1468 bool tagcache_get_next(struct tagcache_search *tcs)
1469 {
1470     while (get_next(tcs))
1471     {
1472         if (tcs->result_len > 1)
1473             return true;
1474     }
1475
1476     return false;
1477 }
1478
1479 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1480                        int tag, char *buf, long size)
1481 {
1482     struct index_entry idx;
1483
1484     *buf = '\0';
1485     if (!get_index(tcs->masterfd, idxid, &idx, true))
1486         return false;
1487
1488     return retrieve(tcs, &idx, tag, buf, size);
1489 }
1490
1491 static bool update_master_header(void)
1492 {
1493     struct master_header myhdr;
1494     int fd;
1495
1496     if (!tc_stat.ready)
1497         return false;
1498
1499     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1500         return false;
1501
1502     myhdr.serial = current_tcmh.serial;
1503     myhdr.commitid = current_tcmh.commitid;
1504     myhdr.dirty = current_tcmh.dirty;
1505
1506     /* Write it back */
1507     lseek(fd, 0, SEEK_SET);
1508     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1509     close(fd);
1510
1511 #ifdef HAVE_TC_RAMCACHE
1512     if (hdr)
1513     {
1514         hdr->h.serial = current_tcmh.serial;
1515         hdr->h.commitid = current_tcmh.commitid;
1516         hdr->h.dirty = current_tcmh.dirty;
1517     }
1518 #endif
1519
1520     return true;
1521 }
1522
1523 #if 0
1524
1525 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1526 {
1527     struct tagentry *entry;
1528
1529     if (tcs->type != tag_title)
1530         return ;
1531
1532     /* We will need reserve buffer for this. */
1533     if (tcs->ramcache)
1534     {
1535         struct tagfile_entry *ep;
1536
1537         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1538         tcs->seek_list[tcs->seek_list_count];
1539     }
1540
1541     entry = find_entry_ram();
1542
1543 }
1544 #endif
1545
1546 void tagcache_search_finish(struct tagcache_search *tcs)
1547 {
1548     int i;
1549
1550     if (!tcs->initialized)
1551         return;
1552
1553     if (tcs->masterfd >= 0)
1554     {
1555         close(tcs->masterfd);
1556         tcs->masterfd = -1;
1557     }
1558
1559     for (i = 0; i < TAG_COUNT; i++)
1560     {
1561         if (tcs->idxfd[i] >= 0)
1562         {
1563             close(tcs->idxfd[i]);
1564             tcs->idxfd[i] = -1;
1565         }
1566     }
1567
1568     tcs->ramsearch = false;
1569     tcs->valid = false;
1570     tcs->initialized = 0;
1571     if (write_lock > 0)
1572         write_lock--;
1573 }
1574
1575 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1576 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1577 {
1578     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1579 }
1580
1581 static long get_tag_numeric(const struct index_entry *entry, int tag)
1582 {
1583     return check_virtual_tags(tag, entry);
1584 }
1585
1586 static char* get_tag_string(const struct index_entry *entry, int tag)
1587 {
1588     char* s = get_tag(entry, tag)->tag_data;
1589     return strcmp(s, UNTAGGED) ? s : NULL;
1590 }
1591
1592 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1593 {
1594     struct index_entry *entry;
1595     int idx_id;
1596
1597     if (!tc_stat.ready || !tc_stat.ramcache)
1598         return false;
1599
1600     /* Find the corresponding entry in tagcache. */
1601     idx_id = find_entry_ram(filename, NULL);
1602     if (idx_id < 0)
1603         return false;
1604
1605     entry = &hdr->indices[idx_id];
1606
1607     id3->title        = get_tag_string(entry, tag_title);
1608     id3->artist       = get_tag_string(entry, tag_artist);
1609     id3->album        = get_tag_string(entry, tag_album);
1610     id3->genre_string = get_tag_string(entry, tag_genre);
1611     id3->composer     = get_tag_string(entry, tag_composer);
1612     id3->comment      = get_tag_string(entry, tag_comment);
1613     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1614     id3->grouping     = get_tag_string(entry, tag_grouping);
1615
1616     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1617     id3->rating     = get_tag_numeric(entry, tag_rating);
1618     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1619     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1620     id3->year       = get_tag_numeric(entry, tag_year);
1621
1622     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1623     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1624     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1625     if (id3->bitrate == 0)
1626         id3->bitrate = 1;
1627
1628     return true;
1629 }
1630 #endif
1631
1632 static inline void write_item(const char *item)
1633 {
1634     int len = strlen(item) + 1;
1635
1636     data_size += len;
1637     write(cachefd, item, len);
1638 }
1639
1640 static int check_if_empty(char **tag)
1641 {
1642     int length;
1643
1644     if (*tag == NULL || **tag == '\0')
1645     {
1646         *tag = UNTAGGED;
1647         return sizeof(UNTAGGED); /* Tag length */
1648     }
1649
1650     length = strlen(*tag);
1651     if (length > TAG_MAXLEN)
1652     {
1653         logf("over length tag: %s", *tag);
1654         length = TAG_MAXLEN;
1655         (*tag)[length] = '\0';
1656     }
1657
1658     return length + 1;
1659 }
1660
1661 #define ADD_TAG(entry,tag,data) \
1662     /* Adding tag */ \
1663     entry.tag_offset[tag] = offset; \
1664     entry.tag_length[tag] = check_if_empty(data); \
1665     offset += entry.tag_length[tag]
1666 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1667  * idea, as it uses lots of stack and is called from a recursive function
1668  * (check_dir).
1669  */
1670 static void __attribute__ ((noinline)) add_tagcache(char *path,
1671                                                     unsigned long mtime
1672 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1673                                                     ,const struct dirent *dc
1674 #endif
1675                                                    )
1676 {
1677     struct mp3entry id3;
1678     struct temp_file_entry entry;
1679     bool ret;
1680     int fd;
1681     int idx_id = -1;
1682     char tracknumfix[3];
1683     int offset = 0;
1684     int path_length = strlen(path);
1685     bool has_albumartist;
1686     bool has_grouping;
1687
1688 #ifdef SIMULATOR
1689     /* Crude logging for the sim - to aid in debugging */
1690     int logfd = open(ROCKBOX_DIR "/database.log",
1691                      O_WRONLY | O_APPEND | O_CREAT);
1692     if (logfd >= 0) {
1693         write(logfd, path, strlen(path));
1694         write(logfd, "\n", 1);
1695         close(logfd);
1696     }
1697 #endif
1698
1699     if (cachefd < 0)
1700         return ;
1701
1702     /* Check for overlength file path. */
1703     if (path_length > TAG_MAXLEN)
1704     {
1705         /* Path can't be shortened. */
1706         logf("Too long path: %s", path);
1707         return ;
1708     }
1709
1710     /* Check if the file is supported. */
1711     if (probe_file_format(path) == AFMT_UNKNOWN)
1712         return ;
1713
1714     /* Check if the file is already cached. */
1715 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1716     if (tc_stat.ramcache && is_dircache_intact())
1717     {
1718         idx_id = find_entry_ram(path, dc);
1719     }
1720 #endif
1721
1722     /* Be sure the entry doesn't exist. */
1723     if (filenametag_fd >= 0 && idx_id < 0)
1724         idx_id = find_entry_disk(path, false);
1725
1726     /* Check if file has been modified. */
1727     if (idx_id >= 0)
1728     {
1729         struct index_entry idx;
1730
1731         /* TODO: Mark that the index exists (for fast reverse scan) */
1732         //found_idx[idx_id/8] |= idx_id%8;
1733
1734         if (!get_index(-1, idx_id, &idx, true))
1735         {
1736             logf("failed to retrieve index entry");
1737             return ;
1738         }
1739
1740         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1741         {
1742             /* No changes to file. */
1743             return ;
1744         }
1745
1746         /* Metadata might have been changed. Delete the entry. */
1747         logf("Re-adding: %s", path);
1748         if (!delete_entry(idx_id))
1749         {
1750             logf("delete_entry failed: %d", idx_id);
1751             return ;
1752         }
1753     }
1754
1755     fd = open(path, O_RDONLY);
1756     if (fd < 0)
1757     {
1758         logf("open fail: %s", path);
1759         return ;
1760     }
1761
1762     memset(&id3, 0, sizeof(struct mp3entry));
1763     memset(&entry, 0, sizeof(struct temp_file_entry));
1764     memset(&tracknumfix, 0, sizeof(tracknumfix));
1765     ret = get_metadata(&id3, fd, path);
1766     close(fd);
1767
1768     if (!ret)
1769         return ;
1770
1771     logf("-> %s", path);
1772
1773     /* Generate track number if missing. */
1774     if (id3.tracknum <= 0)
1775     {
1776         const char *p = strrchr(path, '.');
1777
1778         if (p == NULL)
1779             p = &path[strlen(path)-1];
1780
1781         while (*p != '/')
1782         {
1783             if (isdigit(*p) && isdigit(*(p-1)))
1784             {
1785                 tracknumfix[1] = *p--;
1786                 tracknumfix[0] = *p;
1787                 break;
1788             }
1789             p--;
1790         }
1791
1792         if (tracknumfix[0] != '\0')
1793         {
1794             id3.tracknum = atoi(tracknumfix);
1795             /* Set a flag to indicate track number has been generated. */
1796             entry.flag |= FLAG_TRKNUMGEN;
1797         }
1798         else
1799         {
1800             /* Unable to generate track number. */
1801             id3.tracknum = -1;
1802         }
1803     }
1804
1805     /* Numeric tags */
1806     entry.tag_offset[tag_year] = id3.year;
1807     entry.tag_offset[tag_discnumber] = id3.discnum;
1808     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1809     entry.tag_offset[tag_length] = id3.length;
1810     entry.tag_offset[tag_bitrate] = id3.bitrate;
1811     entry.tag_offset[tag_mtime] = mtime;
1812
1813     /* String tags. */
1814     has_albumartist = id3.albumartist != NULL
1815         && strlen(id3.albumartist) > 0;
1816     has_grouping = id3.grouping != NULL
1817         && strlen(id3.grouping) > 0;
1818
1819     ADD_TAG(entry, tag_filename, &path);
1820     ADD_TAG(entry, tag_title, &id3.title);
1821     ADD_TAG(entry, tag_artist, &id3.artist);
1822     ADD_TAG(entry, tag_album, &id3.album);
1823     ADD_TAG(entry, tag_genre, &id3.genre_string);
1824     ADD_TAG(entry, tag_composer, &id3.composer);
1825     ADD_TAG(entry, tag_comment, &id3.comment);
1826     if (has_albumartist)
1827     {
1828         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1829     }
1830     else
1831     {
1832         ADD_TAG(entry, tag_albumartist, &id3.artist);
1833     }
1834     if (has_grouping)
1835     {
1836         ADD_TAG(entry, tag_grouping, &id3.grouping);
1837     }
1838     else
1839     {
1840         ADD_TAG(entry, tag_grouping, &id3.title);
1841     }
1842     entry.data_length = offset;
1843
1844     /* Write the header */
1845     write(cachefd, &entry, sizeof(struct temp_file_entry));
1846
1847     /* And tags also... Correct order is critical */
1848     write_item(path);
1849     write_item(id3.title);
1850     write_item(id3.artist);
1851     write_item(id3.album);
1852     write_item(id3.genre_string);
1853     write_item(id3.composer);
1854     write_item(id3.comment);
1855     if (has_albumartist)
1856     {
1857         write_item(id3.albumartist);
1858     }
1859     else
1860     {
1861         write_item(id3.artist);
1862     }
1863     if (has_grouping)
1864     {
1865         write_item(id3.grouping);
1866     }
1867     else
1868     {
1869         write_item(id3.title);
1870     }
1871     total_entry_count++;
1872 }
1873
1874 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1875 {
1876     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1877     int len = strlen(str)+1;
1878     int i;
1879     unsigned crc32;
1880     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1881     char buf[TAG_MAXLEN+32];
1882
1883     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1884         buf[i] = tolower(str[i]);
1885     buf[i] = '\0';
1886
1887     crc32 = crc_32(buf, i, 0xffffffff);
1888
1889     if (unique)
1890     {
1891         /* Check if the crc does not exist -> entry does not exist for sure. */
1892         for (i = 0; i < tempbufidx; i++)
1893         {
1894             if (crcbuf[-i] != crc32)
1895                 continue;
1896
1897             if (!strcasecmp(str, index[i].str))
1898             {
1899                 if (id < 0 || id >= lookup_buffer_depth)
1900                 {
1901                     logf("lookup buf overf.: %d", id);
1902                     return false;
1903                 }
1904
1905                 lookup[id] = &index[i];
1906                 return true;
1907             }
1908         }
1909     }
1910
1911     /* Insert to CRC buffer. */
1912     crcbuf[-tempbufidx] = crc32;
1913     tempbuf_left -= 4;
1914
1915     /* Insert it to the buffer. */
1916     tempbuf_left -= len;
1917     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1918         return false;
1919
1920     if (id >= lookup_buffer_depth)
1921     {
1922         logf("lookup buf overf. #2: %d", id);
1923         return false;
1924     }
1925
1926     if (id >= 0)
1927     {
1928         lookup[id] = &index[tempbufidx];
1929         index[tempbufidx].idlist.id = id;
1930     }
1931     else
1932         index[tempbufidx].idlist.id = -1;
1933
1934     index[tempbufidx].idlist.next = NULL;
1935     index[tempbufidx].idx_id = idx_id;
1936     index[tempbufidx].seek = -1;
1937     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1938     memcpy(index[tempbufidx].str, str, len);
1939     tempbuf_pos += len;
1940     tempbufidx++;
1941
1942     return true;
1943 }
1944
1945 static int compare(const void *p1, const void *p2)
1946 {
1947     do_timed_yield();
1948
1949     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1950     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1951
1952     if (strcmp(e1->str, UNTAGGED) == 0)
1953     {
1954         if (strcmp(e2->str, UNTAGGED) == 0)
1955             return 0;
1956         return -1;
1957     }
1958     else if (strcmp(e2->str, UNTAGGED) == 0)
1959         return 1;
1960
1961     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1962 }
1963
1964 static int tempbuf_sort(int fd)
1965 {
1966     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1967     struct tagfile_entry fe;
1968     int i;
1969     int length;
1970
1971     /* Generate reverse lookup entries. */
1972     for (i = 0; i < lookup_buffer_depth; i++)
1973     {
1974         struct tempbuf_id_list *idlist;
1975
1976         if (!lookup[i])
1977             continue;
1978
1979         if (lookup[i]->idlist.id == i)
1980             continue;
1981
1982         idlist = &lookup[i]->idlist;
1983         while (idlist->next != NULL)
1984             idlist = idlist->next;
1985
1986         tempbuf_left -= sizeof(struct tempbuf_id_list);
1987         if (tempbuf_left - 4 < 0)
1988             return -1;
1989
1990         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1991         if (tempbuf_pos & 0x03)
1992         {
1993             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1994             tempbuf_left -= 3;
1995             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1996         }
1997         tempbuf_pos += sizeof(struct tempbuf_id_list);
1998
1999         idlist = idlist->next;
2000         idlist->id = i;
2001         idlist->next = NULL;
2002
2003         do_timed_yield();
2004     }
2005
2006     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2007     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2008
2009     for (i = 0; i < tempbufidx; i++)
2010     {
2011         struct tempbuf_id_list *idlist = &index[i].idlist;
2012
2013         /* Fix the lookup list. */
2014         while (idlist != NULL)
2015         {
2016             if (idlist->id >= 0)
2017                 lookup[idlist->id] = &index[i];
2018             idlist = idlist->next;
2019         }
2020
2021         index[i].seek = lseek(fd, 0, SEEK_CUR);
2022         length = strlen(index[i].str) + 1;
2023         fe.tag_length = length;
2024         fe.idx_id = index[i].idx_id;
2025
2026         /* Check the chunk alignment. */
2027         if ((fe.tag_length + sizeof(struct tagfile_entry))
2028             % TAGFILE_ENTRY_CHUNK_LENGTH)
2029         {
2030             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2031                 ((fe.tag_length + sizeof(struct tagfile_entry))
2032                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2033         }
2034
2035 #ifdef TAGCACHE_STRICT_ALIGN
2036         /* Make sure the entry is long aligned. */
2037         if (index[i].seek & 0x03)
2038         {
2039             logf("tempbuf_sort: alignment error!");
2040             return -3;
2041         }
2042 #endif
2043
2044         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2045             sizeof(struct tagfile_entry))
2046         {
2047             logf("tempbuf_sort: write error #1");
2048             return -1;
2049         }
2050
2051         if (write(fd, index[i].str, length) != length)
2052         {
2053             logf("tempbuf_sort: write error #2");
2054             return -2;
2055         }
2056
2057         /* Write some padding. */
2058         if (fe.tag_length - length > 0)
2059             write(fd, "XXXXXXXX", fe.tag_length - length);
2060     }
2061
2062     return i;
2063 }
2064
2065 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2066 {
2067     if (id < 0 || id >= lookup_buffer_depth)
2068         return NULL;
2069
2070     return lookup[id];
2071 }
2072
2073
2074 inline static int tempbuf_find_location(int id)
2075 {
2076     struct tempbuf_searchidx *entry;
2077
2078     entry = tempbuf_locate(id);
2079     if (entry == NULL)
2080         return -1;
2081
2082     return entry->seek;
2083 }
2084
2085 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2086 {
2087     struct master_header tcmh;
2088     struct index_entry idx;
2089     int masterfd;
2090     int masterfd_pos;
2091     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2092     int max_entries;
2093     int entries_processed = 0;
2094     int i, j;
2095     char buf[TAG_MAXLEN];
2096
2097     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2098
2099     logf("Building numeric indices...");
2100     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2101
2102     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2103         return false;
2104
2105     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2106                          SEEK_CUR);
2107     if (masterfd_pos == filesize(masterfd))
2108     {
2109         logf("we can't append!");
2110         close(masterfd);
2111         return false;
2112     }
2113
2114     while (entries_processed < h->entry_count)
2115     {
2116         int count = MIN(h->entry_count - entries_processed, max_entries);
2117
2118         /* Read in as many entries as possible. */
2119         for (i = 0; i < count; i++)
2120         {
2121             struct temp_file_entry *tfe = &entrybuf[i];
2122             int datastart;
2123
2124             /* Read in numeric data. */
2125             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2126                 sizeof(struct temp_file_entry))
2127             {
2128                 logf("read fail #1");
2129                 close(masterfd);
2130                 return false;
2131             }
2132
2133             datastart = lseek(tmpfd, 0, SEEK_CUR);
2134
2135             /**
2136              * Read string data from the following tags:
2137              * - tag_filename
2138              * - tag_artist
2139              * - tag_album
2140              * - tag_title
2141              *
2142              * A crc32 hash is calculated from the read data
2143              * and stored back to the data offset field kept in memory.
2144              */
2145 #define tmpdb_read_string_tag(tag) \
2146     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2147     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2148     { \
2149         logf("read fail: buffer overflow"); \
2150         close(masterfd); \
2151         return false; \
2152     } \
2153     \
2154     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2155         tfe->tag_length[tag]) \
2156     { \
2157         logf("read fail #2"); \
2158         close(masterfd); \
2159         return false; \
2160     } \
2161     \
2162     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2163     lseek(tmpfd, datastart, SEEK_SET)
2164
2165             tmpdb_read_string_tag(tag_filename);
2166             tmpdb_read_string_tag(tag_artist);
2167             tmpdb_read_string_tag(tag_album);
2168             tmpdb_read_string_tag(tag_title);
2169
2170             /* Seek to the end of the string data. */
2171             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2172         }
2173
2174         /* Backup the master index position. */
2175         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2176         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2177
2178         /* Check if we can resurrect some deleted runtime statistics data. */
2179         for (i = 0; i < tcmh.tch.entry_count; i++)
2180         {
2181             /* Read the index entry. */
2182             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2183                 != sizeof(struct index_entry))
2184             {
2185                 logf("read fail #3");
2186                 close(masterfd);
2187                 return false;
2188             }
2189
2190             /**
2191              * Skip unless the entry is marked as being deleted
2192              * or the data has already been resurrected.
2193              */
2194             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2195                 continue;
2196
2197             /* Now try to match the entry. */
2198             /**
2199              * To succesfully match a song, the following conditions
2200              * must apply:
2201              *
2202              * For numeric fields: tag_length
2203              * - Full identical match is required
2204              *
2205              * If tag_filename matches, no further checking necessary.
2206              *
2207              * For string hashes: tag_artist, tag_album, tag_title
2208              * - Two of these must match
2209              */
2210             for (j = 0; j < count; j++)
2211             {
2212                 struct temp_file_entry *tfe = &entrybuf[j];
2213
2214                 /* Try to match numeric fields first. */
2215                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2216                     continue;
2217
2218                 /* Now it's time to do the hash matching. */
2219                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2220                 {
2221                     int match_count = 0;
2222
2223                     /* No filename match, check if we can match two other tags. */
2224 #define tmpdb_match(tag) \
2225     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2226         match_count++
2227
2228                     tmpdb_match(tag_artist);
2229                     tmpdb_match(tag_album);
2230                     tmpdb_match(tag_title);
2231
2232                     if (match_count < 2)
2233                     {
2234                         /* Still no match found, give up. */
2235                         continue;
2236                     }
2237                 }
2238
2239                 /* A match found, now copy & resurrect the statistical data. */
2240 #define tmpdb_copy_tag(tag) \
2241     tfe->tag_offset[tag] = idx.tag_seek[tag]
2242
2243                 tmpdb_copy_tag(tag_playcount);
2244                 tmpdb_copy_tag(tag_rating);
2245                 tmpdb_copy_tag(tag_playtime);
2246                 tmpdb_copy_tag(tag_lastplayed);
2247                 tmpdb_copy_tag(tag_commitid);
2248
2249                 /* Avoid processing this entry again. */
2250                 idx.flag |= FLAG_RESURRECTED;
2251
2252                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2253                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2254                     != sizeof(struct index_entry))
2255                 {
2256                     logf("masterfd writeback fail #1");
2257                     close(masterfd);
2258                     return false;
2259                 }
2260
2261                 logf("Entry resurrected");
2262             }
2263         }
2264
2265
2266         /* Restore the master index position. */
2267         lseek(masterfd, masterfd_pos, SEEK_SET);
2268
2269         /* Commit the data to the index. */
2270         for (i = 0; i < count; i++)
2271         {
2272             int loc = lseek(masterfd, 0, SEEK_CUR);
2273
2274             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2275                 != sizeof(struct index_entry))
2276             {
2277                 logf("read fail #3");
2278                 close(masterfd);
2279                 return false;
2280             }
2281
2282             for (j = 0; j < TAG_COUNT; j++)
2283             {
2284                 if (!TAGCACHE_IS_NUMERIC(j))
2285                     continue;
2286
2287                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2288             }
2289             idx.flag = entrybuf[i].flag;
2290
2291             if (idx.tag_seek[tag_commitid])
2292             {
2293                 /* Data has been resurrected. */
2294                 idx.flag |= FLAG_DIRTYNUM;
2295             }
2296             else if (tc_stat.ready && current_tcmh.commitid > 0)
2297             {
2298                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2299                 idx.flag |= FLAG_DIRTYNUM;
2300             }
2301
2302             /* Write back the updated index. */
2303             lseek(masterfd, loc, SEEK_SET);
2304             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2305                 != sizeof(struct index_entry))
2306             {
2307                 logf("write fail");
2308                 close(masterfd);
2309                 return false;
2310             }
2311         }
2312
2313         entries_processed += count;
2314         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2315     }
2316
2317     close(masterfd);
2318
2319     return true;
2320 }
2321
2322 /**
2323  * Return values:
2324  *     > 0   success
2325  *    == 0   temporary failure
2326  *     < 0   fatal error
2327  */
2328 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2329 {
2330     int i;
2331     struct tagcache_header tch;
2332     struct master_header   tcmh;
2333     struct index_entry idxbuf[IDX_BUF_DEPTH];
2334     int idxbuf_pos;
2335     char buf[TAG_MAXLEN+32];
2336     int fd = -1, masterfd;
2337     bool error = false;
2338     int init;
2339     int masterfd_pos;
2340
2341     logf("Building index: %d", index_type);
2342
2343     /* Check the number of entries we need to allocate ram for. */
2344     commit_entry_count = h->entry_count + 1;
2345
2346     masterfd = open_master_fd(&tcmh, false);
2347     if (masterfd >= 0)
2348     {
2349         commit_entry_count += tcmh.tch.entry_count;
2350         close(masterfd);
2351     }
2352     else
2353         remove_files(); /* Just to be sure we are clean. */
2354
2355     /* Open the index file, which contains the tag names. */
2356     fd = open_tag_fd(&tch, index_type, true);
2357     if (fd >= 0)
2358     {
2359         logf("tch.datasize=%ld", tch.datasize);
2360         lookup_buffer_depth = 1 +
2361         /* First part */ commit_entry_count +
2362         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2363     }
2364     else
2365     {
2366         lookup_buffer_depth = 1 +
2367         /* First part */ commit_entry_count +
2368         /* Second part */ 0;
2369     }
2370
2371     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2372     logf("commit_entry_count=%ld", commit_entry_count);
2373
2374     /* Allocate buffer for all index entries from both old and new
2375      * tag files. */
2376     tempbufidx = 0;
2377     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2378
2379     /* Allocate lookup buffer. The first portion of commit_entry_count
2380      * contains the new tags in the temporary file and the second
2381      * part for locating entries already in the db.
2382      *
2383      *  New tags  Old tags
2384      * +---------+---------------------------+
2385      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2386      * +---------+---------------------------+
2387      *
2388      * Old tags are inserted to a temporary buffer with position:
2389      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2390      * And new tags with index:
2391      *     tempbuf_insert(idx, ...);
2392      *
2393      * The buffer is sorted and written into tag file:
2394      *     tempbuf_sort(...);
2395      * leaving master index locations messed up.
2396      *
2397      * That is fixed using the lookup buffer for old tags:
2398      *     new_seek = tempbuf_find_location(old_seek, ...);
2399      * and for new tags:
2400      *     new_seek = tempbuf_find_location(idx);
2401      */
2402     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2403     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2404     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2405
2406     /* And calculate the remaining data space used mainly for storing
2407      * tag data (strings). */
2408     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2409     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2410     {
2411         logf("Buffer way too small!");
2412         return 0;
2413     }
2414
2415     if (fd >= 0)
2416     {
2417         /**
2418          * If tag file contains unique tags (sorted index), we will load
2419          * it entirely into memory so we can resort it later for use with
2420          * chunked browsing.
2421          */
2422         if (TAGCACHE_IS_SORTED(index_type))
2423         {
2424             logf("loading tags...");
2425             for (i = 0; i < tch.entry_count; i++)
2426             {
2427                 struct tagfile_entry entry;
2428                 int loc = lseek(fd, 0, SEEK_CUR);
2429                 bool ret;
2430
2431                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2432                     != sizeof(struct tagfile_entry))
2433                 {
2434                     logf("read error #7");
2435                     close(fd);
2436                     return -2;
2437                 }
2438
2439                 if (entry.tag_length >= (int)sizeof(buf))
2440                 {
2441                     logf("too long tag #3");
2442                     close(fd);
2443                     return -2;
2444                 }
2445
2446                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2447                 {
2448                     logf("read error #8");
2449                     close(fd);
2450                     return -2;
2451                 }
2452
2453                 /* Skip deleted entries. */
2454                 if (buf[0] == '\0')
2455                     continue;
2456
2457                 /**
2458                  * Save the tag and tag id in the memory buffer. Tag id
2459                  * is saved so we can later reindex the master lookup
2460                  * table when the index gets resorted.
2461                  */
2462                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2463                                      + commit_entry_count, entry.idx_id,
2464                                      TAGCACHE_IS_UNIQUE(index_type));
2465                 if (!ret)
2466                 {
2467                     close(fd);
2468                     return -3;
2469                 }
2470                 do_timed_yield();
2471             }
2472             logf("done");
2473         }
2474         else
2475             tempbufidx = tch.entry_count;
2476     }
2477     else
2478     {
2479         /**
2480          * Creating new index file to store the tags. No need to preload
2481          * anything whether the index type is sorted or not.
2482          */
2483         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2484         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2485         if (fd < 0)
2486         {
2487             logf("%s open fail", buf);
2488             return -2;
2489         }
2490
2491         tch.magic = TAGCACHE_MAGIC;
2492         tch.entry_count = 0;
2493         tch.datasize = 0;
2494
2495         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2496             != sizeof(struct tagcache_header))
2497         {
2498             logf("header write failed");
2499             close(fd);
2500             return -2;
2501         }
2502     }
2503
2504     /* Loading the tag lookup file as "master file". */
2505     logf("Loading index file");
2506     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2507
2508     if (masterfd < 0)
2509     {
2510         logf("Creating new DB");
2511         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2512
2513         if (masterfd < 0)
2514         {
2515             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2516             close(fd);
2517             return -2;
2518         }
2519
2520         /* Write the header (write real values later). */
2521         memset(&tcmh, 0, sizeof(struct master_header));
2522         tcmh.tch = *h;
2523         tcmh.tch.entry_count = 0;
2524         tcmh.tch.datasize = 0;
2525         tcmh.dirty = true;
2526         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2527         init = true;
2528         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2529     }
2530     else
2531     {
2532         /**
2533          * Master file already exists so we need to process the current
2534          * file first.
2535          */
2536         init = false;
2537
2538         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2539             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2540         {
2541             logf("header error");
2542             close(fd);
2543             close(masterfd);
2544             return -2;
2545         }
2546
2547         /**
2548          * If we reach end of the master file, we need to expand it to
2549          * hold new tags. If the current index is not sorted, we can
2550          * simply append new data to end of the file.
2551          * However, if the index is sorted, we need to update all tag
2552          * pointers in the master file for the current index.
2553          */
2554         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2555             SEEK_CUR);
2556         if (masterfd_pos == filesize(masterfd))
2557         {
2558             logf("appending...");
2559             init = true;
2560         }
2561     }
2562
2563     /**
2564      * Load new unique tags in memory to be sorted later and added
2565      * to the master lookup file.
2566      */
2567     if (TAGCACHE_IS_SORTED(index_type))
2568     {
2569         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2570         /* h is the header of the temporary file containing new tags. */
2571         logf("inserting new tags...");
2572         for (i = 0; i < h->entry_count; i++)
2573         {
2574             struct temp_file_entry entry;
2575
2576             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2577                 sizeof(struct temp_file_entry))
2578             {
2579                 logf("read fail #3");
2580                 error = true;
2581                 goto error_exit;
2582             }
2583
2584             /* Read data. */
2585             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2586             {
2587                 logf("too long entry!");
2588                 error = true;
2589                 goto error_exit;
2590             }
2591
2592             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2593             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2594                 entry.tag_length[index_type])
2595             {
2596                 logf("read fail #4");
2597                 error = true;
2598                 goto error_exit;
2599             }
2600
2601             if (TAGCACHE_IS_UNIQUE(index_type))
2602                 error = !tempbuf_insert(buf, i, -1, true);
2603             else
2604                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2605
2606             if (error)
2607             {
2608                 logf("insert error");
2609                 goto error_exit;
2610             }
2611
2612             /* Skip to next. */
2613             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2614                     entry.tag_length[index_type], SEEK_CUR);
2615             do_timed_yield();
2616         }
2617         logf("done");
2618
2619         /* Sort the buffer data and write it to the index file. */
2620         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2621         /**
2622          * We need to truncate the index file now. There can be junk left
2623          * at the end of file (however, we _should_ always follow the
2624          * entry_count and don't crash with that).
2625          */
2626         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2627
2628         i = tempbuf_sort(fd);
2629         if (i < 0)
2630             goto error_exit;
2631         logf("sorted %d tags", i);
2632
2633         /**
2634          * Now update all indexes in the master lookup file.
2635          */
2636         logf("updating indices...");
2637         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2638         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2639         {
2640             int j;
2641             int loc = lseek(masterfd, 0, SEEK_CUR);
2642
2643             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2644
2645             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2646                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2647             {
2648                 logf("read fail #5");
2649                 error = true;
2650                 goto error_exit ;
2651             }
2652             lseek(masterfd, loc, SEEK_SET);
2653
2654             for (j = 0; j < idxbuf_pos; j++)
2655             {
2656                 if (idxbuf[j].flag & FLAG_DELETED)
2657                 {
2658                     /* We can just ignore deleted entries. */
2659                     // idxbuf[j].tag_seek[index_type] = 0;
2660                     continue;
2661                 }
2662
2663                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2664                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2665                     + commit_entry_count);
2666
2667                 if (idxbuf[j].tag_seek[index_type] < 0)
2668                 {
2669                     logf("update error: %d/%d/%d",
2670                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2671                     error = true;
2672                     goto error_exit;
2673                 }
2674
2675                 do_timed_yield();
2676             }
2677
2678             /* Write back the updated index. */
2679             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2680                         index_entry_ec, tc_stat.econ) !=
2681                 (int)sizeof(struct index_entry)*idxbuf_pos)
2682             {
2683                 logf("write fail");
2684                 error = true;
2685                 goto error_exit;
2686             }
2687         }
2688         logf("done");
2689     }
2690
2691     /**
2692      * Walk through the temporary file containing the new tags.
2693      */
2694     // build_normal_index(h, tmpfd, masterfd, idx);
2695     logf("updating new indices...");
2696     lseek(masterfd, masterfd_pos, SEEK_SET);
2697     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2698     lseek(fd, 0, SEEK_END);
2699     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2700     {
2701         int j;
2702
2703         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2704         if (init)
2705         {
2706             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2707         }
2708         else
2709         {
2710             int loc = lseek(masterfd, 0, SEEK_CUR);
2711
2712             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2713                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2714             {
2715                 logf("read fail #6");
2716                 error = true;
2717                 break ;
2718             }
2719             lseek(masterfd, loc, SEEK_SET);
2720         }
2721
2722         /* Read entry headers. */
2723         for (j = 0; j < idxbuf_pos; j++)
2724         {
2725             if (!TAGCACHE_IS_SORTED(index_type))
2726             {
2727                 struct temp_file_entry entry;
2728                 struct tagfile_entry fe;
2729
2730                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2731                     sizeof(struct temp_file_entry))
2732                 {
2733                     logf("read fail #7");
2734                     error = true;
2735                     break ;
2736                 }
2737
2738                 /* Read data. */
2739                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2740                 {
2741                     logf("too long entry!");
2742                     logf("length=%d", entry.tag_length[index_type]);
2743                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2744                     error = true;
2745                     break ;
2746                 }
2747
2748                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2749                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2750                     entry.tag_length[index_type])
2751                 {
2752                     logf("read fail #8");
2753                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2754                     logf("length=0x%02x", entry.tag_length[index_type]);
2755                     error = true;
2756                     break ;
2757                 }
2758
2759                 /* Write to index file. */
2760                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2761                 fe.tag_length = entry.tag_length[index_type];
2762                 fe.idx_id = tcmh.tch.entry_count + i + j;
2763                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2764                 write(fd, buf, fe.tag_length);
2765                 tempbufidx++;
2766
2767                 /* Skip to next. */
2768                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2769                       entry.tag_length[index_type], SEEK_CUR);
2770             }
2771             else
2772             {
2773                 /* Locate the correct entry from the sorted array. */
2774                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2775                 if (idxbuf[j].tag_seek[index_type] < 0)
2776                 {
2777                     logf("entry not found (%d)", j);
2778                     error = true;
2779                     break ;
2780                 }
2781             }
2782         }
2783
2784         /* Write index. */
2785         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2786                   index_entry_ec, tc_stat.econ) !=
2787             (int)sizeof(struct index_entry)*idxbuf_pos)
2788         {
2789             logf("tagcache: write fail #4");
2790             error = true;
2791             break ;
2792         }
2793
2794         do_timed_yield();
2795     }
2796     logf("done");
2797
2798     /* Finally write the header. */
2799     tch.magic = TAGCACHE_MAGIC;
2800     tch.entry_count = tempbufidx;
2801     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2802     lseek(fd, 0, SEEK_SET);
2803     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2804
2805     if (index_type != tag_filename)
2806         h->datasize += tch.datasize;
2807     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2808     error_exit:
2809
2810     close(fd);
2811     close(masterfd);
2812
2813     if (error)
2814         return -2;
2815
2816     return 1;
2817 }
2818
2819 static bool commit(void)
2820 {
2821     struct tagcache_header tch;
2822     struct master_header   tcmh;
2823     int i, len, rc;
2824     int tmpfd;
2825     int masterfd;
2826 #ifdef HAVE_DIRCACHE
2827     bool dircache_buffer_stolen = false;
2828 #endif
2829     bool local_allocation = false;
2830
2831     logf("committing tagcache");
2832
2833     while (write_lock)
2834         sleep(1);
2835
2836     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2837     if (tmpfd < 0)
2838     {
2839         logf("nothing to commit");
2840         return true;
2841     }
2842
2843
2844     /* Load the header. */
2845     len = sizeof(struct tagcache_header);
2846     rc = read(tmpfd, &tch, len);
2847
2848     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2849     {
2850         logf("incorrect header");
2851         close(tmpfd);
2852         remove(TAGCACHE_FILE_TEMP);
2853         return false;
2854     }
2855
2856     if (tch.entry_count == 0)
2857     {
2858         logf("nothing to commit");
2859         close(tmpfd);
2860         remove(TAGCACHE_FILE_TEMP);
2861         return true;
2862     }
2863
2864 #ifdef HAVE_EEPROM_SETTINGS
2865     remove(TAGCACHE_STATEFILE);
2866 #endif
2867
2868     /* At first be sure to unload the ramcache! */
2869 #ifdef HAVE_TC_RAMCACHE
2870     tc_stat.ramcache = false;
2871 #endif
2872
2873     read_lock++;
2874
2875     /* Try to steal every buffer we can :) */
2876     if (tempbuf_size == 0)
2877         local_allocation = true;
2878
2879 #ifdef HAVE_DIRCACHE
2880     if (tempbuf_size == 0)
2881     {
2882         /* Try to steal the dircache buffer. */
2883         tempbuf = dircache_steal_buffer(&tempbuf_size);
2884         tempbuf_size &= ~0x03;
2885
2886         if (tempbuf_size > 0)
2887         {
2888             dircache_buffer_stolen = true;
2889         }
2890     }
2891 #endif
2892
2893 #ifdef HAVE_TC_RAMCACHE
2894     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2895     {
2896         tempbuf = (char *)(hdr + 1);
2897         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2898         tempbuf_size &= ~0x03;
2899     }
2900 #endif
2901
2902     /* And finally fail if there are no buffers available. */
2903     if (tempbuf_size == 0)
2904     {
2905         logf("delaying commit until next boot");
2906         tc_stat.commit_delayed = true;
2907         close(tmpfd);
2908         read_lock--;
2909         return false;
2910     }
2911
2912     logf("commit %ld entries...", tch.entry_count);
2913
2914     /* Mark DB dirty so it will stay disabled if commit fails. */
2915     current_tcmh.dirty = true;
2916     update_master_header();
2917
2918     /* Now create the index files. */
2919     tc_stat.commit_step = 0;
2920     tch.datasize = 0;
2921     tc_stat.commit_delayed = false;
2922
2923     for (i = 0; i < TAG_COUNT; i++)
2924     {
2925         int ret;
2926
2927         if (TAGCACHE_IS_NUMERIC(i))
2928             continue;
2929
2930         tc_stat.commit_step++;
2931         ret = build_index(i, &tch, tmpfd);
2932         if (ret <= 0)
2933         {
2934             close(tmpfd);
2935             logf("tagcache failed init");
2936             if (ret == 0)
2937                 tc_stat.commit_delayed = true;
2938
2939             tc_stat.commit_step = 0;
2940             read_lock--;
2941             return false;
2942         }
2943     }
2944
2945     if (!build_numeric_indices(&tch, tmpfd))
2946     {
2947         logf("Failure to commit numeric indices");
2948         close(tmpfd);
2949         tc_stat.commit_step = 0;
2950         read_lock--;
2951         return false;
2952     }
2953
2954     close(tmpfd);
2955     remove(TAGCACHE_FILE_TEMP);
2956
2957     tc_stat.commit_step = 0;
2958
2959     /* Update the master index headers. */
2960     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2961     {
2962         read_lock--;
2963         return false;
2964     }
2965
2966     tcmh.tch.entry_count += tch.entry_count;
2967     tcmh.tch.datasize = sizeof(struct master_header)
2968         + sizeof(struct index_entry) * tcmh.tch.entry_count
2969         + tch.datasize;
2970     tcmh.dirty = false;
2971     tcmh.commitid++;
2972
2973     lseek(masterfd, 0, SEEK_SET);
2974     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2975     close(masterfd);
2976
2977     logf("tagcache committed");
2978     tc_stat.ready = check_all_headers();
2979     tc_stat.readyvalid = true;
2980
2981     if (local_allocation)
2982     {
2983         tempbuf = NULL;
2984         tempbuf_size = 0;
2985     }
2986
2987 #ifdef HAVE_DIRCACHE
2988     /* Rebuild the dircache, if we stole the buffer. */
2989     if (dircache_buffer_stolen)
2990         dircache_build(0);
2991 #endif
2992
2993 #ifdef HAVE_TC_RAMCACHE
2994     /* Reload tagcache. */
2995     if (tc_stat.ramcache_allocated > 0)
2996         tagcache_start_scan();
2997 #endif
2998
2999     read_lock--;
3000
3001     return true;
3002 }
3003
3004 static void allocate_tempbuf(void)
3005 {
3006     /* Yeah, malloc would be really nice now :) */
3007 #ifdef __PCTOOL__
3008     tempbuf_size = 32*1024*1024;
3009     tempbuf = malloc(tempbuf_size);
3010 #else
3011     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3012     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3013     audiobuf += tempbuf_size;
3014 #endif
3015 }
3016
3017 static void free_tempbuf(void)
3018 {
3019     if (tempbuf_size == 0)
3020         return ;
3021
3022 #ifdef __PCTOOL__
3023     free(tempbuf);
3024 #else
3025     audiobuf -= tempbuf_size;
3026 #endif
3027     tempbuf = NULL;
3028     tempbuf_size = 0;
3029 }
3030
3031 #ifndef __PCTOOL__
3032
3033 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3034 {
3035     struct index_entry idx;
3036
3037     if (!tc_stat.ready)
3038         return false;
3039
3040     if (!TAGCACHE_IS_NUMERIC(tag))
3041         return false;
3042
3043     if (!get_index(masterfd, idx_id, &idx, false))
3044         return false;
3045
3046     idx.tag_seek[tag] = data;
3047     idx.flag |= FLAG_DIRTYNUM;
3048
3049     return write_index(masterfd, idx_id, &idx);
3050 }
3051
3052 #if 0
3053 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3054                                    int tag, long data)
3055 {
3056     struct master_header myhdr;
3057
3058     if (tcs->masterfd < 0)
3059     {
3060         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3061             return false;
3062     }
3063
3064     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3065 }
3066 #endif
3067
3068 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3069
3070 static bool command_queue_is_full(void)
3071 {
3072     int next;
3073
3074     next = command_queue_widx + 1;
3075     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3076         next = 0;
3077
3078     return (next == command_queue_ridx);
3079 }
3080
3081 static void command_queue_sync_callback(void *data)
3082 {
3083     (void)data;
3084     struct master_header myhdr;
3085     int masterfd;
3086
3087     mutex_lock(&command_queue_mutex);
3088
3089     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3090         return;
3091
3092     while (command_queue_ridx != command_queue_widx)
3093     {
3094         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3095
3096         switch (ce->command)
3097         {
3098             case CMD_UPDATE_MASTER_HEADER:
3099             {
3100                 close(masterfd);
3101                 update_master_header();
3102
3103                 /* Re-open the masterfd. */
3104                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3105                     return;
3106
3107                 break;
3108             }
3109             case CMD_UPDATE_NUMERIC:
3110             {
3111                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3112                 break;
3113             }
3114         }
3115
3116         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3117             command_queue_ridx = 0;
3118     }
3119
3120     close(masterfd);
3121
3122     tc_stat.queue_length = 0;
3123     mutex_unlock(&command_queue_mutex);
3124 }
3125
3126 static void run_command_queue(bool force)
3127 {
3128     if (COMMAND_QUEUE_IS_EMPTY)
3129         return;
3130
3131     if (force || command_queue_is_full())
3132         command_queue_sync_callback(NULL);
3133     else
3134         register_storage_idle_func(command_queue_sync_callback);
3135 }
3136
3137 static void queue_command(int cmd, long idx_id, int tag, long data)
3138 {
3139     while (1)
3140     {
3141         int next;
3142
3143         mutex_lock(&command_queue_mutex);
3144         next = command_queue_widx + 1;
3145         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3146             next = 0;
3147
3148         /* Make sure queue is not full. */
3149         if (next != command_queue_ridx)
3150         {
3151             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3152
3153             ce->command = cmd;
3154             ce->idx_id = idx_id;
3155             ce->tag = tag;
3156             ce->data = data;
3157
3158             command_queue_widx = next;
3159
3160             tc_stat.queue_length++;
3161
3162             mutex_unlock(&command_queue_mutex);
3163             break;
3164         }
3165
3166         /* Queue is full, try again later... */
3167         mutex_unlock(&command_queue_mutex);
3168         sleep(1);
3169     }
3170 }
3171
3172 long tagcache_increase_serial(void)
3173 {
3174     long old;
3175
3176     if (!tc_stat.ready)
3177         return -2;
3178
3179     while (read_lock)
3180         sleep(1);
3181
3182     old = current_tcmh.serial++;
3183     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3184
3185     return old;
3186 }
3187
3188 void tagcache_update_numeric(int idx_id, int tag, long data)
3189 {
3190     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3191 }
3192 #endif /* !__PCTOOL__ */
3193
3194 long tagcache_get_serial(void)
3195 {
3196     return current_tcmh.serial;
3197 }
3198
3199 long tagcache_get_commitid(void)
3200 {
3201     return current_tcmh.commitid;
3202 }
3203
3204 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3205 {
3206     char buf[512];
3207     int i;
3208
3209     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3210     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3211     {
3212         if (*datastr == '\0')
3213             break;
3214
3215         if (*datastr == '"' || *datastr == '\\')
3216             buf[i++] = '\\';
3217
3218         buf[i] = *(datastr++);
3219     }
3220
3221     strcpy(&buf[i], "\" ");
3222
3223     write(fd, buf, i + 2);
3224
3225     return true;
3226 }
3227
3228 #ifndef __PCTOOL__
3229
3230 static bool read_tag(char *dest, long size,
3231                      const char *src, const char *tagstr)
3232 {
3233     int pos;
3234     char current_tag[32];
3235
3236     while (*src != '\0')
3237     {
3238         /* Skip all whitespace */
3239         while (*src == ' ')
3240             src++;
3241
3242         if (*src == '\0')
3243             break;
3244
3245         pos = 0;
3246         /* Read in tag name */
3247         while (*src != '=' && *src != ' ')
3248         {
3249             current_tag[pos] = *src;
3250             src++;
3251             pos++;
3252
3253             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3254                 return false;
3255         }
3256         current_tag[pos] = '\0';
3257
3258         /* Read in tag data */
3259
3260         /* Find the start. */
3261         while (*src != '"' && *src != '\0')
3262             src++;
3263
3264         if (*src == '\0' || *(++src) == '\0')
3265             return false;
3266
3267         /* Read the data. */
3268         for (pos = 0; pos < size; pos++)
3269         {
3270             if (*src == '\0')
3271                 break;
3272
3273             if (*src == '\\')
3274             {
3275                 dest[pos] = *(src+1);
3276                 src += 2;
3277                 continue;
3278             }
3279
3280             dest[pos] = *src;
3281
3282             if (*src == '"')
3283             {
3284                 src++;
3285                 break;
3286             }
3287
3288             if (*src == '\0')
3289                 break;
3290
3291             src++;
3292         }
3293         dest[pos] = '\0';
3294
3295         if (!strcasecmp(tagstr, current_tag))
3296             return true;
3297     }
3298
3299     return false;
3300 }
3301
3302 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3303 {
3304     struct index_entry idx;
3305     char tag_data[TAG_MAXLEN+32];
3306     int idx_id;
3307     long masterfd = (long)parameters;
3308     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3309         tag_commitid };
3310     int i;
3311     (void)line_n;
3312
3313     if (*buf == '#')
3314         return 0;
3315
3316     logf("%d/%s", line_n, buf);
3317     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3318     {
3319         logf("filename missing");
3320         logf("-> %s", buf);
3321         return 0;
3322     }
3323
3324     idx_id = find_index(tag_data);
3325     if (idx_id < 0)
3326     {
3327         logf("entry not found");
3328         return 0;
3329     }
3330
3331     if (!get_index(masterfd, idx_id, &idx, false))
3332     {
3333         logf("failed to retrieve index entry");
3334         return 0;
3335     }
3336
3337     /* Stop if tag has already been modified. */
3338     if (idx.flag & FLAG_DIRTYNUM)
3339         return 0;
3340
3341     logf("import: %s", tag_data);
3342
3343     idx.flag |= FLAG_DIRTYNUM;
3344     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3345     {
3346         int data;
3347
3348         if (!read_tag(tag_data, sizeof tag_data, buf,
3349                       tagcache_tag_to_str(import_tags[i])))
3350         {
3351             continue;
3352         }
3353
3354         data = atoi(tag_data);
3355         if (data < 0)
3356             continue;
3357
3358         idx.tag_seek[import_tags[i]] = data;
3359
3360         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3361             current_tcmh.serial = data + 1;
3362         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3363             current_tcmh.commitid = data + 1;
3364     }
3365
3366     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3367 }
3368
3369 bool tagcache_import_changelog(void)
3370 {
3371     struct master_header myhdr;
3372     struct tagcache_header tch;
3373     int clfd;
3374     long masterfd;
3375     char buf[2048];
3376
3377     if (!tc_stat.ready)
3378         return false;
3379
3380     while (read_lock)
3381         sleep(1);
3382
3383     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3384     if (clfd < 0)
3385     {
3386         logf("failure to open changelog");
3387         return false;
3388     }
3389
3390     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3391     {
3392         close(clfd);
3393         return false;
3394     }
3395
3396     write_lock++;
3397
3398     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3399
3400     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3401                   parse_changelog_line);
3402
3403     close(clfd);
3404     close(masterfd);
3405
3406     if (filenametag_fd >= 0)
3407     {
3408         close(filenametag_fd);
3409         filenametag_fd = -1;
3410     }
3411
3412     write_lock--;
3413
3414     update_master_header();
3415
3416     return true;
3417 }
3418
3419 #endif /* !__PCTOOL__ */
3420
3421 bool tagcache_create_changelog(struct tagcache_search *tcs)
3422 {
3423     struct master_header myhdr;
3424     struct index_entry idx;
3425     char buf[TAG_MAXLEN+32];
3426     char temp[32];
3427     int clfd;
3428     int i, j;
3429
3430     if (!tc_stat.ready)
3431         return false;
3432
3433     if (!tagcache_search(tcs, tag_filename))
3434         return false;
3435
3436     /* Initialize the changelog */
3437     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3438     if (clfd < 0)
3439     {
3440         logf("failure to open changelog");
3441         return false;
3442     }
3443
3444     if (tcs->masterfd < 0)
3445     {
3446         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3447             return false;
3448     }
3449     else
3450     {
3451         lseek(tcs->masterfd, 0, SEEK_SET);
3452         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3453     }
3454
3455     write(clfd, "## Changelog version 1\n", 23);
3456
3457     for (i = 0; i < myhdr.tch.entry_count; i++)
3458     {
3459         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3460             != sizeof(struct index_entry))
3461         {
3462             logf("read error #9");
3463             tagcache_search_finish(tcs);
3464             close(clfd);
3465             return false;
3466         }
3467
3468         /* Skip until the entry found has been modified. */
3469         if (! (idx.flag & FLAG_DIRTYNUM) )
3470             continue;
3471
3472         /* Skip deleted entries too. */
3473         if (idx.flag & FLAG_DELETED)
3474             continue;
3475
3476         /* Now retrieve all tags. */
3477         for (j = 0; j < TAG_COUNT; j++)
3478         {
3479             if (TAGCACHE_IS_NUMERIC(j))
3480             {
3481                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3482                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3483                 continue;
3484             }
3485
3486             tcs->type = j;
3487             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3488             write_tag(clfd, tagcache_tag_to_str(j), buf);
3489         }
3490
3491         write(clfd, "\n", 1);
3492         do_timed_yield();
3493     }
3494
3495     close(clfd);
3496
3497     tagcache_search_finish(tcs);
3498
3499     return true;
3500 }
3501
3502 static bool delete_entry(long idx_id)
3503 {
3504     int fd = -1;
3505     int masterfd = -1;
3506     int tag, i;
3507     struct index_entry idx, myidx;
3508     struct master_header myhdr;
3509     char buf[TAG_MAXLEN+32];
3510     int in_use[TAG_COUNT];
3511
3512     logf("delete_entry(): %ld", idx_id);
3513
3514 #ifdef HAVE_TC_RAMCACHE
3515     /* At first mark the entry removed from ram cache. */
3516     if (tc_stat.ramcache)
3517         hdr->indices[idx_id].flag |= FLAG_DELETED;
3518 #endif
3519
3520     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3521         return false;
3522
3523     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3524     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3525         != sizeof(struct index_entry))
3526     {
3527         logf("delete_entry(): read error");
3528         goto cleanup;
3529     }
3530
3531     if (myidx.flag & FLAG_DELETED)
3532     {
3533         logf("delete_entry(): already deleted!");
3534         goto cleanup;
3535     }
3536
3537     myidx.flag |= FLAG_DELETED;
3538     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3539     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3540         != sizeof(struct index_entry))
3541     {
3542         logf("delete_entry(): write_error #1");
3543         goto cleanup;
3544     }
3545
3546     /* Now check which tags are no longer in use (if any) */
3547     for (tag = 0; tag < TAG_COUNT; tag++)
3548         in_use[tag] = 0;
3549
3550     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3551     for (i = 0; i < myhdr.tch.entry_count; i++)
3552     {
3553         struct index_entry *idxp;
3554
3555 #ifdef HAVE_TC_RAMCACHE
3556         /* Use RAM DB if available for greater speed */
3557         if (tc_stat.ramcache)
3558             idxp = &hdr->indices[i];
3559         else
3560 #endif
3561         {
3562             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3563                 != sizeof(struct index_entry))
3564             {
3565                 logf("delete_entry(): read error #2");
3566                 goto cleanup;
3567             }
3568             idxp = &idx;
3569         }
3570
3571         if (idxp->flag & FLAG_DELETED)
3572             continue;
3573
3574         for (tag = 0; tag < TAG_COUNT; tag++)
3575         {
3576             if (TAGCACHE_IS_NUMERIC(tag))
3577                 continue;
3578
3579             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3580                 in_use[tag]++;
3581         }
3582     }
3583
3584     /* Now delete all tags no longer in use. */
3585     for (tag = 0; tag < TAG_COUNT; tag++)
3586     {
3587         struct tagcache_header tch;
3588         int oldseek = myidx.tag_seek[tag];
3589
3590         if (TAGCACHE_IS_NUMERIC(tag))
3591             continue;
3592
3593         /**
3594          * Replace tag seek with a hash value of the field string data.
3595          * That way runtime statistics of moved or altered files can be
3596          * resurrected.
3597          */
3598 #ifdef HAVE_TC_RAMCACHE
3599         if (tc_stat.ramcache && tag != tag_filename)
3600         {
3601             struct tagfile_entry *tfe;
3602             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3603
3604             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3605             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3606             myidx.tag_seek[tag] = *seek;
3607         }
3608         else
3609 #endif
3610         {
3611             struct tagfile_entry tfe;
3612
3613             /* Open the index file, which contains the tag names. */
3614             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3615                 goto cleanup;
3616
3617             /* Skip the header block */
3618             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3619             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3620                 != sizeof(struct tagfile_entry))
3621             {
3622                 logf("delete_entry(): read error #3");
3623                 goto cleanup;
3624             }
3625
3626             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3627             {
3628                 logf("delete_entry(): read error #4");
3629                 goto cleanup;
3630             }
3631
3632             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3633         }
3634
3635         if (in_use[tag])
3636         {
3637             logf("in use: %d/%d", tag, in_use[tag]);
3638             if (fd >= 0)
3639             {
3640                 close(fd);
3641                 fd = -1;
3642             }
3643             continue;
3644         }
3645
3646 #ifdef HAVE_TC_RAMCACHE
3647         /* Delete from ram. */
3648         if (tc_stat.ramcache && tag != tag_filename)
3649         {
3650             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3651             tagentry->tag_data[0] = '\0';
3652         }
3653 #endif
3654
3655         /* Open the index file, which contains the tag names. */
3656         if (fd < 0)
3657         {
3658             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3659                 goto cleanup;
3660         }
3661
3662         /* Skip the header block */
3663         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3664
3665         /* Debug, print 10 first characters of the tag
3666         read(fd, buf, 10);
3667         buf[10]='\0';
3668         logf("TAG:%s", buf);
3669         lseek(fd, -10, SEEK_CUR);
3670         */
3671
3672         /* Write first data byte in tag as \0 */
3673         write(fd, "", 1);
3674
3675         /* Now tag data has been removed */
3676         close(fd);
3677         fd = -1;
3678     }
3679
3680     /* Write index entry back into master index. */
3681     lseek(masterfd, sizeof(struct master_header) +
3682           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3683     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3684         != sizeof(struct index_entry))
3685     {
3686         logf("delete_entry(): write_error #2");
3687         goto cleanup;
3688     }
3689
3690     close(masterfd);
3691
3692     return true;
3693
3694     cleanup:
3695     if (fd >= 0)
3696         close(fd);
3697     if (masterfd >= 0)
3698         close(masterfd);
3699
3700     return false;
3701 }
3702
3703 #ifndef __PCTOOL__
3704 /**
3705  * Returns true if there is an event waiting in the queue
3706  * that requires the current operation to be aborted.
3707  */
3708 static bool check_event_queue(void)
3709 {
3710     struct queue_event ev;
3711
3712     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3713     switch (ev.id)
3714     {
3715         case Q_STOP_SCAN:
3716         case SYS_POWEROFF:
3717         case SYS_USB_CONNECTED:
3718             /* Put the event back into the queue. */
3719             queue_post(&tagcache_queue, ev.id, ev.data);
3720             return true;
3721     }
3722
3723     return false;
3724 }
3725 #endif
3726
3727 #ifdef HAVE_TC_RAMCACHE
3728 static bool allocate_tagcache(void)
3729 {
3730     struct master_header tcmh;
3731     int fd;
3732
3733     /* Load the header. */
3734     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3735     {
3736         hdr = NULL;
3737         return false;
3738     }
3739
3740     close(fd);
3741
3742     /**
3743      * Now calculate the required cache size plus
3744      * some extra space for alignment fixes.
3745      */
3746     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3747         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3748     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3749     memset(hdr, 0, sizeof(struct ramcache_header));
3750     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3751     hdr->indices = (struct index_entry *)(hdr + 1);
3752     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3753
3754     return true;
3755 }
3756
3757 # ifdef HAVE_EEPROM_SETTINGS
3758 static bool tagcache_dumpload(void)
3759 {
3760     struct statefile_header shdr;
3761     int fd, rc;
3762     long offpos;
3763     int i;
3764
3765     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3766     if (fd < 0)
3767     {
3768         logf("no tagcache statedump");
3769         return false;
3770     }
3771
3772     /* Check the statefile memory placement */
3773     hdr = buffer_alloc(0);
3774     rc = read(fd, &shdr, sizeof(struct statefile_header));
3775     if (rc != sizeof(struct statefile_header)
3776         /* || (long)hdr != (long)shdr.hdr */)
3777     {
3778         logf("incorrect statefile");
3779         hdr = NULL;
3780         close(fd);
3781         return false;
3782     }
3783
3784     offpos = (long)hdr - (long)shdr.hdr;
3785
3786     /* Lets allocate real memory and load it */
3787     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3788     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3789     close(fd);
3790
3791     if (rc != shdr.tc_stat.ramcache_allocated)
3792     {
3793         logf("read failure!");
3794         hdr = NULL;
3795         return false;
3796     }
3797
3798     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3799
3800     /* Now fix the pointers */
3801     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3802     for (i = 0; i < TAG_COUNT; i++)
3803         hdr->tags[i] += offpos;
3804
3805     return true;
3806 }
3807
3808 static bool tagcache_dumpsave(void)
3809 {
3810     struct statefile_header shdr;
3811     int fd;
3812
3813     if (!tc_stat.ramcache)
3814         return false;
3815
3816     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3817     if (fd < 0)
3818     {
3819         logf("failed to create a statedump");
3820         return false;
3821     }
3822
3823     /* Create the header */
3824     shdr.hdr = hdr;
3825     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3826     write(fd, &shdr, sizeof(struct statefile_header));
3827
3828     /* And dump the data too */
3829     write(fd, hdr, tc_stat.ramcache_allocated);
3830     close(fd);
3831
3832     return true;
3833 }
3834 # endif
3835
3836 static bool load_tagcache(void)
3837 {
3838     struct tagcache_header *tch;
3839     long bytesleft = tc_stat.ramcache_allocated;
3840     struct index_entry *idx;
3841     int rc, fd;
3842     char *p;
3843     int i, tag;
3844
3845 # ifdef HAVE_DIRCACHE
3846     while (dircache_is_initializing())
3847         sleep(1);
3848
3849     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3850 # endif
3851
3852     logf("loading tagcache to ram...");
3853
3854     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3855     if (fd < 0)
3856     {
3857         logf("tagcache open failed");
3858         return false;
3859     }
3860
3861     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3862         != sizeof(struct master_header)
3863         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3864     {
3865         logf("incorrect header");
3866         return false;
3867     }
3868
3869     idx = hdr->indices;
3870
3871     /* Load the master index table. */
3872     for (i = 0; i < hdr->h.tch.entry_count; i++)
3873     {
3874         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3875         if (rc != sizeof(struct index_entry))
3876         {
3877             logf("read error #10");
3878             close(fd);
3879             return false;
3880         }
3881
3882         bytesleft -= sizeof(struct index_entry);
3883         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3884         {
3885             logf("too big tagcache.");
3886             close(fd);
3887             return false;
3888         }
3889
3890         idx++;
3891     }
3892
3893     close(fd);
3894
3895     /* Load the tags. */
3896     p = (char *)idx;
3897     for (tag = 0; tag < TAG_COUNT; tag++)
3898     {
3899         struct tagfile_entry *fe;
3900         char buf[TAG_MAXLEN+32];
3901
3902         if (TAGCACHE_IS_NUMERIC(tag))
3903             continue ;
3904
3905         //p = ((void *)p+1);
3906         p = (char *)((long)p & ~0x03) + 0x04;
3907         hdr->tags[tag] = p;
3908
3909         /* Check the header. */
3910         tch = (struct tagcache_header *)p;
3911         p += sizeof(struct tagcache_header);
3912
3913         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3914             return false;
3915
3916         for (hdr->entry_count[tag] = 0;
3917              hdr->entry_count[tag] < tch->entry_count;
3918              hdr->entry_count[tag]++)
3919         {
3920             long pos;
3921
3922             if (do_timed_yield())
3923             {
3924                 /* Abort if we got a critical event in queue */
3925                 if (check_event_queue())
3926                     return false;
3927             }
3928
3929             fe = (struct tagfile_entry *)p;
3930             pos = lseek(fd, 0, SEEK_CUR);
3931             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3932             if (rc != sizeof(struct tagfile_entry))
3933             {
3934                 /* End of lookup table. */
3935                 logf("read error #11");
3936                 close(fd);
3937                 return false;
3938             }
3939
3940             /* We have a special handling for the filename tags. */
3941             if (tag == tag_filename)
3942             {
3943 # ifdef HAVE_DIRCACHE
3944                 const struct dirent *dc;
3945 # endif
3946
3947                 // FIXME: This is wrong!
3948                 // idx = &hdr->indices[hdr->entry_count[i]];
3949                 idx = &hdr->indices[fe->idx_id];
3950
3951                 if (fe->tag_length >= (long)sizeof(buf)-1)
3952                 {
3953                     read(fd, buf, 10);
3954                     buf[10] = '\0';
3955                     logf("TAG:%s", buf);
3956                     logf("too long filename");
3957                     close(fd);
3958                     return false;
3959                 }
3960
3961                 rc = read(fd, buf, fe->tag_length);
3962                 if (rc != fe->tag_length)
3963                 {
3964                     logf("read error #12");
3965                     close(fd);
3966                     return false;
3967                 }
3968
3969                 /* Check if the entry has already been removed */
3970                 if (idx->flag & FLAG_DELETED)
3971                     continue;
3972
3973                 /* This flag must not be used yet. */
3974                 if (idx->flag & FLAG_DIRCACHE)
3975                 {
3976                     logf("internal error!");
3977                     close(fd);
3978                     return false;
3979                 }
3980
3981                 if (idx->tag_seek[tag] != pos)
3982                 {
3983                     logf("corrupt data structures!");
3984                     close(fd);
3985                     return false;
3986                 }
3987
3988 # ifdef HAVE_DIRCACHE
3989                 if (dircache_is_enabled())
3990                 {
3991                     dc = dircache_get_entry_ptr(buf);
3992                     if (dc == NULL)
3993                     {
3994                         logf("Entry no longer valid.");
3995                         logf("-> %s", buf);
3996                         if (global_settings.tagcache_autoupdate)
3997                             delete_entry(fe->idx_id);
3998                         continue ;
3999                     }
4000
4001                     idx->flag |= FLAG_DIRCACHE;
4002                     idx->tag_seek[tag_filename] = (long)dc;
4003                 }
4004                 else
4005 # endif
4006                 {
4007                     /* This will be very slow unless dircache is enabled
4008                        or target is flash based, but do it anyway for
4009                        consistency. */
4010                     /* Check if entry has been removed. */
4011                     if (global_settings.tagcache_autoupdate)
4012                     {
4013                         if (!file_exists(buf))
4014                         {
4015                             logf("Entry no longer valid.");
4016                             logf("-> %s", buf);
4017                             delete_entry(fe->idx_id);
4018                             continue;
4019                         }
4020                     }
4021                 }
4022
4023                 continue ;
4024             }
4025
4026             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4027             if (bytesleft < 0)
4028             {
4029                 logf("too big tagcache #2");
4030                 logf("tl: %d", fe->tag_length);
4031                 logf("bl: %ld", bytesleft);
4032                 close(fd);
4033                 return false;
4034             }
4035
4036             p = fe->tag_data;
4037             rc = read(fd, fe->tag_data, fe->tag_length);
4038             p += rc;
4039
4040             if (rc != fe->tag_length)
4041             {
4042                 logf("read error #13");
4043                 logf("rc=0x%04x", rc); // 0x431
4044                 logf("len=0x%04x", fe->tag_length); // 0x4000
4045                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4046                 logf("tag=0x%02x", tag); // 0x00
4047                 close(fd);
4048                 return false;
4049             }
4050         }
4051         close(fd);
4052     }
4053
4054     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4055     logf("tagcache loaded into ram!");
4056
4057     return true;
4058 }
4059 #endif /* HAVE_TC_RAMCACHE */
4060
4061 static bool check_deleted_files(void)
4062 {
4063     int fd;
4064     char buf[TAG_MAXLEN+32];
4065     struct tagfile_entry tfe;
4066
4067     logf("reverse scan...");
4068     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4069     fd = open(buf, O_RDONLY);
4070
4071     if (fd < 0)
4072     {
4073         logf("%s open fail", buf);
4074         return false;
4075     }
4076
4077     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4078     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4079            == sizeof(struct tagfile_entry)
4080 #ifndef __PCTOOL__
4081            && !check_event_queue()
4082 #endif
4083            )
4084     {
4085         if (tfe.tag_length >= (long)sizeof(buf)-1)
4086         {
4087             logf("too long tag");
4088             close(fd);
4089             return false;
4090         }
4091
4092         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4093         {
4094             logf("read error #14");
4095             close(fd);
4096             return false;
4097         }
4098
4099         /* Check if the file has already deleted from the db. */
4100         if (*buf == '\0')
4101             continue;
4102
4103         /* Now check if the file exists. */
4104         if (!file_exists(buf))
4105         {
4106             logf("Entry no longer valid.");
4107             logf("-> %s / %d", buf, tfe.tag_length);
4108             delete_entry(tfe.idx_id);
4109         }
4110     }
4111
4112     close(fd);
4113
4114     logf("done");
4115
4116     return true;
4117 }
4118
4119
4120 /* Note that this function must not be inlined, otherwise the whole point
4121  * of having the code in a separate function is lost.
4122  */
4123 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4124     int *ignore, int *unignore)
4125 {
4126     char newpath[MAX_PATH];
4127
4128     /* check for a database.ignore file */
4129     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4130     *ignore = file_exists(newpath);
4131     /* check for a database.unignore file */
4132     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4133     *unignore = file_exists(newpath);
4134 }
4135
4136
4137 static bool check_dir(const char *dirname, int add_files)
4138 {
4139     DIR *dir;
4140     int len;
4141     int success = false;
4142     int ignore, unignore;
4143
4144     dir = opendir(dirname);
4145     if (!dir)
4146     {
4147         logf("tagcache: opendir() failed");
4148         return false;
4149     }
4150
4151     /* check for a database.ignore and database.unignore */
4152     check_ignore(dirname, &ignore, &unignore);
4153
4154     /* don't do anything if both ignore and unignore are there */
4155     if (ignore != unignore)
4156         add_files = unignore;
4157
4158     /* Recursively scan the dir. */
4159 #ifdef __PCTOOL__
4160     while (1)
4161 #else
4162     while (!check_event_queue())
4163 #endif
4164     {
4165         struct dirent *entry;
4166
4167         entry = readdir(dir);
4168
4169         if (entry == NULL)
4170         {
4171             success = true;
4172             break ;
4173         }
4174
4175         if (!strcmp((char *)entry->d_name, ".") ||
4176             !strcmp((char *)entry->d_name, ".."))
4177             continue;
4178
4179         yield();
4180
4181         len = strlen(curpath);
4182         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4183                  entry->d_name);
4184
4185         processed_dir_count++;
4186         if (entry->attribute & ATTR_DIRECTORY)
4187             check_dir(curpath, add_files);
4188         else if (add_files)
4189         {
4190             tc_stat.curentry = curpath;
4191
4192             /* Add a new entry to the temporary db file. */
4193             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4194 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4195                          , dir->internal_entry
4196 #endif
4197                          );
4198
4199             /* Wait until current path for debug screen is read and unset. */
4200             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4201                 yield();
4202
4203             tc_stat.curentry = NULL;
4204         }
4205
4206         curpath[len] = '\0';
4207     }
4208
4209     closedir(dir);
4210
4211     return success;
4212 }
4213
4214 void tagcache_screensync_event(void)
4215 {
4216     tc_stat.curentry = NULL;
4217 }
4218
4219 void tagcache_screensync_enable(bool state)
4220 {
4221     tc_stat.syncscreen = state;
4222 }
4223
4224 void tagcache_build(const char *path)
4225 {
4226     struct tagcache_header header;
4227     bool ret;
4228
4229     curpath[0] = '\0';
4230     data_size = 0;
4231     total_entry_count = 0;
4232     processed_dir_count = 0;
4233
4234 #ifdef HAVE_DIRCACHE
4235     while (dircache_is_initializing())
4236         sleep(1);
4237 #endif
4238
4239     logf("updating tagcache");
4240
4241     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4242     if (cachefd >= 0)
4243     {
4244         logf("skipping, cache already waiting for commit");
4245         close(cachefd);
4246         return ;
4247     }
4248
4249     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4250     if (cachefd < 0)
4251     {
4252         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4253         return ;
4254     }
4255
4256     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4257
4258     cpu_boost(true);
4259
4260     logf("Scanning files...");
4261     /* Scan for new files. */
4262     memset(&header, 0, sizeof(struct tagcache_header));
4263     write(cachefd, &header, sizeof(struct tagcache_header));
4264
4265     if (strcmp("/", path) != 0)
4266         strcpy(curpath, path);
4267     ret = check_dir(path, true);
4268
4269     /* Write the header. */
4270     header.magic = TAGCACHE_MAGIC;
4271     header.datasize = data_size;
4272     header.entry_count = total_entry_count;
4273     lseek(cachefd, 0, SEEK_SET);
4274     write(cachefd, &header, sizeof(struct tagcache_header));
4275     close(cachefd);
4276
4277     if (filenametag_fd >= 0)
4278     {
4279         close(filenametag_fd);
4280         filenametag_fd = -1;
4281     }
4282
4283     if (!ret)
4284     {
4285         logf("Aborted.");
4286         cpu_boost(false);
4287         return ;
4288     }
4289
4290     /* Commit changes to the database. */
4291 #ifdef __PCTOOL__
4292     allocate_tempbuf();
4293 #endif
4294     if (commit())
4295     {
4296         remove(TAGCACHE_FILE_TEMP);
4297         logf("tagcache built!");
4298     }
4299 #ifdef __PCTOOL__
4300     free_tempbuf();
4301 #endif
4302
4303 #ifdef HAVE_TC_RAMCACHE
4304     if (hdr)
4305     {
4306         /* Import runtime statistics if we just initialized the db. */
4307         if (hdr->h.serial == 0)
4308             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4309     }
4310 #endif
4311
4312     cpu_boost(false);
4313 }
4314
4315 #ifdef HAVE_TC_RAMCACHE
4316 static void load_ramcache(void)
4317 {
4318     if (!hdr)
4319         return ;
4320
4321     cpu_boost(true);
4322
4323     /* At first we should load the cache (if exists). */
4324     tc_stat.ramcache = load_tagcache();
4325
4326     if (!tc_stat.ramcache)
4327     {
4328         /* If loading failed, it must indicate some problem with the db
4329          * so disable it entirely to prevent further issues. */
4330         tc_stat.ready = false;
4331         hdr = NULL;
4332     }
4333
4334     cpu_boost(false);
4335 }
4336
4337 void tagcache_unload_ramcache(void)
4338 {
4339     tc_stat.ramcache = false;
4340     /* Just to make sure there is no statefile present. */
4341     // remove(TAGCACHE_STATEFILE);
4342 }
4343 #endif
4344
4345 #ifndef __PCTOOL__
4346 static void tagcache_thread(void)
4347 {
4348     struct queue_event ev;
4349     bool check_done = false;
4350
4351     /* If the previous cache build/update was interrupted, commit
4352      * the changes first in foreground. */
4353     cpu_boost(true);
4354     allocate_tempbuf();
4355     commit();
4356     free_tempbuf();
4357
4358 #ifdef HAVE_TC_RAMCACHE
4359 # ifdef HAVE_EEPROM_SETTINGS
4360     if (firmware_settings.initialized && firmware_settings.disk_clean)
4361         check_done = tagcache_dumpload();
4362
4363     remove(TAGCACHE_STATEFILE);
4364 # endif
4365
4366     /* Allocate space for the tagcache if found on disk. */
4367     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4368         allocate_tagcache();
4369 #endif
4370
4371     cpu_boost(false);
4372     tc_stat.initialized = true;
4373
4374     /* Don't delay bootup with the header check but do it on background. */
4375     sleep(HZ);
4376     tc_stat.ready = check_all_headers();
4377     tc_stat.readyvalid = true;
4378
4379     while (1)
4380     {
4381         run_command_queue(false);
4382
4383         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4384
4385         switch (ev.id)
4386         {
4387             case Q_IMPORT_CHANGELOG:
4388                 tagcache_import_changelog();
4389                 break;
4390
4391             case Q_REBUILD:
4392                 remove_files();
4393                 remove(TAGCACHE_FILE_TEMP);
4394                 tagcache_build("/");
4395                 break;
4396
4397             case Q_UPDATE:
4398                 tagcache_build("/");
4399 #ifdef HAVE_TC_RAMCACHE
4400                 load_ramcache();
4401 #endif
4402                 check_deleted_files();
4403                 break ;
4404
4405             case Q_START_SCAN:
4406                 check_done = false;
4407             case SYS_TIMEOUT:
4408                 if (check_done || !tc_stat.ready)
4409                     break ;
4410
4411 #ifdef HAVE_TC_RAMCACHE
4412                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4413                 {
4414                     load_ramcache();
4415                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4416                         tagcache_build("/");
4417                 }
4418                 else
4419 #endif
4420                 if (global_settings.tagcache_autoupdate)
4421                 {
4422                     tagcache_build("/");
4423
4424                     /* This will be very slow unless dircache is enabled
4425                        or target is flash based, but do it anyway for
4426                        consistency. */
4427                     check_deleted_files();
4428                 }
4429
4430                 logf("tagcache check done");
4431
4432                 check_done = true;
4433                 break ;
4434
4435             case Q_STOP_SCAN:
4436                 break ;
4437
4438             case SYS_POWEROFF:
4439                 break ;
4440
4441 #ifndef SIMULATOR
4442             case SYS_USB_CONNECTED:
4443                 logf("USB: TagCache");
4444                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4445                 usb_wait_for_disconnect(&tagcache_queue);
4446                 break ;
4447 #endif
4448         }
4449     }
4450 }
4451
4452 bool tagcache_prepare_shutdown(void)
4453 {
4454     if (tagcache_get_commit_step() > 0)
4455         return false;
4456
4457     tagcache_stop_scan();
4458     while (read_lock || write_lock)
4459         sleep(1);
4460
4461     return true;
4462 }
4463
4464 void tagcache_shutdown(void)
4465 {
4466     /* Flush the command queue. */
4467     run_command_queue(true);
4468
4469 #ifdef HAVE_EEPROM_SETTINGS
4470     if (tc_stat.ramcache)
4471         tagcache_dumpsave();
4472 #endif
4473 }
4474
4475 static int get_progress(void)
4476 {
4477     int total_count = -1;
4478
4479 #ifdef HAVE_DIRCACHE
4480     if (dircache_is_enabled())
4481     {
4482         total_count = dircache_get_entry_count();
4483     }
4484     else
4485 #endif
4486 #ifdef HAVE_TC_RAMCACHE
4487     {
4488         if (hdr && tc_stat.ramcache)
4489             total_count = hdr->h.tch.entry_count;
4490     }
4491 #endif
4492
4493     if (total_count < 0)
4494         return -1;
4495
4496     return processed_dir_count * 100 / total_count;
4497 }
4498
4499 struct tagcache_stat* tagcache_get_stat(void)
4500 {
4501     tc_stat.progress = get_progress();
4502     tc_stat.processed_entries = processed_dir_count;
4503
4504     return &tc_stat;
4505 }
4506
4507 void tagcache_start_scan(void)
4508 {
4509     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4510 }
4511
4512 bool tagcache_update(void)
4513 {
4514     if (!tc_stat.ready)
4515         return false;
4516
4517     queue_post(&tagcache_queue, Q_UPDATE, 0);
4518     return false;
4519 }
4520
4521 bool tagcache_rebuild()
4522 {
4523     queue_post(&tagcache_queue, Q_REBUILD, 0);
4524     return false;
4525 }
4526
4527 void tagcache_stop_scan(void)
4528 {
4529     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4530 }
4531
4532 #ifdef HAVE_TC_RAMCACHE
4533 bool tagcache_is_ramcache(void)
4534 {
4535     return tc_stat.ramcache;
4536 }
4537 #endif
4538
4539 #endif /* !__PCTOOL__ */
4540
4541
4542 void tagcache_init(void)
4543 {
4544     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4545     memset(&current_tcmh, 0, sizeof(struct master_header));
4546     filenametag_fd = -1;
4547     write_lock = read_lock = 0;
4548
4549 #ifndef __PCTOOL__
4550     mutex_init(&command_queue_mutex);
4551     queue_init(&tagcache_queue, true);
4552     create_thread(tagcache_thread, tagcache_stack,
4553                   sizeof(tagcache_stack), 0, tagcache_thread_name
4554                   IF_PRIO(, PRIORITY_BACKGROUND)
4555                   IF_COP(, CPU));
4556 #else
4557     tc_stat.initialized = true;
4558     allocate_tempbuf();
4559     commit();
4560     free_tempbuf();
4561     tc_stat.ready = check_all_headers();
4562 #endif
4563 }
4564
4565 #ifdef __PCTOOL__
4566 void tagcache_reverse_scan(void)
4567 {
4568     logf("Checking for deleted files");
4569     check_deleted_files();
4570 }
4571 #endif
4572
4573 bool tagcache_is_initialized(void)
4574 {
4575     return tc_stat.initialized;
4576 }
4577 bool tagcache_is_usable(void)
4578 {
4579     return tc_stat.initialized && tc_stat.ready;
4580 }
4581 int tagcache_get_commit_step(void)
4582 {
4583     return tc_stat.commit_step;
4584 }
4585 int tagcache_get_max_commit_step(void)
4586 {
4587     return (int)(SORTED_TAGS_COUNT)+1;
4588 }
4589