apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 #define UNTAGGED "<Untagged>"
  99
 100 /* Previous path when scanning directory tree recursively. */
 101 static char curpath[TAG_MAXLEN+32];
 102
 103 /* Used when removing duplicates. */
 104 static char *tempbuf;     /* Allocated when needed. */
 105 static long tempbufidx;   /* Current location in buffer. */
 106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 107 static long tempbuf_left; /* Buffer space left. */
 108 static long tempbuf_pos;
 109
 110 #define SORTED_TAGS_COUNT 8
 111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 114     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 115 /* Tags we want to get sorted (loaded to the tempbuf). */
 116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 117     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 118     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 119
 120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 122     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 123     (1LU << tag_albumartist) | (1LU << tag_grouping))
 124
 125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 126 static const char *tags_str[] = { "artist", "album", "genre", "title",
 127     "filename", "composer", "comment", "albumartist", "grouping", "year",
 128     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 129     "playtime", "lastplayed", "commitid", "mtime" };
 130
 131 /* Status information of the tagcache. */
 132 static struct tagcache_stat tc_stat;
 133
 134 /* Queue commands. */
 135 enum tagcache_queue {
 136     Q_STOP_SCAN = 0,
 137     Q_START_SCAN,
 138     Q_IMPORT_CHANGELOG,
 139     Q_UPDATE,
 140     Q_REBUILD,
 141
 142     /* Internal tagcache command queue. */
 143     CMD_UPDATE_MASTER_HEADER,
 144     CMD_UPDATE_NUMERIC,
 145 };
 146
 147 struct tagcache_command_entry {
 148     int32_t command;
 149     int32_t idx_id;
 150     int32_t tag;
 151     int32_t data;
 152 };
 153
 154 #ifndef __PCTOOL__
 155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 156 static volatile int command_queue_widx = 0;
 157 static volatile int command_queue_ridx = 0;
 158 static struct mutex command_queue_mutex;
 159 #endif
 160
 161 /* Tag database structures. */
 162
 163 /* Variable-length tag entry in tag files. */
 164 struct tagfile_entry {
 165     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 166     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 167     char tag_data[0];  /* Begin of the tag data */
 168 };
 169
 170 /* Fixed-size tag entry in master db index. */
 171 struct index_entry {
 172     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 173     int32_t flag;                /* Status flags */
 174 };
 175
 176 /* Header is the same in every file. */
 177 struct tagcache_header {
 178     int32_t magic;       /* Header version number */
 179     int32_t datasize;    /* Data size in bytes */
 180     int32_t entry_count; /* Number of entries in this file */
 181 };
 182
 183 struct master_header {
 184     struct tagcache_header tch;
 185     int32_t serial; /* Increasing counting number */
 186     int32_t commitid; /* Number of commits so far */
 187     int32_t dirty;
 188 };
 189
 190 /* For the endianess correction */
 191 static const char *tagfile_entry_ec   = "ll";
 192 /**
 193  Note: This should be (1 + TAG_COUNT) amount of l's.
 194  */
 195 static const char *index_entry_ec     = "lllllllllllllllllllll";
 196
 197 static const char *tagcache_header_ec = "lll";
 198 static const char *master_header_ec   = "llllll";
 199
 200 static struct master_header current_tcmh;
 201
 202 #ifdef HAVE_TC_RAMCACHE
 203 /* Header is created when loading database to ram. */
 204 struct ramcache_header {
 205     struct master_header h;      /* Header from the master index */
 206     struct index_entry *indices; /* Master index file content */
 207     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 208     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 209 };
 210
 211 # ifdef HAVE_EEPROM_SETTINGS
 212 struct statefile_header {
 213     struct ramcache_header *hdr;
 214     struct tagcache_stat tc_stat;
 215 };
 216 # endif
 217
 218 /* Pointer to allocated ramcache_header */
 219 static struct ramcache_header *hdr;
 220 #endif
 221
 222 /**
 223  * Full tag entries stored in a temporary file waiting
 224  * for commit to the cache. */
 225 struct temp_file_entry {
 226     long tag_offset[TAG_COUNT];
 227     short tag_length[TAG_COUNT];
 228     long flag;
 229
 230     long data_length;
 231 };
 232
 233 struct tempbuf_id_list {
 234     long id;
 235     struct tempbuf_id_list *next;
 236 };
 237
 238 struct tempbuf_searchidx {
 239     long idx_id;
 240     char *str;
 241     int seek;
 242     struct tempbuf_id_list idlist;
 243 };
 244
 245 /* Lookup buffer for fixing messed up index while after sorting. */
 246 static long commit_entry_count;
 247 static long lookup_buffer_depth;
 248 static struct tempbuf_searchidx **lookup;
 249
 250 /* Used when building the temporary file. */
 251 static int cachefd = -1, filenametag_fd;
 252 static int total_entry_count = 0;
 253 static int data_size = 0;
 254 static int processed_dir_count;
 255
 256 /* Thread safe locking */
 257 static volatile int write_lock;
 258 static volatile int read_lock;
 259
 260 static bool delete_entry(long idx_id);
 261
 262 const char* tagcache_tag_to_str(int tag)
 263 {
 264     return tags_str[tag];
 265 }
 266
 267 #ifdef HAVE_DIRCACHE
 268 /**
 269  * Returns true if specified flag is still present, i.e., dircache
 270  * has not been reloaded.
 271  */
 272 static bool is_dircache_intact(void)
 273 {
 274     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 275 }
 276 #endif
 277
 278 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 279 {
 280     int fd;
 281     char buf[MAX_PATH];
 282     int rc;
 283
 284     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 285         return -1;
 286
 287     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 288
 289     fd = open(buf, write ? O_RDWR : O_RDONLY);
 290     if (fd < 0)
 291     {
 292         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 293         tc_stat.ready = false;
 294         return fd;
 295     }
 296
 297     /* Check the header. */
 298     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 299     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 300     {
 301         logf("header error");
 302         tc_stat.ready = false;
 303         close(fd);
 304         return -2;
 305     }
 306
 307     return fd;
 308 }
 309
 310 static int open_master_fd(struct master_header *hdr, bool write)
 311 {
 312     int fd;
 313     int rc;
 314
 315     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 316     if (fd < 0)
 317     {
 318         logf("master file open failed for R/W");
 319         tc_stat.ready = false;
 320         return fd;
 321     }
 322
 323     tc_stat.econ = false;
 324
 325     /* Check the header. */
 326     rc = read(fd, hdr, sizeof(struct master_header));
 327     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 328     {
 329         /* Success. */
 330         return fd;
 331     }
 332
 333     /* Trying to read again, this time with endianess correction enabled. */
 334     lseek(fd, 0, SEEK_SET);
 335
 336     rc = ecread(fd, hdr, 1, master_header_ec, true);
 337     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 338     {
 339         logf("header error");
 340         tc_stat.ready = false;
 341         close(fd);
 342         return -2;
 343     }
 344
 345     tc_stat.econ = true;
 346
 347     return fd;
 348 }
 349
 350 #ifndef __PCTOOL__
 351 static bool do_timed_yield(void)
 352 {
 353     /* Sorting can lock up for quite a while, so yield occasionally */
 354     static long wakeup_tick = 0;
 355     if (TIME_AFTER(current_tick, wakeup_tick))
 356     {
 357         wakeup_tick = current_tick + (HZ/4);
 358         yield();
 359         return true;
 360     }
 361     return false;
 362 }
 363 #endif
 364
 365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 366 static long find_entry_ram(const char *filename,
 367                            const struct dircache_entry *dc)
 368 {
 369     static long last_pos = 0;
 370     int i;
 371
 372     /* Check if we tagcache is loaded into ram. */
 373     if (!tc_stat.ramcache)
 374         return -1;
 375
 376     if (dc == NULL)
 377         dc = dircache_get_entry_ptr(filename);
 378
 379     if (dc == NULL)
 380     {
 381         logf("tagcache: file not found.");
 382         return -1;
 383     }
 384
 385     try_again:
 386
 387     if (last_pos > 0)
 388         i = last_pos;
 389     else
 390         i = 0;
 391
 392     for (; i < hdr->h.tch.entry_count; i++)
 393     {
 394         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 395         {
 396             last_pos = MAX(0, i - 3);
 397             return i;
 398         }
 399
 400         do_timed_yield();
 401     }
 402
 403     if (last_pos > 0)
 404     {
 405         last_pos = 0;
 406         goto try_again;
 407     }
 408
 409     return -1;
 410 }
 411 #endif
 412
 413 static long find_entry_disk(const char *filename, bool localfd)
 414 {
 415     struct tagcache_header tch;
 416     static long last_pos = -1;
 417     long pos_history[POS_HISTORY_COUNT];
 418     long pos_history_idx = 0;
 419     bool found = false;
 420     struct tagfile_entry tfe;
 421     int fd;
 422     char buf[TAG_MAXLEN+32];
 423     int i;
 424     int pos = -1;
 425
 426     if (!tc_stat.ready)
 427         return -2;
 428
 429     fd = filenametag_fd;
 430     if (fd < 0 || localfd)
 431     {
 432         last_pos = -1;
 433         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 434             return -1;
 435     }
 436
 437     check_again:
 438
 439     if (last_pos > 0)
 440         lseek(fd, last_pos, SEEK_SET);
 441     else
 442         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 443
 444     while (true)
 445     {
 446         pos = lseek(fd, 0, SEEK_CUR);
 447         for (i = pos_history_idx-1; i >= 0; i--)
 448             pos_history[i+1] = pos_history[i];
 449         pos_history[0] = pos;
 450
 451         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 452             != sizeof(struct tagfile_entry))
 453         {
 454             break ;
 455         }
 456
 457         if (tfe.tag_length >= (long)sizeof(buf))
 458         {
 459             logf("too long tag #1");
 460             close(fd);
 461             if (!localfd)
 462                 filenametag_fd = -1;
 463             last_pos = -1;
 464             return -2;
 465         }
 466
 467         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 468         {
 469             logf("read error #2");
 470             close(fd);
 471             if (!localfd)
 472                 filenametag_fd = -1;
 473             last_pos = -1;
 474             return -3;
 475         }
 476
 477         if (!strcasecmp(filename, buf))
 478         {
 479             last_pos = pos_history[pos_history_idx];
 480             found = true;
 481             break ;
 482         }
 483
 484         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 485             pos_history_idx++;
 486     }
 487
 488     /* Not found? */
 489     if (!found)
 490     {
 491         if (last_pos > 0)
 492         {
 493             last_pos = -1;
 494             logf("seek again");
 495             goto check_again;
 496         }
 497
 498         if (fd != filenametag_fd || localfd)
 499             close(fd);
 500         return -4;
 501     }
 502
 503     if (fd != filenametag_fd || localfd)
 504         close(fd);
 505
 506     return tfe.idx_id;
 507 }
 508
 509 static int find_index(const char *filename)
 510 {
 511     long idx_id = -1;
 512
 513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 514     if (tc_stat.ramcache && is_dircache_intact())
 515         idx_id = find_entry_ram(filename, NULL);
 516 #endif
 517
 518     if (idx_id < 0)
 519         idx_id = find_entry_disk(filename, true);
 520
 521     return idx_id;
 522 }
 523
 524 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 525 {
 526     int idx_id;
 527
 528     if (!tc_stat.ready)
 529         return false;
 530
 531     idx_id = find_index(filename);
 532     if (idx_id < 0)
 533         return false;
 534
 535     if (!tagcache_search(tcs, tag_filename))
 536         return false;
 537
 538     tcs->entry_count = 0;
 539     tcs->idx_id = idx_id;
 540
 541     return true;
 542 }
 543
 544 static bool get_index(int masterfd, int idxid,
 545                       struct index_entry *idx, bool use_ram)
 546 {
 547     bool localfd = false;
 548
 549     if (idxid < 0)
 550     {
 551         logf("Incorrect idxid: %d", idxid);
 552         return false;
 553     }
 554
 555 #ifdef HAVE_TC_RAMCACHE
 556     if (tc_stat.ramcache && use_ram)
 557     {
 558         if (hdr->indices[idxid].flag & FLAG_DELETED)
 559             return false;
 560
 561 # ifdef HAVE_DIRCACHE
 562         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 563             || is_dircache_intact())
 564 #endif
 565         {
 566             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 567             return true;
 568         }
 569     }
 570 #else
 571     (void)use_ram;
 572 #endif
 573
 574     if (masterfd < 0)
 575     {
 576         struct master_header tcmh;
 577
 578         localfd = true;
 579         masterfd = open_master_fd(&tcmh, false);
 580         if (masterfd < 0)
 581             return false;
 582     }
 583
 584     lseek(masterfd, idxid * sizeof(struct index_entry)
 585           + sizeof(struct master_header), SEEK_SET);
 586     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 587         != sizeof(struct index_entry))
 588     {
 589         logf("read error #3");
 590         if (localfd)
 591             close(masterfd);
 592
 593         return false;
 594     }
 595
 596     if (localfd)
 597         close(masterfd);
 598
 599     if (idx->flag & FLAG_DELETED)
 600         return false;
 601
 602     return true;
 603 }
 604
 605 #ifndef __PCTOOL__
 606
 607 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 608 {
 609     /* We need to exclude all memory only flags & tags when writing to disk. */
 610     if (idx->flag & FLAG_DIRCACHE)
 611     {
 612         logf("memory only flags!");
 613         return false;
 614     }
 615
 616 #ifdef HAVE_TC_RAMCACHE
 617     /* Only update numeric data. Writing the whole index to RAM by memcpy
 618      * destroys dircache pointers!
 619      */
 620     if (tc_stat.ramcache)
 621     {
 622         int tag;
 623         struct index_entry *idx_ram = &hdr->indices[idxid];
 624
 625         for (tag = 0; tag < TAG_COUNT; tag++)
 626         {
 627             if (TAGCACHE_IS_NUMERIC(tag))
 628             {
 629                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 630             }
 631         }
 632
 633         /* Don't touch the dircache flag or attributes. */
 634         idx_ram->flag = (idx->flag & 0x0000ffff)
 635             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 636     }
 637 #endif
 638
 639     lseek(masterfd, idxid * sizeof(struct index_entry)
 640           + sizeof(struct master_header), SEEK_SET);
 641     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 642         != sizeof(struct index_entry))
 643     {
 644         logf("write error #3");
 645         logf("idxid: %d", idxid);
 646         return false;
 647     }
 648
 649     return true;
 650 }
 651
 652 #endif /* !__PCTOOL__ */
 653
 654 static bool open_files(struct tagcache_search *tcs, int tag)
 655 {
 656     if (tcs->idxfd[tag] < 0)
 657     {
 658         char fn[MAX_PATH];
 659
 660         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 661         tcs->idxfd[tag] = open(fn, O_RDONLY);
 662     }
 663
 664     if (tcs->idxfd[tag] < 0)
 665     {
 666         logf("File not open!");
 667         return false;
 668     }
 669
 670     return true;
 671 }
 672
 673 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 674                      int tag, char *buf, long size)
 675 {
 676     struct tagfile_entry tfe;
 677     long seek;
 678
 679     *buf = '\0';
 680
 681     if (TAGCACHE_IS_NUMERIC(tag))
 682         return false;
 683
 684     seek = idx->tag_seek[tag];
 685     if (seek < 0)
 686     {
 687         logf("Retrieve failed");
 688         return false;
 689     }
 690
 691 #ifdef HAVE_TC_RAMCACHE
 692     if (tcs->ramsearch)
 693     {
 694         struct tagfile_entry *ep;
 695
 696 # ifdef HAVE_DIRCACHE
 697         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 698             && is_dircache_intact())
 699         {
 700             dircache_copy_path((struct dircache_entry *)seek,
 701                                buf, size);
 702             return true;
 703         }
 704         else
 705 # endif
 706         if (tag != tag_filename)
 707         {
 708             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 709             strlcpy(buf, ep->tag_data, size);
 710
 711             return true;
 712         }
 713     }
 714 #endif
 715
 716     if (!open_files(tcs, tag))
 717         return false;
 718
 719     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 720     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 721         != sizeof(struct tagfile_entry))
 722     {
 723         logf("read error #5");
 724         return false;
 725     }
 726
 727     if (tfe.tag_length >= size)
 728     {
 729         logf("too small buffer");
 730         return false;
 731     }
 732
 733     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 734         tfe.tag_length)
 735     {
 736         logf("read error #6");
 737         return false;
 738     }
 739
 740     buf[tfe.tag_length] = '\0';
 741
 742     return true;
 743 }
 744
 745 static long check_virtual_tags(int tag, const struct index_entry *idx)
 746 {
 747     long data = 0;
 748
 749     switch (tag)
 750     {
 751         case tag_virt_length_sec:
 752             data = (idx->tag_seek[tag_length]/1000) % 60;
 753             break;
 754
 755         case tag_virt_length_min:
 756             data = (idx->tag_seek[tag_length]/1000) / 60;
 757             break;
 758
 759         case tag_virt_playtime_sec:
 760             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 761             break;
 762
 763         case tag_virt_playtime_min:
 764             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 765             break;
 766
 767         case tag_virt_autoscore:
 768             if (idx->tag_seek[tag_length] == 0
 769                 || idx->tag_seek[tag_playcount] == 0)
 770             {
 771                 data = 0;
 772             }
 773             else
 774             {
 775                 /* A straight calculus gives:
 776                      autoscore = 100 * playtime / length / playcout (1)
 777                    Now, consider the euclidian division of playtime by length:
 778                      playtime = alpha * length + beta
 779                    With:
 780                      0 <= beta < length
 781                    Now, (1) becomes:
 782                      autoscore = 100 * (alpha / playcout + beta / length / playcount)
 783                    Both terms should be small enough to avoid any overflow
 784                 */
 785                 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
 786                      + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
 787                 data /=  idx->tag_seek[tag_playcount];
 788             }
 789             break;
 790
 791         /* How many commits before the file has been added to the DB. */
 792         case tag_virt_entryage:
 793             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 794             break;
 795
 796         default:
 797             data = idx->tag_seek[tag];
 798     }
 799
 800     return data;
 801 }
 802
 803 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 804 {
 805     struct index_entry idx;
 806
 807     if (!tc_stat.ready)
 808         return false;
 809
 810     if (!TAGCACHE_IS_NUMERIC(tag))
 811         return -1;
 812
 813     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 814         return -2;
 815
 816     return check_virtual_tags(tag, &idx);
 817 }
 818
 819 inline static bool str_ends_with(const char *str1, const char *str2)
 820 {
 821     int str_len = strlen(str1);
 822     int clause_len = strlen(str2);
 823
 824     if (clause_len > str_len)
 825         return false;
 826
 827     return !strcasecmp(&str1[str_len - clause_len], str2);
 828 }
 829
 830 inline static bool str_oneof(const char *str, const char *list)
 831 {
 832     const char *sep;
 833     int l, len = strlen(str);
 834
 835     while (*list)
 836     {
 837         sep = strchr(list, '|');
 838         l = sep ? (long)sep - (long)list : (int)strlen(list);
 839         if ((l==len) && !strncasecmp(str, list, len))
 840             return true;
 841         list += sep ? l + 1 : l;
 842     }
 843
 844     return false;
 845 }
 846
 847 static bool check_against_clause(long numeric, const char *str,
 848                                  const struct tagcache_search_clause *clause)
 849 {
 850     if (clause->numeric)
 851     {
 852         switch (clause->type)
 853         {
 854             case clause_is:
 855                 return numeric == clause->numeric_data;
 856             case clause_is_not:
 857                 return numeric != clause->numeric_data;
 858             case clause_gt:
 859                 return numeric > clause->numeric_data;
 860             case clause_gteq:
 861                 return numeric >= clause->numeric_data;
 862             case clause_lt:
 863                 return numeric < clause->numeric_data;
 864             case clause_lteq:
 865                 return numeric <= clause->numeric_data;
 866             default:
 867                 logf("Incorrect numeric tag: %d", clause->type);
 868         }
 869     }
 870     else
 871     {
 872         switch (clause->type)
 873         {
 874             case clause_is:
 875                 return !strcasecmp(clause->str, str);
 876             case clause_is_not:
 877                 return strcasecmp(clause->str, str);
 878             case clause_gt:
 879                 return 0>strcasecmp(clause->str, str);
 880             case clause_gteq:
 881                 return 0>=strcasecmp(clause->str, str);
 882             case clause_lt:
 883                 return 0<strcasecmp(clause->str, str);
 884             case clause_lteq:
 885                 return 0<=strcasecmp(clause->str, str);
 886             case clause_contains:
 887                 return (strcasestr(str, clause->str) != NULL);
 888             case clause_not_contains:
 889                 return (strcasestr(str, clause->str) == NULL);
 890             case clause_begins_with:
 891                 return (strcasestr(str, clause->str) == str);
 892             case clause_not_begins_with:
 893                 return (strcasestr(str, clause->str) != str);
 894             case clause_ends_with:
 895                 return str_ends_with(str, clause->str);
 896             case clause_not_ends_with:
 897                 return !str_ends_with(str, clause->str);
 898             case clause_oneof:
 899                 return str_oneof(str, clause->str);
 900
 901             default:
 902                 logf("Incorrect tag: %d", clause->type);
 903         }
 904     }
 905
 906     return false;
 907 }
 908
 909 static bool check_clauses(struct tagcache_search *tcs,
 910                           struct index_entry *idx,
 911                           struct tagcache_search_clause **clause, int count)
 912 {
 913     int i;
 914
 915 #ifdef HAVE_TC_RAMCACHE
 916     if (tcs->ramsearch)
 917     {
 918         /* Go through all conditional clauses. */
 919         for (i = 0; i < count; i++)
 920         {
 921             struct tagfile_entry *tfe;
 922             int seek;
 923             char buf[256];
 924             char *str = NULL;
 925
 926             seek = check_virtual_tags(clause[i]->tag, idx);
 927
 928             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 929             {
 930                 if (clause[i]->tag == tag_filename)
 931                 {
 932                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 933                     str = buf;
 934                 }
 935                 else
 936                 {
 937                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 938                     str = tfe->tag_data;
 939                 }
 940             }
 941
 942             if (!check_against_clause(seek, str, clause[i]))
 943                 return false;
 944         }
 945     }
 946     else
 947 #endif
 948     {
 949         /* Check for conditions. */
 950         for (i = 0; i < count; i++)
 951         {
 952             struct tagfile_entry tfe;
 953             int seek;
 954             char str[256];
 955
 956             seek = check_virtual_tags(clause[i]->tag, idx);
 957
 958             memset(str, 0, sizeof str);
 959             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 960             {
 961                 int fd = tcs->idxfd[clause[i]->tag];
 962                 lseek(fd, seek, SEEK_SET);
 963                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 964                 if (tfe.tag_length >= (int)sizeof(str))
 965                 {
 966                     logf("Too long tag read!");
 967                     break ;
 968                 }
 969
 970                 read(fd, str, tfe.tag_length);
 971
 972                 /* Check if entry has been deleted. */
 973                 if (str[0] == '\0')
 974                     break;
 975             }
 976
 977             if (!check_against_clause(seek, str, clause[i]))
 978                 return false;
 979         }
 980     }
 981
 982     return true;
 983 }
 984
 985 bool tagcache_check_clauses(struct tagcache_search *tcs,
 986                             struct tagcache_search_clause **clause, int count)
 987 {
 988     struct index_entry idx;
 989
 990     if (count == 0)
 991         return true;
 992
 993     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 994         return false;
 995
 996     return check_clauses(tcs, &idx, clause, count);
 997 }
 998
 999 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1000 {
1001     int i;
1002
1003     /* If uniq buffer is not defined we must return true for search to work. */
1004     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1005         && !TAGCACHE_IS_NUMERIC(tcs->type)))
1006     {
1007         return true;
1008     }
1009
1010     for (i = 0; i < tcs->unique_list_count; i++)
1011     {
1012         /* Return false if entry is found. */
1013         if (tcs->unique_list[i] == id)
1014             return false;
1015     }
1016
1017     if (tcs->unique_list_count < tcs->unique_list_capacity)
1018     {
1019         tcs->unique_list[i] = id;
1020         tcs->unique_list_count++;
1021     }
1022
1023     return true;
1024 }
1025
1026 static bool build_lookup_list(struct tagcache_search *tcs)
1027 {
1028     struct index_entry entry;
1029     int i, j;
1030
1031     tcs->seek_list_count = 0;
1032
1033 #ifdef HAVE_TC_RAMCACHE
1034     if (tcs->ramsearch
1035 # ifdef HAVE_DIRCACHE
1036         && (tcs->type != tag_filename || is_dircache_intact())
1037 # endif
1038         )
1039     {
1040         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1041         {
1042             struct tagcache_seeklist_entry *seeklist;
1043             struct index_entry *idx = &hdr->indices[i];
1044             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1045                 break ;
1046
1047             /* Skip deleted files. */
1048             if (idx->flag & FLAG_DELETED)
1049                 continue;
1050
1051             /* Go through all filters.. */
1052             for (j = 0; j < tcs->filter_count; j++)
1053             {
1054                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1055                 {
1056                     break ;
1057                 }
1058             }
1059
1060             if (j < tcs->filter_count)
1061                 continue ;
1062
1063             /* Check for conditions. */
1064             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1065                 continue;
1066
1067             /* Add to the seek list if not already in uniq buffer. */
1068             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1069                 continue;
1070
1071             /* Lets add it. */
1072             seeklist = &tcs->seeklist[tcs->seek_list_count];
1073             seeklist->seek = idx->tag_seek[tcs->type];
1074             seeklist->flag = idx->flag;
1075             seeklist->idx_id = i;
1076             tcs->seek_list_count++;
1077         }
1078
1079         tcs->seek_pos = i;
1080
1081         return tcs->seek_list_count > 0;
1082     }
1083 #endif
1084
1085     if (tcs->masterfd < 0)
1086     {
1087         struct master_header tcmh;
1088         tcs->masterfd = open_master_fd(&tcmh, false);
1089     }
1090
1091     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1092             sizeof(struct master_header), SEEK_SET);
1093
1094     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1095            == sizeof(struct index_entry))
1096     {
1097         struct tagcache_seeklist_entry *seeklist;
1098
1099         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1100             break ;
1101
1102         i = tcs->seek_pos;
1103         tcs->seek_pos++;
1104
1105         /* Check if entry has been deleted. */
1106         if (entry.flag & FLAG_DELETED)
1107             continue;
1108
1109         /* Go through all filters.. */
1110         for (j = 0; j < tcs->filter_count; j++)
1111         {
1112             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1113                 break ;
1114         }
1115
1116         if (j < tcs->filter_count)
1117             continue ;
1118
1119         /* Check for conditions. */
1120         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1121             continue;
1122
1123         /* Add to the seek list if not already in uniq buffer. */
1124         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1125             continue;
1126
1127         /* Lets add it. */
1128         seeklist = &tcs->seeklist[tcs->seek_list_count];
1129         seeklist->seek = entry.tag_seek[tcs->type];
1130         seeklist->flag = entry.flag;
1131         seeklist->idx_id = i;
1132         tcs->seek_list_count++;
1133
1134         yield();
1135     }
1136
1137     return tcs->seek_list_count > 0;
1138 }
1139
1140
1141 static void remove_files(void)
1142 {
1143     int i;
1144     char buf[MAX_PATH];
1145
1146     tc_stat.ready = false;
1147     tc_stat.ramcache = false;
1148     tc_stat.econ = false;
1149     remove(TAGCACHE_FILE_MASTER);
1150     for (i = 0; i < TAG_COUNT; i++)
1151     {
1152         if (TAGCACHE_IS_NUMERIC(i))
1153             continue;
1154
1155         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1156         remove(buf);
1157     }
1158 }
1159
1160
1161 static bool check_all_headers(void)
1162 {
1163     struct master_header myhdr;
1164     struct tagcache_header tch;
1165     int tag;
1166     int fd;
1167
1168     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1169         return false;
1170
1171     close(fd);
1172     if (myhdr.dirty)
1173     {
1174         logf("tagcache is dirty!");
1175         return false;
1176     }
1177
1178     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1179
1180     for (tag = 0; tag < TAG_COUNT; tag++)
1181     {
1182         if (TAGCACHE_IS_NUMERIC(tag))
1183             continue;
1184
1185         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1186             return false;
1187
1188         close(fd);
1189     }
1190
1191     return true;
1192 }
1193
1194 bool tagcache_is_busy(void)
1195 {
1196     return read_lock || write_lock;
1197 }
1198
1199 bool tagcache_search(struct tagcache_search *tcs, int tag)
1200 {
1201     struct tagcache_header tag_hdr;
1202     struct master_header   master_hdr;
1203     int i;
1204
1205     while (read_lock)
1206         sleep(1);
1207
1208     memset(tcs, 0, sizeof(struct tagcache_search));
1209     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1210         return false;
1211
1212     tcs->position = sizeof(struct tagcache_header);
1213     tcs->type = tag;
1214     tcs->seek_pos = 0;
1215     tcs->list_position = 0;
1216     tcs->seek_list_count = 0;
1217     tcs->filter_count = 0;
1218     tcs->masterfd = -1;
1219
1220     for (i = 0; i < TAG_COUNT; i++)
1221         tcs->idxfd[i] = -1;
1222
1223 #ifndef HAVE_TC_RAMCACHE
1224     tcs->ramsearch = false;
1225 #else
1226     tcs->ramsearch = tc_stat.ramcache;
1227     if (tcs->ramsearch)
1228     {
1229         tcs->entry_count = hdr->entry_count[tcs->type];
1230     }
1231     else
1232 #endif
1233     {
1234         /* Always open as R/W so we can pass tcs to functions that modify data also
1235          * without failing. */
1236         tcs->masterfd = open_master_fd(&master_hdr, true);
1237         if (tcs->masterfd < 0)
1238             return false;
1239
1240         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1241         {
1242             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1243             if (tcs->idxfd[tcs->type] < 0)
1244                 return false;
1245
1246             tcs->entry_count = tag_hdr.entry_count;
1247         }
1248         else
1249         {
1250             tcs->entry_count = master_hdr.tch.entry_count;
1251         }
1252     }
1253
1254     tcs->valid = true;
1255     tcs->initialized = true;
1256     write_lock++;
1257
1258     return true;
1259 }
1260
1261 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1262                                  void *buffer, long length)
1263 {
1264     tcs->unique_list = (unsigned long *)buffer;
1265     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1266     tcs->unique_list_count = 0;
1267 }
1268
1269 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1270                                 int tag, int seek)
1271 {
1272     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1273         return false;
1274
1275     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1276         return false;
1277
1278     tcs->filter_tag[tcs->filter_count] = tag;
1279     tcs->filter_seek[tcs->filter_count] = seek;
1280     tcs->filter_count++;
1281
1282     return true;
1283 }
1284
1285 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1286                                 struct tagcache_search_clause *clause)
1287 {
1288     int i;
1289
1290     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1291     {
1292         logf("Too many clauses");
1293         return false;
1294     }
1295
1296     /* Check if there is already a similar filter in present (filters are
1297      * much faster than clauses).
1298      */
1299     for (i = 0; i < tcs->filter_count; i++)
1300     {
1301         if (tcs->filter_tag[i] == clause->tag)
1302             return true;
1303     }
1304
1305     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1306     {
1307         char buf[MAX_PATH];
1308
1309         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1310         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1311     }
1312
1313     tcs->clause[tcs->clause_count] = clause;
1314     tcs->clause_count++;
1315
1316     return true;
1317 }
1318
1319 static bool get_next(struct tagcache_search *tcs)
1320 {
1321     static char buf[TAG_MAXLEN+32];
1322     struct tagfile_entry entry;
1323     long flag = 0;
1324
1325     if (!tcs->valid || !tc_stat.ready)
1326         return false;
1327
1328     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1329 #ifdef HAVE_TC_RAMCACHE
1330         && !tcs->ramsearch
1331 #endif
1332         )
1333         return false;
1334
1335     /* Relative fetch. */
1336     if (tcs->filter_count > 0 || tcs->clause_count > 0
1337         || TAGCACHE_IS_NUMERIC(tcs->type)
1338 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1339         /* We need to retrieve flag status for dircache. */
1340         || (tcs->ramsearch && tcs->type == tag_filename)
1341 #endif
1342         )
1343     {
1344         struct tagcache_seeklist_entry *seeklist;
1345
1346         /* Check for end of list. */
1347         if (tcs->list_position == tcs->seek_list_count)
1348         {
1349             tcs->list_position = 0;
1350
1351             /* Try to fetch more. */
1352             if (!build_lookup_list(tcs))
1353             {
1354                 tcs->valid = false;
1355                 return false;
1356             }
1357         }
1358
1359         seeklist = &tcs->seeklist[tcs->list_position];
1360         flag = seeklist->flag;
1361         tcs->position = seeklist->seek;
1362         tcs->idx_id = seeklist->idx_id;
1363         tcs->list_position++;
1364     }
1365     else
1366     {
1367         if (tcs->entry_count == 0)
1368         {
1369             tcs->valid = false;
1370             return false;
1371         }
1372
1373         tcs->entry_count--;
1374     }
1375
1376     tcs->result_seek = tcs->position;
1377
1378     if (TAGCACHE_IS_NUMERIC(tcs->type))
1379     {
1380         snprintf(buf, sizeof(buf), "%d", tcs->position);
1381         tcs->result = buf;
1382         tcs->result_len = strlen(buf) + 1;
1383         return true;
1384     }
1385
1386     /* Direct fetch. */
1387 #ifdef HAVE_TC_RAMCACHE
1388     if (tcs->ramsearch)
1389     {
1390
1391 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1392         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1393             && is_dircache_intact())
1394         {
1395             dircache_copy_path((struct dircache_entry *)tcs->position,
1396                                buf, sizeof buf);
1397             tcs->result = buf;
1398             tcs->result_len = strlen(buf) + 1;
1399             tcs->ramresult = false;
1400
1401             return true;
1402         }
1403         else
1404 #endif
1405         if (tcs->type != tag_filename)
1406         {
1407             struct tagfile_entry *ep;
1408
1409             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1410             tcs->result = ep->tag_data;
1411             tcs->result_len = strlen(tcs->result) + 1;
1412             tcs->idx_id = ep->idx_id;
1413             tcs->ramresult = true;
1414
1415             /* Increase position for the next run. This may get overwritten. */
1416             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1417
1418             return true;
1419         }
1420     }
1421 #endif
1422
1423     if (!open_files(tcs, tcs->type))
1424     {
1425         tcs->valid = false;
1426         return false;
1427     }
1428
1429     /* Seek stream to the correct position and continue to direct fetch. */
1430     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1431
1432     if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1433                tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1434     {
1435         logf("read error #5");
1436         tcs->valid = false;
1437         return false;
1438     }
1439
1440     if (entry.tag_length > (long)sizeof(buf))
1441     {
1442         tcs->valid = false;
1443         logf("too long tag #2");
1444         logf("P:%X/%X", tcs->position, entry.tag_length);
1445         return false;
1446     }
1447
1448     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1449     {
1450         tcs->valid = false;
1451         logf("read error #4");
1452         return false;
1453     }
1454
1455     /**
1456      Update the position for the next read (this may be overridden
1457      if filters or clauses are being used).
1458      */
1459     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1460     tcs->result = buf;
1461     tcs->result_len = strlen(tcs->result) + 1;
1462     tcs->idx_id = entry.idx_id;
1463     tcs->ramresult = false;
1464
1465     return true;
1466 }
1467
1468 bool tagcache_get_next(struct tagcache_search *tcs)
1469 {
1470     while (get_next(tcs))
1471     {
1472         if (tcs->result_len > 1)
1473             return true;
1474     }
1475
1476     return false;
1477 }
1478
1479 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1480                        int tag, char *buf, long size)
1481 {
1482     struct index_entry idx;
1483
1484     *buf = '\0';
1485     if (!get_index(tcs->masterfd, idxid, &idx, true))
1486         return false;
1487
1488     return retrieve(tcs, &idx, tag, buf, size);
1489 }
1490
1491 static bool update_master_header(void)
1492 {
1493     struct master_header myhdr;
1494     int fd;
1495
1496     if (!tc_stat.ready)
1497         return false;
1498
1499     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1500         return false;
1501
1502     myhdr.serial = current_tcmh.serial;
1503     myhdr.commitid = current_tcmh.commitid;
1504     myhdr.dirty = current_tcmh.dirty;
1505
1506     /* Write it back */
1507     lseek(fd, 0, SEEK_SET);
1508     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1509     close(fd);
1510
1511 #ifdef HAVE_TC_RAMCACHE
1512     if (hdr)
1513     {
1514         hdr->h.serial = current_tcmh.serial;
1515         hdr->h.commitid = current_tcmh.commitid;
1516         hdr->h.dirty = current_tcmh.dirty;
1517     }
1518 #endif
1519
1520     return true;
1521 }
1522
1523 #if 0
1524
1525 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1526 {
1527     struct tagentry *entry;
1528
1529     if (tcs->type != tag_title)
1530         return ;
1531
1532     /* We will need reserve buffer for this. */
1533     if (tcs->ramcache)
1534     {
1535         struct tagfile_entry *ep;
1536
1537         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1538         tcs->seek_list[tcs->seek_list_count];
1539     }
1540
1541     entry = find_entry_ram();
1542
1543 }
1544 #endif
1545
1546 void tagcache_search_finish(struct tagcache_search *tcs)
1547 {
1548     int i;
1549
1550     if (!tcs->initialized)
1551         return;
1552
1553     if (tcs->masterfd >= 0)
1554     {
1555         close(tcs->masterfd);
1556         tcs->masterfd = -1;
1557     }
1558
1559     for (i = 0; i < TAG_COUNT; i++)
1560     {
1561         if (tcs->idxfd[i] >= 0)
1562         {
1563             close(tcs->idxfd[i]);
1564             tcs->idxfd[i] = -1;
1565         }
1566     }
1567
1568     tcs->ramsearch = false;
1569     tcs->valid = false;
1570     tcs->initialized = 0;
1571     if (write_lock > 0)
1572         write_lock--;
1573 }
1574
1575 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1576 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1577 {
1578     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1579 }
1580
1581 static long get_tag_numeric(const struct index_entry *entry, int tag)
1582 {
1583     return check_virtual_tags(tag, entry);
1584 }
1585
1586 static char* get_tag_string(const struct index_entry *entry, int tag)
1587 {
1588     char* s = get_tag(entry, tag)->tag_data;
1589     return strcmp(s, UNTAGGED) ? s : NULL;
1590 }
1591
1592 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1593 {
1594     struct index_entry *entry;
1595     int idx_id;
1596
1597     if (!tc_stat.ready || !tc_stat.ramcache)
1598         return false;
1599
1600     /* Find the corresponding entry in tagcache. */
1601     idx_id = find_entry_ram(filename, NULL);
1602     if (idx_id < 0)
1603         return false;
1604
1605     entry = &hdr->indices[idx_id];
1606
1607     id3->title        = get_tag_string(entry, tag_title);
1608     id3->artist       = get_tag_string(entry, tag_artist);
1609     id3->album        = get_tag_string(entry, tag_album);
1610     id3->genre_string = get_tag_string(entry, tag_genre);
1611     id3->composer     = get_tag_string(entry, tag_composer);
1612     id3->comment      = get_tag_string(entry, tag_comment);
1613     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1614     id3->grouping     = get_tag_string(entry, tag_grouping);
1615
1616     id3->length     = get_tag_numeric(entry, tag_length);
1617     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1618     id3->rating     = get_tag_numeric(entry, tag_rating);
1619     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1620     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1621     id3->year       = get_tag_numeric(entry, tag_year);
1622
1623     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1624     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1625     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1626     if (id3->bitrate == 0)
1627         id3->bitrate = 1;
1628
1629     return true;
1630 }
1631 #endif
1632
1633 static inline void write_item(const char *item)
1634 {
1635     int len = strlen(item) + 1;
1636
1637     data_size += len;
1638     write(cachefd, item, len);
1639 }
1640
1641 static int check_if_empty(char **tag)
1642 {
1643     int length;
1644
1645     if (*tag == NULL || **tag == '\0')
1646     {
1647         *tag = UNTAGGED;
1648         return sizeof(UNTAGGED); /* Tag length */
1649     }
1650
1651     length = strlen(*tag);
1652     if (length > TAG_MAXLEN)
1653     {
1654         logf("over length tag: %s", *tag);
1655         length = TAG_MAXLEN;
1656         (*tag)[length] = '\0';
1657     }
1658
1659     return length + 1;
1660 }
1661
1662 #define ADD_TAG(entry,tag,data) \
1663     /* Adding tag */ \
1664     entry.tag_offset[tag] = offset; \
1665     entry.tag_length[tag] = check_if_empty(data); \
1666     offset += entry.tag_length[tag]
1667 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1668  * idea, as it uses lots of stack and is called from a recursive function
1669  * (check_dir).
1670  */
1671 static void __attribute__ ((noinline)) add_tagcache(char *path,
1672                                                     unsigned long mtime
1673 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1674                                                     ,const struct dircache_entry *dc
1675 #endif
1676                                                    )
1677 {
1678     struct mp3entry id3;
1679     struct temp_file_entry entry;
1680     bool ret;
1681     int fd;
1682     int idx_id = -1;
1683     char tracknumfix[3];
1684     int offset = 0;
1685     int path_length = strlen(path);
1686     bool has_albumartist;
1687     bool has_grouping;
1688
1689 #ifdef SIMULATOR
1690     /* Crude logging for the sim - to aid in debugging */
1691     int logfd = open(ROCKBOX_DIR "/database.log",
1692                      O_WRONLY | O_APPEND | O_CREAT);
1693     if (logfd >= 0) {
1694         write(logfd, path, strlen(path));
1695         write(logfd, "\n", 1);
1696         close(logfd);
1697     }
1698 #endif
1699
1700     if (cachefd < 0)
1701         return ;
1702
1703     /* Check for overlength file path. */
1704     if (path_length > TAG_MAXLEN)
1705     {
1706         /* Path can't be shortened. */
1707         logf("Too long path: %s", path);
1708         return ;
1709     }
1710
1711     /* Check if the file is supported. */
1712     if (probe_file_format(path) == AFMT_UNKNOWN)
1713         return ;
1714
1715     /* Check if the file is already cached. */
1716 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1717     if (tc_stat.ramcache && is_dircache_intact())
1718     {
1719         idx_id = find_entry_ram(path, dc);
1720     }
1721 #endif
1722
1723     /* Be sure the entry doesn't exist. */
1724     if (filenametag_fd >= 0 && idx_id < 0)
1725         idx_id = find_entry_disk(path, false);
1726
1727     /* Check if file has been modified. */
1728     if (idx_id >= 0)
1729     {
1730         struct index_entry idx;
1731
1732         /* TODO: Mark that the index exists (for fast reverse scan) */
1733         //found_idx[idx_id/8] |= idx_id%8;
1734
1735         if (!get_index(-1, idx_id, &idx, true))
1736         {
1737             logf("failed to retrieve index entry");
1738             return ;
1739         }
1740
1741         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1742         {
1743             /* No changes to file. */
1744             return ;
1745         }
1746
1747         /* Metadata might have been changed. Delete the entry. */
1748         logf("Re-adding: %s", path);
1749         if (!delete_entry(idx_id))
1750         {
1751             logf("delete_entry failed: %d", idx_id);
1752             return ;
1753         }
1754     }
1755
1756     fd = open(path, O_RDONLY);
1757     if (fd < 0)
1758     {
1759         logf("open fail: %s", path);
1760         return ;
1761     }
1762
1763     memset(&id3, 0, sizeof(struct mp3entry));
1764     memset(&entry, 0, sizeof(struct temp_file_entry));
1765     memset(&tracknumfix, 0, sizeof(tracknumfix));
1766     ret = get_metadata(&id3, fd, path);
1767     close(fd);
1768
1769     if (!ret)
1770         return ;
1771
1772     logf("-> %s", path);
1773
1774     /* Generate track number if missing. */
1775     if (id3.tracknum <= 0)
1776     {
1777         const char *p = strrchr(path, '.');
1778
1779         if (p == NULL)
1780             p = &path[strlen(path)-1];
1781
1782         while (*p != '/')
1783         {
1784             if (isdigit(*p) && isdigit(*(p-1)))
1785             {
1786                 tracknumfix[1] = *p--;
1787                 tracknumfix[0] = *p;
1788                 break;
1789             }
1790             p--;
1791         }
1792
1793         if (tracknumfix[0] != '\0')
1794         {
1795             id3.tracknum = atoi(tracknumfix);
1796             /* Set a flag to indicate track number has been generated. */
1797             entry.flag |= FLAG_TRKNUMGEN;
1798         }
1799         else
1800         {
1801             /* Unable to generate track number. */
1802             id3.tracknum = -1;
1803         }
1804     }
1805
1806     /* Numeric tags */
1807     entry.tag_offset[tag_year] = id3.year;
1808     entry.tag_offset[tag_discnumber] = id3.discnum;
1809     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1810     entry.tag_offset[tag_length] = id3.length;
1811     entry.tag_offset[tag_bitrate] = id3.bitrate;
1812     entry.tag_offset[tag_mtime] = mtime;
1813
1814     /* String tags. */
1815     has_albumartist = id3.albumartist != NULL
1816         && strlen(id3.albumartist) > 0;
1817     has_grouping = id3.grouping != NULL
1818         && strlen(id3.grouping) > 0;
1819
1820     ADD_TAG(entry, tag_filename, &path);
1821     ADD_TAG(entry, tag_title, &id3.title);
1822     ADD_TAG(entry, tag_artist, &id3.artist);
1823     ADD_TAG(entry, tag_album, &id3.album);
1824     ADD_TAG(entry, tag_genre, &id3.genre_string);
1825     ADD_TAG(entry, tag_composer, &id3.composer);
1826     ADD_TAG(entry, tag_comment, &id3.comment);
1827     if (has_albumartist)
1828     {
1829         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1830     }
1831     else
1832     {
1833         ADD_TAG(entry, tag_albumartist, &id3.artist);
1834     }
1835     if (has_grouping)
1836     {
1837         ADD_TAG(entry, tag_grouping, &id3.grouping);
1838     }
1839     else
1840     {
1841         ADD_TAG(entry, tag_grouping, &id3.title);
1842     }
1843     entry.data_length = offset;
1844
1845     /* Write the header */
1846     write(cachefd, &entry, sizeof(struct temp_file_entry));
1847
1848     /* And tags also... Correct order is critical */
1849     write_item(path);
1850     write_item(id3.title);
1851     write_item(id3.artist);
1852     write_item(id3.album);
1853     write_item(id3.genre_string);
1854     write_item(id3.composer);
1855     write_item(id3.comment);
1856     if (has_albumartist)
1857     {
1858         write_item(id3.albumartist);
1859     }
1860     else
1861     {
1862         write_item(id3.artist);
1863     }
1864     if (has_grouping)
1865     {
1866         write_item(id3.grouping);
1867     }
1868     else
1869     {
1870         write_item(id3.title);
1871     }
1872     total_entry_count++;
1873 }
1874
1875 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1876 {
1877     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1878     int len = strlen(str)+1;
1879     int i;
1880     unsigned crc32;
1881     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1882     char buf[TAG_MAXLEN+32];
1883
1884     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1885         buf[i] = tolower(str[i]);
1886     buf[i] = '\0';
1887
1888     crc32 = crc_32(buf, i, 0xffffffff);
1889
1890     if (unique)
1891     {
1892         /* Check if the crc does not exist -> entry does not exist for sure. */
1893         for (i = 0; i < tempbufidx; i++)
1894         {
1895             if (crcbuf[-i] != crc32)
1896                 continue;
1897
1898             if (!strcasecmp(str, index[i].str))
1899             {
1900                 if (id < 0 || id >= lookup_buffer_depth)
1901                 {
1902                     logf("lookup buf overf.: %d", id);
1903                     return false;
1904                 }
1905
1906                 lookup[id] = &index[i];
1907                 return true;
1908             }
1909         }
1910     }
1911
1912     /* Insert to CRC buffer. */
1913     crcbuf[-tempbufidx] = crc32;
1914     tempbuf_left -= 4;
1915
1916     /* Insert it to the buffer. */
1917     tempbuf_left -= len;
1918     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1919         return false;
1920
1921     if (id >= lookup_buffer_depth)
1922     {
1923         logf("lookup buf overf. #2: %d", id);
1924         return false;
1925     }
1926
1927     if (id >= 0)
1928     {
1929         lookup[id] = &index[tempbufidx];
1930         index[tempbufidx].idlist.id = id;
1931     }
1932     else
1933         index[tempbufidx].idlist.id = -1;
1934
1935     index[tempbufidx].idlist.next = NULL;
1936     index[tempbufidx].idx_id = idx_id;
1937     index[tempbufidx].seek = -1;
1938     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1939     memcpy(index[tempbufidx].str, str, len);
1940     tempbuf_pos += len;
1941     tempbufidx++;
1942
1943     return true;
1944 }
1945
1946 static int compare(const void *p1, const void *p2)
1947 {
1948     do_timed_yield();
1949
1950     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1951     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1952
1953     if (strcmp(e1->str, UNTAGGED) == 0)
1954     {
1955         if (strcmp(e2->str, UNTAGGED) == 0)
1956             return 0;
1957         return -1;
1958     }
1959     else if (strcmp(e2->str, UNTAGGED) == 0)
1960         return 1;
1961
1962     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1963 }
1964
1965 static int tempbuf_sort(int fd)
1966 {
1967     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1968     struct tagfile_entry fe;
1969     int i;
1970     int length;
1971
1972     /* Generate reverse lookup entries. */
1973     for (i = 0; i < lookup_buffer_depth; i++)
1974     {
1975         struct tempbuf_id_list *idlist;
1976
1977         if (!lookup[i])
1978             continue;
1979
1980         if (lookup[i]->idlist.id == i)
1981             continue;
1982
1983         idlist = &lookup[i]->idlist;
1984         while (idlist->next != NULL)
1985             idlist = idlist->next;
1986
1987         tempbuf_left -= sizeof(struct tempbuf_id_list);
1988         if (tempbuf_left - 4 < 0)
1989             return -1;
1990
1991         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1992         if (tempbuf_pos & 0x03)
1993         {
1994             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1995             tempbuf_left -= 3;
1996             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1997         }
1998         tempbuf_pos += sizeof(struct tempbuf_id_list);
1999
2000         idlist = idlist->next;
2001         idlist->id = i;
2002         idlist->next = NULL;
2003
2004         do_timed_yield();
2005     }
2006
2007     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2008     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2009
2010     for (i = 0; i < tempbufidx; i++)
2011     {
2012         struct tempbuf_id_list *idlist = &index[i].idlist;
2013
2014         /* Fix the lookup list. */
2015         while (idlist != NULL)
2016         {
2017             if (idlist->id >= 0)
2018                 lookup[idlist->id] = &index[i];
2019             idlist = idlist->next;
2020         }
2021
2022         index[i].seek = lseek(fd, 0, SEEK_CUR);
2023         length = strlen(index[i].str) + 1;
2024         fe.tag_length = length;
2025         fe.idx_id = index[i].idx_id;
2026
2027         /* Check the chunk alignment. */
2028         if ((fe.tag_length + sizeof(struct tagfile_entry))
2029             % TAGFILE_ENTRY_CHUNK_LENGTH)
2030         {
2031             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2032                 ((fe.tag_length + sizeof(struct tagfile_entry))
2033                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2034         }
2035
2036 #ifdef TAGCACHE_STRICT_ALIGN
2037         /* Make sure the entry is long aligned. */
2038         if (index[i].seek & 0x03)
2039         {
2040             logf("tempbuf_sort: alignment error!");
2041             return -3;
2042         }
2043 #endif
2044
2045         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2046             sizeof(struct tagfile_entry))
2047         {
2048             logf("tempbuf_sort: write error #1");
2049             return -1;
2050         }
2051
2052         if (write(fd, index[i].str, length) != length)
2053         {
2054             logf("tempbuf_sort: write error #2");
2055             return -2;
2056         }
2057
2058         /* Write some padding. */
2059         if (fe.tag_length - length > 0)
2060             write(fd, "XXXXXXXX", fe.tag_length - length);
2061     }
2062
2063     return i;
2064 }
2065
2066 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2067 {
2068     if (id < 0 || id >= lookup_buffer_depth)
2069         return NULL;
2070
2071     return lookup[id];
2072 }
2073
2074
2075 inline static int tempbuf_find_location(int id)
2076 {
2077     struct tempbuf_searchidx *entry;
2078
2079     entry = tempbuf_locate(id);
2080     if (entry == NULL)
2081         return -1;
2082
2083     return entry->seek;
2084 }
2085
2086 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2087 {
2088     struct master_header tcmh;
2089     struct index_entry idx;
2090     int masterfd;
2091     int masterfd_pos;
2092     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2093     int max_entries;
2094     int entries_processed = 0;
2095     int i, j;
2096     char buf[TAG_MAXLEN];
2097
2098     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2099
2100     logf("Building numeric indices...");
2101     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2102
2103     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2104         return false;
2105
2106     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2107                          SEEK_CUR);
2108     if (masterfd_pos == filesize(masterfd))
2109     {
2110         logf("we can't append!");
2111         close(masterfd);
2112         return false;
2113     }
2114
2115     while (entries_processed < h->entry_count)
2116     {
2117         int count = MIN(h->entry_count - entries_processed, max_entries);
2118
2119         /* Read in as many entries as possible. */
2120         for (i = 0; i < count; i++)
2121         {
2122             struct temp_file_entry *tfe = &entrybuf[i];
2123             int datastart;
2124
2125             /* Read in numeric data. */
2126             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2127                 sizeof(struct temp_file_entry))
2128             {
2129                 logf("read fail #1");
2130                 close(masterfd);
2131                 return false;
2132             }
2133
2134             datastart = lseek(tmpfd, 0, SEEK_CUR);
2135
2136             /**
2137              * Read string data from the following tags:
2138              * - tag_filename
2139              * - tag_artist
2140              * - tag_album
2141              * - tag_title
2142              *
2143              * A crc32 hash is calculated from the read data
2144              * and stored back to the data offset field kept in memory.
2145              */
2146 #define tmpdb_read_string_tag(tag) \
2147     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2148     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2149     { \
2150         logf("read fail: buffer overflow"); \
2151         close(masterfd); \
2152         return false; \
2153     } \
2154     \
2155     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2156         tfe->tag_length[tag]) \
2157     { \
2158         logf("read fail #2"); \
2159         close(masterfd); \
2160         return false; \
2161     } \
2162     \
2163     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2164     lseek(tmpfd, datastart, SEEK_SET)
2165
2166             tmpdb_read_string_tag(tag_filename);
2167             tmpdb_read_string_tag(tag_artist);
2168             tmpdb_read_string_tag(tag_album);
2169             tmpdb_read_string_tag(tag_title);
2170
2171             /* Seek to the end of the string data. */
2172             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2173         }
2174
2175         /* Backup the master index position. */
2176         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2177         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2178
2179         /* Check if we can resurrect some deleted runtime statistics data. */
2180         for (i = 0; i < tcmh.tch.entry_count; i++)
2181         {
2182             /* Read the index entry. */
2183             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2184                 != sizeof(struct index_entry))
2185             {
2186                 logf("read fail #3");
2187                 close(masterfd);
2188                 return false;
2189             }
2190
2191             /**
2192              * Skip unless the entry is marked as being deleted
2193              * or the data has already been resurrected.
2194              */
2195             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2196                 continue;
2197
2198             /* Now try to match the entry. */
2199             /**
2200              * To succesfully match a song, the following conditions
2201              * must apply:
2202              *
2203              * For numeric fields: tag_length
2204              * - Full identical match is required
2205              *
2206              * If tag_filename matches, no further checking necessary.
2207              *
2208              * For string hashes: tag_artist, tag_album, tag_title
2209              * - Two of these must match
2210              */
2211             for (j = 0; j < count; j++)
2212             {
2213                 struct temp_file_entry *tfe = &entrybuf[j];
2214
2215                 /* Try to match numeric fields first. */
2216                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2217                     continue;
2218
2219                 /* Now it's time to do the hash matching. */
2220                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2221                 {
2222                     int match_count = 0;
2223
2224                     /* No filename match, check if we can match two other tags. */
2225 #define tmpdb_match(tag) \
2226     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2227         match_count++
2228
2229                     tmpdb_match(tag_artist);
2230                     tmpdb_match(tag_album);
2231                     tmpdb_match(tag_title);
2232
2233                     if (match_count < 2)
2234                     {
2235                         /* Still no match found, give up. */
2236                         continue;
2237                     }
2238                 }
2239
2240                 /* A match found, now copy & resurrect the statistical data. */
2241 #define tmpdb_copy_tag(tag) \
2242     tfe->tag_offset[tag] = idx.tag_seek[tag]
2243
2244                 tmpdb_copy_tag(tag_playcount);
2245                 tmpdb_copy_tag(tag_rating);
2246                 tmpdb_copy_tag(tag_playtime);
2247                 tmpdb_copy_tag(tag_lastplayed);
2248                 tmpdb_copy_tag(tag_commitid);
2249
2250                 /* Avoid processing this entry again. */
2251                 idx.flag |= FLAG_RESURRECTED;
2252
2253                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2254                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2255                     != sizeof(struct index_entry))
2256                 {
2257                     logf("masterfd writeback fail #1");
2258                     close(masterfd);
2259                     return false;
2260                 }
2261
2262                 logf("Entry resurrected");
2263             }
2264         }
2265
2266
2267         /* Restore the master index position. */
2268         lseek(masterfd, masterfd_pos, SEEK_SET);
2269
2270         /* Commit the data to the index. */
2271         for (i = 0; i < count; i++)
2272         {
2273             int loc = lseek(masterfd, 0, SEEK_CUR);
2274
2275             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2276                 != sizeof(struct index_entry))
2277             {
2278                 logf("read fail #3");
2279                 close(masterfd);
2280                 return false;
2281             }
2282
2283             for (j = 0; j < TAG_COUNT; j++)
2284             {
2285                 if (!TAGCACHE_IS_NUMERIC(j))
2286                     continue;
2287
2288                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2289             }
2290             idx.flag = entrybuf[i].flag;
2291
2292             if (idx.tag_seek[tag_commitid])
2293             {
2294                 /* Data has been resurrected. */
2295                 idx.flag |= FLAG_DIRTYNUM;
2296             }
2297             else if (tc_stat.ready && current_tcmh.commitid > 0)
2298             {
2299                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2300                 idx.flag |= FLAG_DIRTYNUM;
2301             }
2302
2303             /* Write back the updated index. */
2304             lseek(masterfd, loc, SEEK_SET);
2305             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2306                 != sizeof(struct index_entry))
2307             {
2308                 logf("write fail");
2309                 close(masterfd);
2310                 return false;
2311             }
2312         }
2313
2314         entries_processed += count;
2315         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2316     }
2317
2318     close(masterfd);
2319
2320     return true;
2321 }
2322
2323 /**
2324  * Return values:
2325  *     > 0   success
2326  *    == 0   temporary failure
2327  *     < 0   fatal error
2328  */
2329 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2330 {
2331     int i;
2332     struct tagcache_header tch;
2333     struct master_header   tcmh;
2334     struct index_entry idxbuf[IDX_BUF_DEPTH];
2335     int idxbuf_pos;
2336     char buf[TAG_MAXLEN+32];
2337     int fd = -1, masterfd;
2338     bool error = false;
2339     int init;
2340     int masterfd_pos;
2341
2342     logf("Building index: %d", index_type);
2343
2344     /* Check the number of entries we need to allocate ram for. */
2345     commit_entry_count = h->entry_count + 1;
2346
2347     masterfd = open_master_fd(&tcmh, false);
2348     if (masterfd >= 0)
2349     {
2350         commit_entry_count += tcmh.tch.entry_count;
2351         close(masterfd);
2352     }
2353     else
2354         remove_files(); /* Just to be sure we are clean. */
2355
2356     /* Open the index file, which contains the tag names. */
2357     fd = open_tag_fd(&tch, index_type, true);
2358     if (fd >= 0)
2359     {
2360         logf("tch.datasize=%ld", tch.datasize);
2361         lookup_buffer_depth = 1 +
2362         /* First part */ commit_entry_count +
2363         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2364     }
2365     else
2366     {
2367         lookup_buffer_depth = 1 +
2368         /* First part */ commit_entry_count +
2369         /* Second part */ 0;
2370     }
2371
2372     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2373     logf("commit_entry_count=%ld", commit_entry_count);
2374
2375     /* Allocate buffer for all index entries from both old and new
2376      * tag files. */
2377     tempbufidx = 0;
2378     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2379
2380     /* Allocate lookup buffer. The first portion of commit_entry_count
2381      * contains the new tags in the temporary file and the second
2382      * part for locating entries already in the db.
2383      *
2384      *  New tags  Old tags
2385      * +---------+---------------------------+
2386      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2387      * +---------+---------------------------+
2388      *
2389      * Old tags are inserted to a temporary buffer with position:
2390      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2391      * And new tags with index:
2392      *     tempbuf_insert(idx, ...);
2393      *
2394      * The buffer is sorted and written into tag file:
2395      *     tempbuf_sort(...);
2396      * leaving master index locations messed up.
2397      *
2398      * That is fixed using the lookup buffer for old tags:
2399      *     new_seek = tempbuf_find_location(old_seek, ...);
2400      * and for new tags:
2401      *     new_seek = tempbuf_find_location(idx);
2402      */
2403     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2404     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2405     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2406
2407     /* And calculate the remaining data space used mainly for storing
2408      * tag data (strings). */
2409     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2410     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2411     {
2412         logf("Buffer way too small!");
2413         return 0;
2414     }
2415
2416     if (fd >= 0)
2417     {
2418         /**
2419          * If tag file contains unique tags (sorted index), we will load
2420          * it entirely into memory so we can resort it later for use with
2421          * chunked browsing.
2422          */
2423         if (TAGCACHE_IS_SORTED(index_type))
2424         {
2425             logf("loading tags...");
2426             for (i = 0; i < tch.entry_count; i++)
2427             {
2428                 struct tagfile_entry entry;
2429                 int loc = lseek(fd, 0, SEEK_CUR);
2430                 bool ret;
2431
2432                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2433                     != sizeof(struct tagfile_entry))
2434                 {
2435                     logf("read error #7");
2436                     close(fd);
2437                     return -2;
2438                 }
2439
2440                 if (entry.tag_length >= (int)sizeof(buf))
2441                 {
2442                     logf("too long tag #3");
2443                     close(fd);
2444                     return -2;
2445                 }
2446
2447                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2448                 {
2449                     logf("read error #8");
2450                     close(fd);
2451                     return -2;
2452                 }
2453
2454                 /* Skip deleted entries. */
2455                 if (buf[0] == '\0')
2456                     continue;
2457
2458                 /**
2459                  * Save the tag and tag id in the memory buffer. Tag id
2460                  * is saved so we can later reindex the master lookup
2461                  * table when the index gets resorted.
2462                  */
2463                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2464                                      + commit_entry_count, entry.idx_id,
2465                                      TAGCACHE_IS_UNIQUE(index_type));
2466                 if (!ret)
2467                 {
2468                     close(fd);
2469                     return -3;
2470                 }
2471                 do_timed_yield();
2472             }
2473             logf("done");
2474         }
2475         else
2476             tempbufidx = tch.entry_count;
2477     }
2478     else
2479     {
2480         /**
2481          * Creating new index file to store the tags. No need to preload
2482          * anything whether the index type is sorted or not.
2483          */
2484         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2485         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2486         if (fd < 0)
2487         {
2488             logf("%s open fail", buf);
2489             return -2;
2490         }
2491
2492         tch.magic = TAGCACHE_MAGIC;
2493         tch.entry_count = 0;
2494         tch.datasize = 0;
2495
2496         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2497             != sizeof(struct tagcache_header))
2498         {
2499             logf("header write failed");
2500             close(fd);
2501             return -2;
2502         }
2503     }
2504
2505     /* Loading the tag lookup file as "master file". */
2506     logf("Loading index file");
2507     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2508
2509     if (masterfd < 0)
2510     {
2511         logf("Creating new DB");
2512         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2513
2514         if (masterfd < 0)
2515         {
2516             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2517             close(fd);
2518             return -2;
2519         }
2520
2521         /* Write the header (write real values later). */
2522         memset(&tcmh, 0, sizeof(struct master_header));
2523         tcmh.tch = *h;
2524         tcmh.tch.entry_count = 0;
2525         tcmh.tch.datasize = 0;
2526         tcmh.dirty = true;
2527         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2528         init = true;
2529         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2530     }
2531     else
2532     {
2533         /**
2534          * Master file already exists so we need to process the current
2535          * file first.
2536          */
2537         init = false;
2538
2539         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2540             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2541         {
2542             logf("header error");
2543             close(fd);
2544             close(masterfd);
2545             return -2;
2546         }
2547
2548         /**
2549          * If we reach end of the master file, we need to expand it to
2550          * hold new tags. If the current index is not sorted, we can
2551          * simply append new data to end of the file.
2552          * However, if the index is sorted, we need to update all tag
2553          * pointers in the master file for the current index.
2554          */
2555         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2556             SEEK_CUR);
2557         if (masterfd_pos == filesize(masterfd))
2558         {
2559             logf("appending...");
2560             init = true;
2561         }
2562     }
2563
2564     /**
2565      * Load new unique tags in memory to be sorted later and added
2566      * to the master lookup file.
2567      */
2568     if (TAGCACHE_IS_SORTED(index_type))
2569     {
2570         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2571         /* h is the header of the temporary file containing new tags. */
2572         logf("inserting new tags...");
2573         for (i = 0; i < h->entry_count; i++)
2574         {
2575             struct temp_file_entry entry;
2576
2577             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2578                 sizeof(struct temp_file_entry))
2579             {
2580                 logf("read fail #3");
2581                 error = true;
2582                 goto error_exit;
2583             }
2584
2585             /* Read data. */
2586             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2587             {
2588                 logf("too long entry!");
2589                 error = true;
2590                 goto error_exit;
2591             }
2592
2593             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2594             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2595                 entry.tag_length[index_type])
2596             {
2597                 logf("read fail #4");
2598                 error = true;
2599                 goto error_exit;
2600             }
2601
2602             if (TAGCACHE_IS_UNIQUE(index_type))
2603                 error = !tempbuf_insert(buf, i, -1, true);
2604             else
2605                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2606
2607             if (error)
2608             {
2609                 logf("insert error");
2610                 goto error_exit;
2611             }
2612
2613             /* Skip to next. */
2614             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2615                     entry.tag_length[index_type], SEEK_CUR);
2616             do_timed_yield();
2617         }
2618         logf("done");
2619
2620         /* Sort the buffer data and write it to the index file. */
2621         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2622         /**
2623          * We need to truncate the index file now. There can be junk left
2624          * at the end of file (however, we _should_ always follow the
2625          * entry_count and don't crash with that).
2626          */
2627         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2628
2629         i = tempbuf_sort(fd);
2630         if (i < 0)
2631             goto error_exit;
2632         logf("sorted %d tags", i);
2633
2634         /**
2635          * Now update all indexes in the master lookup file.
2636          */
2637         logf("updating indices...");
2638         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2639         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2640         {
2641             int j;
2642             int loc = lseek(masterfd, 0, SEEK_CUR);
2643
2644             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2645
2646             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2647                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2648             {
2649                 logf("read fail #5");
2650                 error = true;
2651                 goto error_exit ;
2652             }
2653             lseek(masterfd, loc, SEEK_SET);
2654
2655             for (j = 0; j < idxbuf_pos; j++)
2656             {
2657                 if (idxbuf[j].flag & FLAG_DELETED)
2658                 {
2659                     /* We can just ignore deleted entries. */
2660                     // idxbuf[j].tag_seek[index_type] = 0;
2661                     continue;
2662                 }
2663
2664                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2665                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2666                     + commit_entry_count);
2667
2668                 if (idxbuf[j].tag_seek[index_type] < 0)
2669                 {
2670                     logf("update error: %d/%d/%d",
2671                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2672                     error = true;
2673                     goto error_exit;
2674                 }
2675
2676                 do_timed_yield();
2677             }
2678
2679             /* Write back the updated index. */
2680             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2681                         index_entry_ec, tc_stat.econ) !=
2682                 (int)sizeof(struct index_entry)*idxbuf_pos)
2683             {
2684                 logf("write fail");
2685                 error = true;
2686                 goto error_exit;
2687             }
2688         }
2689         logf("done");
2690     }
2691
2692     /**
2693      * Walk through the temporary file containing the new tags.
2694      */
2695     // build_normal_index(h, tmpfd, masterfd, idx);
2696     logf("updating new indices...");
2697     lseek(masterfd, masterfd_pos, SEEK_SET);
2698     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2699     lseek(fd, 0, SEEK_END);
2700     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2701     {
2702         int j;
2703
2704         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2705         if (init)
2706         {
2707             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2708         }
2709         else
2710         {
2711             int loc = lseek(masterfd, 0, SEEK_CUR);
2712
2713             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2714                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2715             {
2716                 logf("read fail #6");
2717                 error = true;
2718                 break ;
2719             }
2720             lseek(masterfd, loc, SEEK_SET);
2721         }
2722
2723         /* Read entry headers. */
2724         for (j = 0; j < idxbuf_pos; j++)
2725         {
2726             if (!TAGCACHE_IS_SORTED(index_type))
2727             {
2728                 struct temp_file_entry entry;
2729                 struct tagfile_entry fe;
2730
2731                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2732                     sizeof(struct temp_file_entry))
2733                 {
2734                     logf("read fail #7");
2735                     error = true;
2736                     break ;
2737                 }
2738
2739                 /* Read data. */
2740                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2741                 {
2742                     logf("too long entry!");
2743                     logf("length=%d", entry.tag_length[index_type]);
2744                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2745                     error = true;
2746                     break ;
2747                 }
2748
2749                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2750                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2751                     entry.tag_length[index_type])
2752                 {
2753                     logf("read fail #8");
2754                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2755                     logf("length=0x%02x", entry.tag_length[index_type]);
2756                     error = true;
2757                     break ;
2758                 }
2759
2760                 /* Write to index file. */
2761                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2762                 fe.tag_length = entry.tag_length[index_type];
2763                 fe.idx_id = tcmh.tch.entry_count + i + j;
2764                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2765                 write(fd, buf, fe.tag_length);
2766                 tempbufidx++;
2767
2768                 /* Skip to next. */
2769                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2770                       entry.tag_length[index_type], SEEK_CUR);
2771             }
2772             else
2773             {
2774                 /* Locate the correct entry from the sorted array. */
2775                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2776                 if (idxbuf[j].tag_seek[index_type] < 0)
2777                 {
2778                     logf("entry not found (%d)", j);
2779                     error = true;
2780                     break ;
2781                 }
2782             }
2783         }
2784
2785         /* Write index. */
2786         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2787                   index_entry_ec, tc_stat.econ) !=
2788             (int)sizeof(struct index_entry)*idxbuf_pos)
2789         {
2790             logf("tagcache: write fail #4");
2791             error = true;
2792             break ;
2793         }
2794
2795         do_timed_yield();
2796     }
2797     logf("done");
2798
2799     /* Finally write the header. */
2800     tch.magic = TAGCACHE_MAGIC;
2801     tch.entry_count = tempbufidx;
2802     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2803     lseek(fd, 0, SEEK_SET);
2804     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2805
2806     if (index_type != tag_filename)
2807         h->datasize += tch.datasize;
2808     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2809     error_exit:
2810
2811     close(fd);
2812     close(masterfd);
2813
2814     if (error)
2815         return -2;
2816
2817     return 1;
2818 }
2819
2820 static bool commit(void)
2821 {
2822     struct tagcache_header tch;
2823     struct master_header   tcmh;
2824     int i, len, rc;
2825     int tmpfd;
2826     int masterfd;
2827 #ifdef HAVE_DIRCACHE
2828     bool dircache_buffer_stolen = false;
2829 #endif
2830     bool local_allocation = false;
2831
2832     logf("committing tagcache");
2833
2834     while (write_lock)
2835         sleep(1);
2836
2837     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2838     if (tmpfd < 0)
2839     {
2840         logf("nothing to commit");
2841         return true;
2842     }
2843
2844
2845     /* Load the header. */
2846     len = sizeof(struct tagcache_header);
2847     rc = read(tmpfd, &tch, len);
2848
2849     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2850     {
2851         logf("incorrect header");
2852         close(tmpfd);
2853         remove(TAGCACHE_FILE_TEMP);
2854         return false;
2855     }
2856
2857     if (tch.entry_count == 0)
2858     {
2859         logf("nothing to commit");
2860         close(tmpfd);
2861         remove(TAGCACHE_FILE_TEMP);
2862         return true;
2863     }
2864
2865 #ifdef HAVE_EEPROM_SETTINGS
2866     remove(TAGCACHE_STATEFILE);
2867 #endif
2868
2869     /* At first be sure to unload the ramcache! */
2870 #ifdef HAVE_TC_RAMCACHE
2871     tc_stat.ramcache = false;
2872 #endif
2873
2874     read_lock++;
2875
2876     /* Try to steal every buffer we can :) */
2877     if (tempbuf_size == 0)
2878         local_allocation = true;
2879
2880 #ifdef HAVE_DIRCACHE
2881     if (tempbuf_size == 0)
2882     {
2883         /* Try to steal the dircache buffer. */
2884         tempbuf = dircache_steal_buffer(&tempbuf_size);
2885         tempbuf_size &= ~0x03;
2886
2887         if (tempbuf_size > 0)
2888         {
2889             dircache_buffer_stolen = true;
2890         }
2891     }
2892 #endif
2893
2894 #ifdef HAVE_TC_RAMCACHE
2895     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2896     {
2897         tempbuf = (char *)(hdr + 1);
2898         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2899         tempbuf_size &= ~0x03;
2900     }
2901 #endif
2902
2903     /* And finally fail if there are no buffers available. */
2904     if (tempbuf_size == 0)
2905     {
2906         logf("delaying commit until next boot");
2907         tc_stat.commit_delayed = true;
2908         close(tmpfd);
2909         read_lock--;
2910         return false;
2911     }
2912
2913     logf("commit %ld entries...", tch.entry_count);
2914
2915     /* Mark DB dirty so it will stay disabled if commit fails. */
2916     current_tcmh.dirty = true;
2917     update_master_header();
2918
2919     /* Now create the index files. */
2920     tc_stat.commit_step = 0;
2921     tch.datasize = 0;
2922     tc_stat.commit_delayed = false;
2923
2924     for (i = 0; i < TAG_COUNT; i++)
2925     {
2926         int ret;
2927
2928         if (TAGCACHE_IS_NUMERIC(i))
2929             continue;
2930
2931         tc_stat.commit_step++;
2932         ret = build_index(i, &tch, tmpfd);
2933         if (ret <= 0)
2934         {
2935             close(tmpfd);
2936             logf("tagcache failed init");
2937             if (ret == 0)
2938                 tc_stat.commit_delayed = true;
2939
2940             tc_stat.commit_step = 0;
2941             read_lock--;
2942             return false;
2943         }
2944     }
2945
2946     if (!build_numeric_indices(&tch, tmpfd))
2947     {
2948         logf("Failure to commit numeric indices");
2949         close(tmpfd);
2950         tc_stat.commit_step = 0;
2951         read_lock--;
2952         return false;
2953     }
2954
2955     close(tmpfd);
2956     remove(TAGCACHE_FILE_TEMP);
2957
2958     tc_stat.commit_step = 0;
2959
2960     /* Update the master index headers. */
2961     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2962     {
2963         read_lock--;
2964         return false;
2965     }
2966
2967     tcmh.tch.entry_count += tch.entry_count;
2968     tcmh.tch.datasize = sizeof(struct master_header)
2969         + sizeof(struct index_entry) * tcmh.tch.entry_count
2970         + tch.datasize;
2971     tcmh.dirty = false;
2972     tcmh.commitid++;
2973
2974     lseek(masterfd, 0, SEEK_SET);
2975     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2976     close(masterfd);
2977
2978     logf("tagcache committed");
2979     tc_stat.ready = check_all_headers();
2980     tc_stat.readyvalid = true;
2981
2982     if (local_allocation)
2983     {
2984         tempbuf = NULL;
2985         tempbuf_size = 0;
2986     }
2987
2988 #ifdef HAVE_DIRCACHE
2989     /* Rebuild the dircache, if we stole the buffer. */
2990     if (dircache_buffer_stolen)
2991         dircache_build(0);
2992 #endif
2993
2994 #ifdef HAVE_TC_RAMCACHE
2995     /* Reload tagcache. */
2996     if (tc_stat.ramcache_allocated > 0)
2997         tagcache_start_scan();
2998 #endif
2999
3000     read_lock--;
3001
3002     return true;
3003 }
3004
3005 static void allocate_tempbuf(void)
3006 {
3007     /* Yeah, malloc would be really nice now :) */
3008 #ifdef __PCTOOL__
3009     tempbuf_size = 32*1024*1024;
3010     tempbuf = malloc(tempbuf_size);
3011 #else
3012     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3013     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3014     audiobuf += tempbuf_size;
3015 #endif
3016 }
3017
3018 static void free_tempbuf(void)
3019 {
3020     if (tempbuf_size == 0)
3021         return ;
3022
3023 #ifdef __PCTOOL__
3024     free(tempbuf);
3025 #else
3026     audiobuf -= tempbuf_size;
3027 #endif
3028     tempbuf = NULL;
3029     tempbuf_size = 0;
3030 }
3031
3032 #ifndef __PCTOOL__
3033
3034 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3035 {
3036     struct index_entry idx;
3037
3038     if (!tc_stat.ready)
3039         return false;
3040
3041     if (!TAGCACHE_IS_NUMERIC(tag))
3042         return false;
3043
3044     if (!get_index(masterfd, idx_id, &idx, false))
3045         return false;
3046
3047     idx.tag_seek[tag] = data;
3048     idx.flag |= FLAG_DIRTYNUM;
3049
3050     return write_index(masterfd, idx_id, &idx);
3051 }
3052
3053 #if 0
3054 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3055                                    int tag, long data)
3056 {
3057     struct master_header myhdr;
3058
3059     if (tcs->masterfd < 0)
3060     {
3061         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3062             return false;
3063     }
3064
3065     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3066 }
3067 #endif
3068
3069 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3070
3071 static bool command_queue_is_full(void)
3072 {
3073     int next;
3074
3075     next = command_queue_widx + 1;
3076     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3077         next = 0;
3078
3079     return (next == command_queue_ridx);
3080 }
3081
3082 static void command_queue_sync_callback(void *data)
3083 {
3084     (void)data;
3085     struct master_header myhdr;
3086     int masterfd;
3087
3088     mutex_lock(&command_queue_mutex);
3089
3090     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3091         return;
3092
3093     while (command_queue_ridx != command_queue_widx)
3094     {
3095         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3096
3097         switch (ce->command)
3098         {
3099             case CMD_UPDATE_MASTER_HEADER:
3100             {
3101                 close(masterfd);
3102                 update_master_header();
3103
3104                 /* Re-open the masterfd. */
3105                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3106                     return;
3107
3108                 break;
3109             }
3110             case CMD_UPDATE_NUMERIC:
3111             {
3112                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3113                 break;
3114             }
3115         }
3116
3117         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3118             command_queue_ridx = 0;
3119     }
3120
3121     close(masterfd);
3122
3123     tc_stat.queue_length = 0;
3124     mutex_unlock(&command_queue_mutex);
3125 }
3126
3127 static void run_command_queue(bool force)
3128 {
3129     if (COMMAND_QUEUE_IS_EMPTY)
3130         return;
3131
3132     if (force || command_queue_is_full())
3133         command_queue_sync_callback(NULL);
3134     else
3135         register_storage_idle_func(command_queue_sync_callback);
3136 }
3137
3138 static void queue_command(int cmd, long idx_id, int tag, long data)
3139 {
3140     while (1)
3141     {
3142         int next;
3143
3144         mutex_lock(&command_queue_mutex);
3145         next = command_queue_widx + 1;
3146         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3147             next = 0;
3148
3149         /* Make sure queue is not full. */
3150         if (next != command_queue_ridx)
3151         {
3152             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3153
3154             ce->command = cmd;
3155             ce->idx_id = idx_id;
3156             ce->tag = tag;
3157             ce->data = data;
3158
3159             command_queue_widx = next;
3160
3161             tc_stat.queue_length++;
3162
3163             mutex_unlock(&command_queue_mutex);
3164             break;
3165         }
3166
3167         /* Queue is full, try again later... */
3168         mutex_unlock(&command_queue_mutex);
3169         sleep(1);
3170     }
3171 }
3172
3173 long tagcache_increase_serial(void)
3174 {
3175     long old;
3176
3177     if (!tc_stat.ready)
3178         return -2;
3179
3180     while (read_lock)
3181         sleep(1);
3182
3183     old = current_tcmh.serial++;
3184     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3185
3186     return old;
3187 }
3188
3189 void tagcache_update_numeric(int idx_id, int tag, long data)
3190 {
3191     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3192 }
3193 #endif /* !__PCTOOL__ */
3194
3195 long tagcache_get_serial(void)
3196 {
3197     return current_tcmh.serial;
3198 }
3199
3200 long tagcache_get_commitid(void)
3201 {
3202     return current_tcmh.commitid;
3203 }
3204
3205 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3206 {
3207     char buf[512];
3208     int i;
3209
3210     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3211     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3212     {
3213         if (*datastr == '\0')
3214             break;
3215
3216         if (*datastr == '"' || *datastr == '\\')
3217             buf[i++] = '\\';
3218
3219         buf[i] = *(datastr++);
3220     }
3221
3222     strcpy(&buf[i], "\" ");
3223
3224     write(fd, buf, i + 2);
3225
3226     return true;
3227 }
3228
3229 #ifndef __PCTOOL__
3230
3231 static bool read_tag(char *dest, long size,
3232                      const char *src, const char *tagstr)
3233 {
3234     int pos;
3235     char current_tag[32];
3236
3237     while (*src != '\0')
3238     {
3239         /* Skip all whitespace */
3240         while (*src == ' ')
3241             src++;
3242
3243         if (*src == '\0')
3244             break;
3245
3246         pos = 0;
3247         /* Read in tag name */
3248         while (*src != '=' && *src != ' ')
3249         {
3250             current_tag[pos] = *src;
3251             src++;
3252             pos++;
3253
3254             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3255                 return false;
3256         }
3257         current_tag[pos] = '\0';
3258
3259         /* Read in tag data */
3260
3261         /* Find the start. */
3262         while (*src != '"' && *src != '\0')
3263             src++;
3264
3265         if (*src == '\0' || *(++src) == '\0')
3266             return false;
3267
3268         /* Read the data. */
3269         for (pos = 0; pos < size; pos++)
3270         {
3271             if (*src == '\0')
3272                 break;
3273
3274             if (*src == '\\')
3275             {
3276                 dest[pos] = *(src+1);
3277                 src += 2;
3278                 continue;
3279             }
3280
3281             dest[pos] = *src;
3282
3283             if (*src == '"')
3284             {
3285                 src++;
3286                 break;
3287             }
3288
3289             if (*src == '\0')
3290                 break;
3291
3292             src++;
3293         }
3294         dest[pos] = '\0';
3295
3296         if (!strcasecmp(tagstr, current_tag))
3297             return true;
3298     }
3299
3300     return false;
3301 }
3302
3303 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3304 {
3305     struct index_entry idx;
3306     char tag_data[TAG_MAXLEN+32];
3307     int idx_id;
3308     long masterfd = (long)parameters;
3309     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3310         tag_commitid };
3311     int i;
3312     (void)line_n;
3313
3314     if (*buf == '#')
3315         return 0;
3316
3317     logf("%d/%s", line_n, buf);
3318     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3319     {
3320         logf("filename missing");
3321         logf("-> %s", buf);
3322         return 0;
3323     }
3324
3325     idx_id = find_index(tag_data);
3326     if (idx_id < 0)
3327     {
3328         logf("entry not found");
3329         return 0;
3330     }
3331
3332     if (!get_index(masterfd, idx_id, &idx, false))
3333     {
3334         logf("failed to retrieve index entry");
3335         return 0;
3336     }
3337
3338     /* Stop if tag has already been modified. */
3339     if (idx.flag & FLAG_DIRTYNUM)
3340         return 0;
3341
3342     logf("import: %s", tag_data);
3343
3344     idx.flag |= FLAG_DIRTYNUM;
3345     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3346     {
3347         int data;
3348
3349         if (!read_tag(tag_data, sizeof tag_data, buf,
3350                       tagcache_tag_to_str(import_tags[i])))
3351         {
3352             continue;
3353         }
3354
3355         data = atoi(tag_data);
3356         if (data < 0)
3357             continue;
3358
3359         idx.tag_seek[import_tags[i]] = data;
3360
3361         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3362             current_tcmh.serial = data + 1;
3363         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3364             current_tcmh.commitid = data + 1;
3365     }
3366
3367     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3368 }
3369
3370 bool tagcache_import_changelog(void)
3371 {
3372     struct master_header myhdr;
3373     struct tagcache_header tch;
3374     int clfd;
3375     long masterfd;
3376     char buf[2048];
3377
3378     if (!tc_stat.ready)
3379         return false;
3380
3381     while (read_lock)
3382         sleep(1);
3383
3384     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3385     if (clfd < 0)
3386     {
3387         logf("failure to open changelog");
3388         return false;
3389     }
3390
3391     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3392     {
3393         close(clfd);
3394         return false;
3395     }
3396
3397     write_lock++;
3398
3399     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3400
3401     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3402                   parse_changelog_line);
3403
3404     close(clfd);
3405     close(masterfd);
3406
3407     if (filenametag_fd >= 0)
3408     {
3409         close(filenametag_fd);
3410         filenametag_fd = -1;
3411     }
3412
3413     write_lock--;
3414
3415     update_master_header();
3416
3417     return true;
3418 }
3419
3420 #endif /* !__PCTOOL__ */
3421
3422 bool tagcache_create_changelog(struct tagcache_search *tcs)
3423 {
3424     struct master_header myhdr;
3425     struct index_entry idx;
3426     char buf[TAG_MAXLEN+32];
3427     char temp[32];
3428     int clfd;
3429     int i, j;
3430
3431     if (!tc_stat.ready)
3432         return false;
3433
3434     if (!tagcache_search(tcs, tag_filename))
3435         return false;
3436
3437     /* Initialize the changelog */
3438     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3439     if (clfd < 0)
3440     {
3441         logf("failure to open changelog");
3442         return false;
3443     }
3444
3445     if (tcs->masterfd < 0)
3446     {
3447         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3448             return false;
3449     }
3450     else
3451     {
3452         lseek(tcs->masterfd, 0, SEEK_SET);
3453         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3454     }
3455
3456     write(clfd, "## Changelog version 1\n", 23);
3457
3458     for (i = 0; i < myhdr.tch.entry_count; i++)
3459     {
3460         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3461             != sizeof(struct index_entry))
3462         {
3463             logf("read error #9");
3464             tagcache_search_finish(tcs);
3465             close(clfd);
3466             return false;
3467         }
3468
3469         /* Skip until the entry found has been modified. */
3470         if (! (idx.flag & FLAG_DIRTYNUM) )
3471             continue;
3472
3473         /* Skip deleted entries too. */
3474         if (idx.flag & FLAG_DELETED)
3475             continue;
3476
3477         /* Now retrieve all tags. */
3478         for (j = 0; j < TAG_COUNT; j++)
3479         {
3480             if (TAGCACHE_IS_NUMERIC(j))
3481             {
3482                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3483                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3484                 continue;
3485             }
3486
3487             tcs->type = j;
3488             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3489             write_tag(clfd, tagcache_tag_to_str(j), buf);
3490         }
3491
3492         write(clfd, "\n", 1);
3493         do_timed_yield();
3494     }
3495
3496     close(clfd);
3497
3498     tagcache_search_finish(tcs);
3499
3500     return true;
3501 }
3502
3503 static bool delete_entry(long idx_id)
3504 {
3505     int fd = -1;
3506     int masterfd = -1;
3507     int tag, i;
3508     struct index_entry idx, myidx;
3509     struct master_header myhdr;
3510     char buf[TAG_MAXLEN+32];
3511     int in_use[TAG_COUNT];
3512
3513     logf("delete_entry(): %ld", idx_id);
3514
3515 #ifdef HAVE_TC_RAMCACHE
3516     /* At first mark the entry removed from ram cache. */
3517     if (tc_stat.ramcache)
3518         hdr->indices[idx_id].flag |= FLAG_DELETED;
3519 #endif
3520
3521     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3522         return false;
3523
3524     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3525     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3526         != sizeof(struct index_entry))
3527     {
3528         logf("delete_entry(): read error");
3529         goto cleanup;
3530     }
3531
3532     if (myidx.flag & FLAG_DELETED)
3533     {
3534         logf("delete_entry(): already deleted!");
3535         goto cleanup;
3536     }
3537
3538     myidx.flag |= FLAG_DELETED;
3539     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3540     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3541         != sizeof(struct index_entry))
3542     {
3543         logf("delete_entry(): write_error #1");
3544         goto cleanup;
3545     }
3546
3547     /* Now check which tags are no longer in use (if any) */
3548     for (tag = 0; tag < TAG_COUNT; tag++)
3549         in_use[tag] = 0;
3550
3551     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3552     for (i = 0; i < myhdr.tch.entry_count; i++)
3553     {
3554         struct index_entry *idxp;
3555
3556 #ifdef HAVE_TC_RAMCACHE
3557         /* Use RAM DB if available for greater speed */
3558         if (tc_stat.ramcache)
3559             idxp = &hdr->indices[i];
3560         else
3561 #endif
3562         {
3563             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3564                 != sizeof(struct index_entry))
3565             {
3566                 logf("delete_entry(): read error #2");
3567                 goto cleanup;
3568             }
3569             idxp = &idx;
3570         }
3571
3572         if (idxp->flag & FLAG_DELETED)
3573             continue;
3574
3575         for (tag = 0; tag < TAG_COUNT; tag++)
3576         {
3577             if (TAGCACHE_IS_NUMERIC(tag))
3578                 continue;
3579
3580             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3581                 in_use[tag]++;
3582         }
3583     }
3584
3585     /* Now delete all tags no longer in use. */
3586     for (tag = 0; tag < TAG_COUNT; tag++)
3587     {
3588         struct tagcache_header tch;
3589         int oldseek = myidx.tag_seek[tag];
3590
3591         if (TAGCACHE_IS_NUMERIC(tag))
3592             continue;
3593
3594         /**
3595          * Replace tag seek with a hash value of the field string data.
3596          * That way runtime statistics of moved or altered files can be
3597          * resurrected.
3598          */
3599 #ifdef HAVE_TC_RAMCACHE
3600         if (tc_stat.ramcache && tag != tag_filename)
3601         {
3602             struct tagfile_entry *tfe;
3603             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3604
3605             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3606             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3607             myidx.tag_seek[tag] = *seek;
3608         }
3609         else
3610 #endif
3611         {
3612             struct tagfile_entry tfe;
3613
3614             /* Open the index file, which contains the tag names. */
3615             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3616                 goto cleanup;
3617
3618             /* Skip the header block */
3619             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3620             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3621                 != sizeof(struct tagfile_entry))
3622             {
3623                 logf("delete_entry(): read error #3");
3624                 goto cleanup;
3625             }
3626
3627             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3628             {
3629                 logf("delete_entry(): read error #4");
3630                 goto cleanup;
3631             }
3632
3633             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3634         }
3635
3636         if (in_use[tag])
3637         {
3638             logf("in use: %d/%d", tag, in_use[tag]);
3639             if (fd >= 0)
3640             {
3641                 close(fd);
3642                 fd = -1;
3643             }
3644             continue;
3645         }
3646
3647 #ifdef HAVE_TC_RAMCACHE
3648         /* Delete from ram. */
3649         if (tc_stat.ramcache && tag != tag_filename)
3650         {
3651             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3652             tagentry->tag_data[0] = '\0';
3653         }
3654 #endif
3655
3656         /* Open the index file, which contains the tag names. */
3657         if (fd < 0)
3658         {
3659             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3660                 goto cleanup;
3661         }
3662
3663         /* Skip the header block */
3664         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3665
3666         /* Debug, print 10 first characters of the tag
3667         read(fd, buf, 10);
3668         buf[10]='\0';
3669         logf("TAG:%s", buf);
3670         lseek(fd, -10, SEEK_CUR);
3671         */
3672
3673         /* Write first data byte in tag as \0 */
3674         write(fd, "", 1);
3675
3676         /* Now tag data has been removed */
3677         close(fd);
3678         fd = -1;
3679     }
3680
3681     /* Write index entry back into master index. */
3682     lseek(masterfd, sizeof(struct master_header) +
3683           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3684     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3685         != sizeof(struct index_entry))
3686     {
3687         logf("delete_entry(): write_error #2");
3688         goto cleanup;
3689     }
3690
3691     close(masterfd);
3692
3693     return true;
3694
3695     cleanup:
3696     if (fd >= 0)
3697         close(fd);
3698     if (masterfd >= 0)
3699         close(masterfd);
3700
3701     return false;
3702 }
3703
3704 #ifndef __PCTOOL__
3705 /**
3706  * Returns true if there is an event waiting in the queue
3707  * that requires the current operation to be aborted.
3708  */
3709 static bool check_event_queue(void)
3710 {
3711     struct queue_event ev;
3712
3713     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3714     switch (ev.id)
3715     {
3716         case Q_STOP_SCAN:
3717         case SYS_POWEROFF:
3718         case SYS_USB_CONNECTED:
3719             /* Put the event back into the queue. */
3720             queue_post(&tagcache_queue, ev.id, ev.data);
3721             return true;
3722     }
3723
3724     return false;
3725 }
3726 #endif
3727
3728 #ifdef HAVE_TC_RAMCACHE
3729 static bool allocate_tagcache(void)
3730 {
3731     struct master_header tcmh;
3732     int fd;
3733
3734     /* Load the header. */
3735     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3736     {
3737         hdr = NULL;
3738         return false;
3739     }
3740
3741     close(fd);
3742
3743     /**
3744      * Now calculate the required cache size plus
3745      * some extra space for alignment fixes.
3746      */
3747     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3748         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3749     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3750     memset(hdr, 0, sizeof(struct ramcache_header));
3751     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3752     hdr->indices = (struct index_entry *)(hdr + 1);
3753     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3754
3755     return true;
3756 }
3757
3758 # ifdef HAVE_EEPROM_SETTINGS
3759 static bool tagcache_dumpload(void)
3760 {
3761     struct statefile_header shdr;
3762     int fd, rc;
3763     long offpos;
3764     int i;
3765
3766     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3767     if (fd < 0)
3768     {
3769         logf("no tagcache statedump");
3770         return false;
3771     }
3772
3773     /* Check the statefile memory placement */
3774     hdr = buffer_alloc(0);
3775     rc = read(fd, &shdr, sizeof(struct statefile_header));
3776     if (rc != sizeof(struct statefile_header)
3777         /* || (long)hdr != (long)shdr.hdr */)
3778     {
3779         logf("incorrect statefile");
3780         hdr = NULL;
3781         close(fd);
3782         return false;
3783     }
3784
3785     offpos = (long)hdr - (long)shdr.hdr;
3786
3787     /* Lets allocate real memory and load it */
3788     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3789     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3790     close(fd);
3791
3792     if (rc != shdr.tc_stat.ramcache_allocated)
3793     {
3794         logf("read failure!");
3795         hdr = NULL;
3796         return false;
3797     }
3798
3799     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3800
3801     /* Now fix the pointers */
3802     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3803     for (i = 0; i < TAG_COUNT; i++)
3804         hdr->tags[i] += offpos;
3805
3806     return true;
3807 }
3808
3809 static bool tagcache_dumpsave(void)
3810 {
3811     struct statefile_header shdr;
3812     int fd;
3813
3814     if (!tc_stat.ramcache)
3815         return false;
3816
3817     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3818     if (fd < 0)
3819     {
3820         logf("failed to create a statedump");
3821         return false;
3822     }
3823
3824     /* Create the header */
3825     shdr.hdr = hdr;
3826     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3827     write(fd, &shdr, sizeof(struct statefile_header));
3828
3829     /* And dump the data too */
3830     write(fd, hdr, tc_stat.ramcache_allocated);
3831     close(fd);
3832
3833     return true;
3834 }
3835 # endif
3836
3837 static bool load_tagcache(void)
3838 {
3839     struct tagcache_header *tch;
3840     long bytesleft = tc_stat.ramcache_allocated;
3841     struct index_entry *idx;
3842     int rc, fd;
3843     char *p;
3844     int i, tag;
3845
3846 # ifdef HAVE_DIRCACHE
3847     while (dircache_is_initializing())
3848         sleep(1);
3849
3850     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3851 # endif
3852
3853     logf("loading tagcache to ram...");
3854
3855     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3856     if (fd < 0)
3857     {
3858         logf("tagcache open failed");
3859         return false;
3860     }
3861
3862     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3863         != sizeof(struct master_header)
3864         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3865     {
3866         logf("incorrect header");
3867         return false;
3868     }
3869
3870     idx = hdr->indices;
3871
3872     /* Load the master index table. */
3873     for (i = 0; i < hdr->h.tch.entry_count; i++)
3874     {
3875         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3876         if (rc != sizeof(struct index_entry))
3877         {
3878             logf("read error #10");
3879             close(fd);
3880             return false;
3881         }
3882
3883         bytesleft -= sizeof(struct index_entry);
3884         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3885         {
3886             logf("too big tagcache.");
3887             close(fd);
3888             return false;
3889         }
3890
3891         idx++;
3892     }
3893
3894     close(fd);
3895
3896     /* Load the tags. */
3897     p = (char *)idx;
3898     for (tag = 0; tag < TAG_COUNT; tag++)
3899     {
3900         struct tagfile_entry *fe;
3901         char buf[TAG_MAXLEN+32];
3902
3903         if (TAGCACHE_IS_NUMERIC(tag))
3904             continue ;
3905
3906         //p = ((void *)p+1);
3907         p = (char *)((long)p & ~0x03) + 0x04;
3908         hdr->tags[tag] = p;
3909
3910         /* Check the header. */
3911         tch = (struct tagcache_header *)p;
3912         p += sizeof(struct tagcache_header);
3913
3914         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3915             return false;
3916
3917         for (hdr->entry_count[tag] = 0;
3918              hdr->entry_count[tag] < tch->entry_count;
3919              hdr->entry_count[tag]++)
3920         {
3921             long pos;
3922
3923             if (do_timed_yield())
3924             {
3925                 /* Abort if we got a critical event in queue */
3926                 if (check_event_queue())
3927                     return false;
3928             }
3929
3930             fe = (struct tagfile_entry *)p;
3931             pos = lseek(fd, 0, SEEK_CUR);
3932             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3933             if (rc != sizeof(struct tagfile_entry))
3934             {
3935                 /* End of lookup table. */
3936                 logf("read error #11");
3937                 close(fd);
3938                 return false;
3939             }
3940
3941             /* We have a special handling for the filename tags. */
3942             if (tag == tag_filename)
3943             {
3944 # ifdef HAVE_DIRCACHE
3945                 const struct dircache_entry *dc;
3946 # endif
3947
3948                 // FIXME: This is wrong!
3949                 // idx = &hdr->indices[hdr->entry_count[i]];
3950                 idx = &hdr->indices[fe->idx_id];
3951
3952                 if (fe->tag_length >= (long)sizeof(buf)-1)
3953                 {
3954                     read(fd, buf, 10);
3955                     buf[10] = '\0';
3956                     logf("TAG:%s", buf);
3957                     logf("too long filename");
3958                     close(fd);
3959                     return false;
3960                 }
3961
3962                 rc = read(fd, buf, fe->tag_length);
3963                 if (rc != fe->tag_length)
3964                 {
3965                     logf("read error #12");
3966                     close(fd);
3967                     return false;
3968                 }
3969
3970                 /* Check if the entry has already been removed */
3971                 if (idx->flag & FLAG_DELETED)
3972                     continue;
3973
3974                 /* This flag must not be used yet. */
3975                 if (idx->flag & FLAG_DIRCACHE)
3976                 {
3977                     logf("internal error!");
3978                     close(fd);
3979                     return false;
3980                 }
3981
3982                 if (idx->tag_seek[tag] != pos)
3983                 {
3984                     logf("corrupt data structures!");
3985                     close(fd);
3986                     return false;
3987                 }
3988
3989 # ifdef HAVE_DIRCACHE
3990                 if (dircache_is_enabled())
3991                 {
3992                     dc = dircache_get_entry_ptr(buf);
3993                     if (dc == NULL)
3994                     {
3995                         logf("Entry no longer valid.");
3996                         logf("-> %s", buf);
3997                         if (global_settings.tagcache_autoupdate)
3998                             delete_entry(fe->idx_id);
3999                         continue ;
4000                     }
4001
4002                     idx->flag |= FLAG_DIRCACHE;
4003                     idx->tag_seek[tag_filename] = (long)dc;
4004                 }
4005                 else
4006 # endif
4007                 {
4008                     /* This will be very slow unless dircache is enabled
4009                        or target is flash based, but do it anyway for
4010                        consistency. */
4011                     /* Check if entry has been removed. */
4012                     if (global_settings.tagcache_autoupdate)
4013                     {
4014                         if (!file_exists(buf))
4015                         {
4016                             logf("Entry no longer valid.");
4017                             logf("-> %s", buf);
4018                             delete_entry(fe->idx_id);
4019                             continue;
4020                         }
4021                     }
4022                 }
4023
4024                 continue ;
4025             }
4026
4027             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4028             if (bytesleft < 0)
4029             {
4030                 logf("too big tagcache #2");
4031                 logf("tl: %d", fe->tag_length);
4032                 logf("bl: %ld", bytesleft);
4033                 close(fd);
4034                 return false;
4035             }
4036
4037             p = fe->tag_data;
4038             rc = read(fd, fe->tag_data, fe->tag_length);
4039             p += rc;
4040
4041             if (rc != fe->tag_length)
4042             {
4043                 logf("read error #13");
4044                 logf("rc=0x%04x", rc); // 0x431
4045                 logf("len=0x%04x", fe->tag_length); // 0x4000
4046                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4047                 logf("tag=0x%02x", tag); // 0x00
4048                 close(fd);
4049                 return false;
4050             }
4051         }
4052         close(fd);
4053     }
4054
4055     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4056     logf("tagcache loaded into ram!");
4057
4058     return true;
4059 }
4060 #endif /* HAVE_TC_RAMCACHE */
4061
4062 static bool check_deleted_files(void)
4063 {
4064     int fd;
4065     char buf[TAG_MAXLEN+32];
4066     struct tagfile_entry tfe;
4067
4068     logf("reverse scan...");
4069     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4070     fd = open(buf, O_RDONLY);
4071
4072     if (fd < 0)
4073     {
4074         logf("%s open fail", buf);
4075         return false;
4076     }
4077
4078     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4079     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4080            == sizeof(struct tagfile_entry)
4081 #ifndef __PCTOOL__
4082            && !check_event_queue()
4083 #endif
4084            )
4085     {
4086         if (tfe.tag_length >= (long)sizeof(buf)-1)
4087         {
4088             logf("too long tag");
4089             close(fd);
4090             return false;
4091         }
4092
4093         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4094         {
4095             logf("read error #14");
4096             close(fd);
4097             return false;
4098         }
4099
4100         /* Check if the file has already deleted from the db. */
4101         if (*buf == '\0')
4102             continue;
4103
4104         /* Now check if the file exists. */
4105         if (!file_exists(buf))
4106         {
4107             logf("Entry no longer valid.");
4108             logf("-> %s / %d", buf, tfe.tag_length);
4109             delete_entry(tfe.idx_id);
4110         }
4111     }
4112
4113     close(fd);
4114
4115     logf("done");
4116
4117     return true;
4118 }
4119
4120
4121 /* Note that this function must not be inlined, otherwise the whole point
4122  * of having the code in a separate function is lost.
4123  */
4124 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4125     int *ignore, int *unignore)
4126 {
4127     char newpath[MAX_PATH];
4128
4129     /* check for a database.ignore file */
4130     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4131     *ignore = file_exists(newpath);
4132     /* check for a database.unignore file */
4133     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4134     *unignore = file_exists(newpath);
4135 }
4136
4137
4138 static bool check_dir(const char *dirname, int add_files)
4139 {
4140     DIR *dir;
4141     int len;
4142     int success = false;
4143     int ignore, unignore;
4144
4145     dir = opendir(dirname);
4146     if (!dir)
4147     {
4148         logf("tagcache: opendir() failed");
4149         return false;
4150     }
4151
4152     /* check for a database.ignore and database.unignore */
4153     check_ignore(dirname, &ignore, &unignore);
4154
4155     /* don't do anything if both ignore and unignore are there */
4156     if (ignore != unignore)
4157         add_files = unignore;
4158
4159     /* Recursively scan the dir. */
4160 #ifdef __PCTOOL__
4161     while (1)
4162 #else
4163     while (!check_event_queue())
4164 #endif
4165     {
4166         struct dirent *entry;
4167
4168         entry = readdir(dir);
4169
4170         if (entry == NULL)
4171         {
4172             success = true;
4173             break ;
4174         }
4175
4176         if (!strcmp((char *)entry->d_name, ".") ||
4177             !strcmp((char *)entry->d_name, ".."))
4178             continue;
4179
4180         yield();
4181
4182         len = strlen(curpath);
4183         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4184                  entry->d_name);
4185
4186         processed_dir_count++;
4187         if (entry->attribute & ATTR_DIRECTORY)
4188             check_dir(curpath, add_files);
4189         else if (add_files)
4190         {
4191             tc_stat.curentry = curpath;
4192
4193             /* Add a new entry to the temporary db file. */
4194             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4195 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4196                          , dir->internal_entry
4197 #endif
4198                          );
4199
4200             /* Wait until current path for debug screen is read and unset. */
4201             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4202                 yield();
4203
4204             tc_stat.curentry = NULL;
4205         }
4206
4207         curpath[len] = '\0';
4208     }
4209
4210     closedir(dir);
4211
4212     return success;
4213 }
4214
4215 void tagcache_screensync_event(void)
4216 {
4217     tc_stat.curentry = NULL;
4218 }
4219
4220 void tagcache_screensync_enable(bool state)
4221 {
4222     tc_stat.syncscreen = state;
4223 }
4224
4225 void tagcache_build(const char *path)
4226 {
4227     struct tagcache_header header;
4228     bool ret;
4229
4230     curpath[0] = '\0';
4231     data_size = 0;
4232     total_entry_count = 0;
4233     processed_dir_count = 0;
4234
4235 #ifdef HAVE_DIRCACHE
4236     while (dircache_is_initializing())
4237         sleep(1);
4238 #endif
4239
4240     logf("updating tagcache");
4241
4242     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4243     if (cachefd >= 0)
4244     {
4245         logf("skipping, cache already waiting for commit");
4246         close(cachefd);
4247         return ;
4248     }
4249
4250     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4251     if (cachefd < 0)
4252     {
4253         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4254         return ;
4255     }
4256
4257     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4258
4259     cpu_boost(true);
4260
4261     logf("Scanning files...");
4262     /* Scan for new files. */
4263     memset(&header, 0, sizeof(struct tagcache_header));
4264     write(cachefd, &header, sizeof(struct tagcache_header));
4265
4266     if (strcmp("/", path) != 0)
4267         strcpy(curpath, path);
4268     ret = check_dir(path, true);
4269
4270     /* Write the header. */
4271     header.magic = TAGCACHE_MAGIC;
4272     header.datasize = data_size;
4273     header.entry_count = total_entry_count;
4274     lseek(cachefd, 0, SEEK_SET);
4275     write(cachefd, &header, sizeof(struct tagcache_header));
4276     close(cachefd);
4277
4278     if (filenametag_fd >= 0)
4279     {
4280         close(filenametag_fd);
4281         filenametag_fd = -1;
4282     }
4283
4284     if (!ret)
4285     {
4286         logf("Aborted.");
4287         cpu_boost(false);
4288         return ;
4289     }
4290
4291     /* Commit changes to the database. */
4292 #ifdef __PCTOOL__
4293     allocate_tempbuf();
4294 #endif
4295     if (commit())
4296     {
4297         remove(TAGCACHE_FILE_TEMP);
4298         logf("tagcache built!");
4299     }
4300 #ifdef __PCTOOL__
4301     free_tempbuf();
4302 #endif
4303
4304 #ifdef HAVE_TC_RAMCACHE
4305     if (hdr)
4306     {
4307         /* Import runtime statistics if we just initialized the db. */
4308         if (hdr->h.serial == 0)
4309             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4310     }
4311 #endif
4312
4313     cpu_boost(false);
4314 }
4315
4316 #ifdef HAVE_TC_RAMCACHE
4317 static void load_ramcache(void)
4318 {
4319     if (!hdr)
4320         return ;
4321
4322     cpu_boost(true);
4323
4324     /* At first we should load the cache (if exists). */
4325     tc_stat.ramcache = load_tagcache();
4326
4327     if (!tc_stat.ramcache)
4328     {
4329         /* If loading failed, it must indicate some problem with the db
4330          * so disable it entirely to prevent further issues. */
4331         tc_stat.ready = false;
4332         hdr = NULL;
4333     }
4334
4335     cpu_boost(false);
4336 }
4337
4338 void tagcache_unload_ramcache(void)
4339 {
4340     tc_stat.ramcache = false;
4341     /* Just to make sure there is no statefile present. */
4342     // remove(TAGCACHE_STATEFILE);
4343 }
4344 #endif
4345
4346 #ifndef __PCTOOL__
4347 static void tagcache_thread(void)
4348 {
4349     struct queue_event ev;
4350     bool check_done = false;
4351
4352     /* If the previous cache build/update was interrupted, commit
4353      * the changes first in foreground. */
4354     cpu_boost(true);
4355     allocate_tempbuf();
4356     commit();
4357     free_tempbuf();
4358
4359 #ifdef HAVE_TC_RAMCACHE
4360 # ifdef HAVE_EEPROM_SETTINGS
4361     if (firmware_settings.initialized && firmware_settings.disk_clean)
4362         check_done = tagcache_dumpload();
4363
4364     remove(TAGCACHE_STATEFILE);
4365 # endif
4366
4367     /* Allocate space for the tagcache if found on disk. */
4368     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4369         allocate_tagcache();
4370 #endif
4371
4372     cpu_boost(false);
4373     tc_stat.initialized = true;
4374
4375     /* Don't delay bootup with the header check but do it on background. */
4376     sleep(HZ);
4377     tc_stat.ready = check_all_headers();
4378     tc_stat.readyvalid = true;
4379
4380     while (1)
4381     {
4382         run_command_queue(false);
4383
4384         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4385
4386         switch (ev.id)
4387         {
4388             case Q_IMPORT_CHANGELOG:
4389                 tagcache_import_changelog();
4390                 break;
4391
4392             case Q_REBUILD:
4393                 remove_files();
4394                 remove(TAGCACHE_FILE_TEMP);
4395                 tagcache_build("/");
4396                 break;
4397
4398             case Q_UPDATE:
4399                 tagcache_build("/");
4400 #ifdef HAVE_TC_RAMCACHE
4401                 load_ramcache();
4402 #endif
4403                 check_deleted_files();
4404                 break ;
4405
4406             case Q_START_SCAN:
4407                 check_done = false;
4408             case SYS_TIMEOUT:
4409                 if (check_done || !tc_stat.ready)
4410                     break ;
4411
4412 #ifdef HAVE_TC_RAMCACHE
4413                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4414                 {
4415                     load_ramcache();
4416                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4417                         tagcache_build("/");
4418                 }
4419                 else
4420 #endif
4421                 if (global_settings.tagcache_autoupdate)
4422                 {
4423                     tagcache_build("/");
4424
4425                     /* This will be very slow unless dircache is enabled
4426                        or target is flash based, but do it anyway for
4427                        consistency. */
4428                     check_deleted_files();
4429                 }
4430
4431                 logf("tagcache check done");
4432
4433                 check_done = true;
4434                 break ;
4435
4436             case Q_STOP_SCAN:
4437                 break ;
4438
4439             case SYS_POWEROFF:
4440                 break ;
4441
4442 #ifndef SIMULATOR
4443             case SYS_USB_CONNECTED:
4444                 logf("USB: TagCache");
4445                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4446                 usb_wait_for_disconnect(&tagcache_queue);
4447                 break ;
4448 #endif
4449         }
4450     }
4451 }
4452
4453 bool tagcache_prepare_shutdown(void)
4454 {
4455     if (tagcache_get_commit_step() > 0)
4456         return false;
4457
4458     tagcache_stop_scan();
4459     while (read_lock || write_lock)
4460         sleep(1);
4461
4462     return true;
4463 }
4464
4465 void tagcache_shutdown(void)
4466 {
4467     /* Flush the command queue. */
4468     run_command_queue(true);
4469
4470 #ifdef HAVE_EEPROM_SETTINGS
4471     if (tc_stat.ramcache)
4472         tagcache_dumpsave();
4473 #endif
4474 }
4475
4476 static int get_progress(void)
4477 {
4478     int total_count = -1;
4479
4480 #ifdef HAVE_DIRCACHE
4481     if (dircache_is_enabled())
4482     {
4483         total_count = dircache_get_entry_count();
4484     }
4485     else
4486 #endif
4487 #ifdef HAVE_TC_RAMCACHE
4488     {
4489         if (hdr && tc_stat.ramcache)
4490             total_count = hdr->h.tch.entry_count;
4491     }
4492 #endif
4493
4494     if (total_count < 0)
4495         return -1;
4496
4497     return processed_dir_count * 100 / total_count;
4498 }
4499
4500 struct tagcache_stat* tagcache_get_stat(void)
4501 {
4502     tc_stat.progress = get_progress();
4503     tc_stat.processed_entries = processed_dir_count;
4504
4505     return &tc_stat;
4506 }
4507
4508 void tagcache_start_scan(void)
4509 {
4510     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4511 }
4512
4513 bool tagcache_update(void)
4514 {
4515     if (!tc_stat.ready)
4516         return false;
4517
4518     queue_post(&tagcache_queue, Q_UPDATE, 0);
4519     return false;
4520 }
4521
4522 bool tagcache_rebuild()
4523 {
4524     queue_post(&tagcache_queue, Q_REBUILD, 0);
4525     return false;
4526 }
4527
4528 void tagcache_stop_scan(void)
4529 {
4530     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4531 }
4532
4533 #ifdef HAVE_TC_RAMCACHE
4534 bool tagcache_is_ramcache(void)
4535 {
4536     return tc_stat.ramcache;
4537 }
4538 #endif
4539
4540 #endif /* !__PCTOOL__ */
4541
4542
4543 void tagcache_init(void)
4544 {
4545     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4546     memset(&current_tcmh, 0, sizeof(struct master_header));
4547     filenametag_fd = -1;
4548     write_lock = read_lock = 0;
4549
4550 #ifndef __PCTOOL__
4551     mutex_init(&command_queue_mutex);
4552     queue_init(&tagcache_queue, true);
4553     create_thread(tagcache_thread, tagcache_stack,
4554                   sizeof(tagcache_stack), 0, tagcache_thread_name
4555                   IF_PRIO(, PRIORITY_BACKGROUND)
4556                   IF_COP(, CPU));
4557 #else
4558     tc_stat.initialized = true;
4559     allocate_tempbuf();
4560     commit();
4561     free_tempbuf();
4562     tc_stat.ready = check_all_headers();
4563 #endif
4564 }
4565
4566 #ifdef __PCTOOL__
4567 void tagcache_reverse_scan(void)
4568 {
4569     logf("Checking for deleted files");
4570     check_deleted_files();
4571 }
4572 #endif
4573
4574 bool tagcache_is_initialized(void)
4575 {
4576     return tc_stat.initialized;
4577 }
4578 bool tagcache_is_usable(void)
4579 {
4580     return tc_stat.initialized && tc_stat.ready;
4581 }
4582 int tagcache_get_commit_step(void)
4583 {
4584     return tc_stat.commit_step;
4585 }
4586 int tagcache_get_max_commit_step(void)
4587 {
4588     return (int)(SORTED_TAGS_COUNT)+1;
4589 }
4590