apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /*#define LOGF_ENABLE*/
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "tagcache.h"
  73 #include "buffer.h"
  74 #include "crc32.h"
  75 #include "misc.h"
  76 #include "settings.h"
  77 #include "dir.h"
  78 #include "structec.h"
  79
  80 #ifndef __PCTOOL__
  81 #include "lang.h"
  82 #include "eeprom_settings.h"
  83 #endif
  84
  85 #ifdef __PCTOOL__
  86 #define yield() do { } while(0)
  87 #define sim_sleep(timeout) do { } while(0)
  88 #define do_timed_yield() do { } while(0)
  89 #endif
  90
  91 #ifndef __PCTOOL__
  92 /* Tag Cache thread. */
  93 static struct event_queue tagcache_queue;
  94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  95 static const char tagcache_thread_name[] = "tagcache";
  96 #endif
  97
  98 #define UNTAGGED "<Untagged>"
  99
 100 /* Previous path when scanning directory tree recursively. */
 101 static char curpath[TAG_MAXLEN+32];
 102
 103 /* Used when removing duplicates. */
 104 static char *tempbuf;     /* Allocated when needed. */
 105 static long tempbufidx;   /* Current location in buffer. */
 106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 107 static long tempbuf_left; /* Buffer space left. */
 108 static long tempbuf_pos;
 109
 110 #define SORTED_TAGS_COUNT 8
 111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
 112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
 113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
 114     (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
 115 /* Tags we want to get sorted (loaded to the tempbuf). */
 116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 117     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 118     (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
 119
 120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
 122     (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
 123     (1LU << tag_albumartist) | (1LU << tag_grouping))
 124
 125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 126 static const char *tags_str[] = { "artist", "album", "genre", "title",
 127     "filename", "composer", "comment", "albumartist", "grouping", "year",
 128     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 129     "playtime", "lastplayed", "commitid", "mtime" };
 130
 131 /* Status information of the tagcache. */
 132 static struct tagcache_stat tc_stat;
 133
 134 /* Queue commands. */
 135 enum tagcache_queue {
 136     Q_STOP_SCAN = 0,
 137     Q_START_SCAN,
 138     Q_IMPORT_CHANGELOG,
 139     Q_UPDATE,
 140     Q_REBUILD,
 141
 142     /* Internal tagcache command queue. */
 143     CMD_UPDATE_MASTER_HEADER,
 144     CMD_UPDATE_NUMERIC,
 145 };
 146
 147 struct tagcache_command_entry {
 148     int32_t command;
 149     int32_t idx_id;
 150     int32_t tag;
 151     int32_t data;
 152 };
 153
 154 #ifndef __PCTOOL__
 155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 156 static volatile int command_queue_widx = 0;
 157 static volatile int command_queue_ridx = 0;
 158 static struct mutex command_queue_mutex;
 159 #endif
 160
 161 /* Tag database structures. */
 162
 163 /* Variable-length tag entry in tag files. */
 164 struct tagfile_entry {
 165     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 166     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 167     char tag_data[0];  /* Begin of the tag data */
 168 };
 169
 170 /* Fixed-size tag entry in master db index. */
 171 struct index_entry {
 172     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 173     int32_t flag;                /* Status flags */
 174 };
 175
 176 /* Header is the same in every file. */
 177 struct tagcache_header {
 178     int32_t magic;       /* Header version number */
 179     int32_t datasize;    /* Data size in bytes */
 180     int32_t entry_count; /* Number of entries in this file */
 181 };
 182
 183 struct master_header {
 184     struct tagcache_header tch;
 185     int32_t serial; /* Increasing counting number */
 186     int32_t commitid; /* Number of commits so far */
 187     int32_t dirty;
 188 };
 189
 190 /* For the endianess correction */
 191 static const char *tagfile_entry_ec   = "ll";
 192 /**
 193  Note: This should be (1 + TAG_COUNT) amount of l's.
 194  */
 195 static const char *index_entry_ec     = "lllllllllllllllllllll";
 196
 197 static const char *tagcache_header_ec = "lll";
 198 static const char *master_header_ec   = "llllll";
 199
 200 static struct master_header current_tcmh;
 201
 202 #ifdef HAVE_TC_RAMCACHE
 203 /* Header is created when loading database to ram. */
 204 struct ramcache_header {
 205     struct master_header h;      /* Header from the master index */
 206     struct index_entry *indices; /* Master index file content */
 207     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 208     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 209 };
 210
 211 # ifdef HAVE_EEPROM_SETTINGS
 212 struct statefile_header {
 213     struct ramcache_header *hdr;
 214     struct tagcache_stat tc_stat;
 215 };
 216 # endif
 217
 218 /* Pointer to allocated ramcache_header */
 219 static struct ramcache_header *hdr;
 220 #endif
 221
 222 /**
 223  * Full tag entries stored in a temporary file waiting
 224  * for commit to the cache. */
 225 struct temp_file_entry {
 226     long tag_offset[TAG_COUNT];
 227     short tag_length[TAG_COUNT];
 228     long flag;
 229
 230     long data_length;
 231 };
 232
 233 struct tempbuf_id_list {
 234     long id;
 235     struct tempbuf_id_list *next;
 236 };
 237
 238 struct tempbuf_searchidx {
 239     long idx_id;
 240     char *str;
 241     int seek;
 242     struct tempbuf_id_list idlist;
 243 };
 244
 245 /* Lookup buffer for fixing messed up index while after sorting. */
 246 static long commit_entry_count;
 247 static long lookup_buffer_depth;
 248 static struct tempbuf_searchidx **lookup;
 249
 250 /* Used when building the temporary file. */
 251 static int cachefd = -1, filenametag_fd;
 252 static int total_entry_count = 0;
 253 static int data_size = 0;
 254 static int processed_dir_count;
 255
 256 /* Thread safe locking */
 257 static volatile int write_lock;
 258 static volatile int read_lock;
 259
 260 static bool delete_entry(long idx_id);
 261
 262 const char* tagcache_tag_to_str(int tag)
 263 {
 264     return tags_str[tag];
 265 }
 266
 267 #ifdef HAVE_DIRCACHE
 268 /**
 269  * Returns true if specified flag is still present, i.e., dircache
 270  * has not been reloaded.
 271  */
 272 static bool is_dircache_intact(void)
 273 {
 274     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 275 }
 276 #endif
 277
 278 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 279 {
 280     int fd;
 281     char buf[MAX_PATH];
 282     int rc;
 283
 284     if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
 285         return -1;
 286
 287     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 288
 289     fd = open(buf, write ? O_RDWR : O_RDONLY);
 290     if (fd < 0)
 291     {
 292         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 293         tc_stat.ready = false;
 294         return fd;
 295     }
 296
 297     /* Check the header. */
 298     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 299     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 300     {
 301         logf("header error");
 302         tc_stat.ready = false;
 303         close(fd);
 304         return -2;
 305     }
 306
 307     return fd;
 308 }
 309
 310 static int open_master_fd(struct master_header *hdr, bool write)
 311 {
 312     int fd;
 313     int rc;
 314
 315     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 316     if (fd < 0)
 317     {
 318         logf("master file open failed for R/W");
 319         tc_stat.ready = false;
 320         return fd;
 321     }
 322
 323     tc_stat.econ = false;
 324
 325     /* Check the header. */
 326     rc = read(fd, hdr, sizeof(struct master_header));
 327     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 328     {
 329         /* Success. */
 330         return fd;
 331     }
 332
 333     /* Trying to read again, this time with endianess correction enabled. */
 334     lseek(fd, 0, SEEK_SET);
 335
 336     rc = ecread(fd, hdr, 1, master_header_ec, true);
 337     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 338     {
 339         logf("header error");
 340         tc_stat.ready = false;
 341         close(fd);
 342         return -2;
 343     }
 344
 345     tc_stat.econ = true;
 346
 347     return fd;
 348 }
 349
 350 #ifndef __PCTOOL__
 351 static bool do_timed_yield(void)
 352 {
 353     /* Sorting can lock up for quite a while, so yield occasionally */
 354     static long wakeup_tick = 0;
 355     if (TIME_AFTER(current_tick, wakeup_tick))
 356     {
 357         wakeup_tick = current_tick + (HZ/4);
 358         yield();
 359         return true;
 360     }
 361     return false;
 362 }
 363 #endif
 364
 365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 366 static long find_entry_ram(const char *filename,
 367                            const struct dirent *dc)
 368 {
 369     static long last_pos = 0;
 370     int i;
 371
 372     /* Check if we tagcache is loaded into ram. */
 373     if (!tc_stat.ramcache)
 374         return -1;
 375
 376     if (dc == NULL)
 377         dc = dircache_get_entry_ptr(filename);
 378
 379     if (dc == NULL)
 380     {
 381         logf("tagcache: file not found.");
 382         return -1;
 383     }
 384
 385     try_again:
 386
 387     if (last_pos > 0)
 388         i = last_pos;
 389     else
 390         i = 0;
 391
 392     for (; i < hdr->h.tch.entry_count; i++)
 393     {
 394         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 395         {
 396             last_pos = MAX(0, i - 3);
 397             return i;
 398         }
 399
 400         do_timed_yield();
 401     }
 402
 403     if (last_pos > 0)
 404     {
 405         last_pos = 0;
 406         goto try_again;
 407     }
 408
 409     return -1;
 410 }
 411 #endif
 412
 413 static long find_entry_disk(const char *filename, bool localfd)
 414 {
 415     struct tagcache_header tch;
 416     static long last_pos = -1;
 417     long pos_history[POS_HISTORY_COUNT];
 418     long pos_history_idx = 0;
 419     bool found = false;
 420     struct tagfile_entry tfe;
 421     int fd;
 422     char buf[TAG_MAXLEN+32];
 423     int i;
 424     int pos = -1;
 425
 426     if (!tc_stat.ready)
 427         return -2;
 428
 429     fd = filenametag_fd;
 430     if (fd < 0 || localfd)
 431     {
 432         last_pos = -1;
 433         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 434             return -1;
 435     }
 436
 437     check_again:
 438
 439     if (last_pos > 0)
 440         lseek(fd, last_pos, SEEK_SET);
 441     else
 442         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 443
 444     while (true)
 445     {
 446         pos = lseek(fd, 0, SEEK_CUR);
 447         for (i = pos_history_idx-1; i >= 0; i--)
 448             pos_history[i+1] = pos_history[i];
 449         pos_history[0] = pos;
 450
 451         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 452             != sizeof(struct tagfile_entry))
 453         {
 454             break ;
 455         }
 456
 457         if (tfe.tag_length >= (long)sizeof(buf))
 458         {
 459             logf("too long tag #1");
 460             close(fd);
 461             if (!localfd)
 462                 filenametag_fd = -1;
 463             last_pos = -1;
 464             return -2;
 465         }
 466
 467         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 468         {
 469             logf("read error #2");
 470             close(fd);
 471             if (!localfd)
 472                 filenametag_fd = -1;
 473             last_pos = -1;
 474             return -3;
 475         }
 476
 477         if (!strcasecmp(filename, buf))
 478         {
 479             last_pos = pos_history[pos_history_idx];
 480             found = true;
 481             break ;
 482         }
 483
 484         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 485             pos_history_idx++;
 486     }
 487
 488     /* Not found? */
 489     if (!found)
 490     {
 491         if (last_pos > 0)
 492         {
 493             last_pos = -1;
 494             logf("seek again");
 495             goto check_again;
 496         }
 497
 498         if (fd != filenametag_fd || localfd)
 499             close(fd);
 500         return -4;
 501     }
 502
 503     if (fd != filenametag_fd || localfd)
 504         close(fd);
 505
 506     return tfe.idx_id;
 507 }
 508
 509 static int find_index(const char *filename)
 510 {
 511     long idx_id = -1;
 512
 513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 514     if (tc_stat.ramcache && is_dircache_intact())
 515         idx_id = find_entry_ram(filename, NULL);
 516 #endif
 517
 518     if (idx_id < 0)
 519         idx_id = find_entry_disk(filename, true);
 520
 521     return idx_id;
 522 }
 523
 524 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 525 {
 526     int idx_id;
 527
 528     if (!tc_stat.ready)
 529         return false;
 530
 531     idx_id = find_index(filename);
 532     if (idx_id < 0)
 533         return false;
 534
 535     if (!tagcache_search(tcs, tag_filename))
 536         return false;
 537
 538     tcs->entry_count = 0;
 539     tcs->idx_id = idx_id;
 540
 541     return true;
 542 }
 543
 544 static bool get_index(int masterfd, int idxid,
 545                       struct index_entry *idx, bool use_ram)
 546 {
 547     bool localfd = false;
 548
 549     if (idxid < 0)
 550     {
 551         logf("Incorrect idxid: %d", idxid);
 552         return false;
 553     }
 554
 555 #ifdef HAVE_TC_RAMCACHE
 556     if (tc_stat.ramcache && use_ram)
 557     {
 558         if (hdr->indices[idxid].flag & FLAG_DELETED)
 559             return false;
 560
 561 # ifdef HAVE_DIRCACHE
 562         if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
 563             || is_dircache_intact())
 564 #endif
 565         {
 566             memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 567             return true;
 568         }
 569     }
 570 #else
 571     (void)use_ram;
 572 #endif
 573
 574     if (masterfd < 0)
 575     {
 576         struct master_header tcmh;
 577
 578         localfd = true;
 579         masterfd = open_master_fd(&tcmh, false);
 580         if (masterfd < 0)
 581             return false;
 582     }
 583
 584     lseek(masterfd, idxid * sizeof(struct index_entry)
 585           + sizeof(struct master_header), SEEK_SET);
 586     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 587         != sizeof(struct index_entry))
 588     {
 589         logf("read error #3");
 590         if (localfd)
 591             close(masterfd);
 592
 593         return false;
 594     }
 595
 596     if (localfd)
 597         close(masterfd);
 598
 599     if (idx->flag & FLAG_DELETED)
 600         return false;
 601
 602     return true;
 603 }
 604
 605 #ifndef __PCTOOL__
 606
 607 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 608 {
 609     /* We need to exclude all memory only flags & tags when writing to disk. */
 610     if (idx->flag & FLAG_DIRCACHE)
 611     {
 612         logf("memory only flags!");
 613         return false;
 614     }
 615
 616 #ifdef HAVE_TC_RAMCACHE
 617     /* Only update numeric data. Writing the whole index to RAM by memcpy
 618      * destroys dircache pointers!
 619      */
 620     if (tc_stat.ramcache)
 621     {
 622         int tag;
 623         struct index_entry *idx_ram = &hdr->indices[idxid];
 624
 625         for (tag = 0; tag < TAG_COUNT; tag++)
 626         {
 627             if (TAGCACHE_IS_NUMERIC(tag))
 628             {
 629                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 630             }
 631         }
 632
 633         /* Don't touch the dircache flag or attributes. */
 634         idx_ram->flag = (idx->flag & 0x0000ffff)
 635             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 636     }
 637 #endif
 638
 639     lseek(masterfd, idxid * sizeof(struct index_entry)
 640           + sizeof(struct master_header), SEEK_SET);
 641     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 642         != sizeof(struct index_entry))
 643     {
 644         logf("write error #3");
 645         logf("idxid: %d", idxid);
 646         return false;
 647     }
 648
 649     return true;
 650 }
 651
 652 #endif /* !__PCTOOL__ */
 653
 654 static bool open_files(struct tagcache_search *tcs, int tag)
 655 {
 656     if (tcs->idxfd[tag] < 0)
 657     {
 658         char fn[MAX_PATH];
 659
 660         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 661         tcs->idxfd[tag] = open(fn, O_RDONLY);
 662     }
 663
 664     if (tcs->idxfd[tag] < 0)
 665     {
 666         logf("File not open!");
 667         return false;
 668     }
 669
 670     return true;
 671 }
 672
 673 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 674                      int tag, char *buf, long size)
 675 {
 676     struct tagfile_entry tfe;
 677     long seek;
 678
 679     *buf = '\0';
 680
 681     if (TAGCACHE_IS_NUMERIC(tag))
 682         return false;
 683
 684     seek = idx->tag_seek[tag];
 685     if (seek < 0)
 686     {
 687         logf("Retrieve failed");
 688         return false;
 689     }
 690
 691 #ifdef HAVE_TC_RAMCACHE
 692     if (tcs->ramsearch)
 693     {
 694         struct tagfile_entry *ep;
 695
 696 # ifdef HAVE_DIRCACHE
 697         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 698             && is_dircache_intact())
 699         {
 700             dircache_copy_path((struct dirent *)seek,
 701                                buf, size);
 702             return true;
 703         }
 704         else
 705 # endif
 706         if (tag != tag_filename)
 707         {
 708             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 709             strlcpy(buf, ep->tag_data, size);
 710
 711             return true;
 712         }
 713     }
 714 #endif
 715
 716     if (!open_files(tcs, tag))
 717         return false;
 718
 719     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 720     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 721         != sizeof(struct tagfile_entry))
 722     {
 723         logf("read error #5");
 724         return false;
 725     }
 726
 727     if (tfe.tag_length >= size)
 728     {
 729         logf("too small buffer");
 730         return false;
 731     }
 732
 733     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 734         tfe.tag_length)
 735     {
 736         logf("read error #6");
 737         return false;
 738     }
 739
 740     buf[tfe.tag_length] = '\0';
 741
 742     return true;
 743 }
 744
 745 static long check_virtual_tags(int tag, const struct index_entry *idx)
 746 {
 747     long data = 0;
 748
 749     switch (tag)
 750     {
 751         case tag_virt_length_sec:
 752             data = (idx->tag_seek[tag_length]/1000) % 60;
 753             break;
 754
 755         case tag_virt_length_min:
 756             data = (idx->tag_seek[tag_length]/1000) / 60;
 757             break;
 758
 759         case tag_virt_playtime_sec:
 760             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 761             break;
 762
 763         case tag_virt_playtime_min:
 764             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 765             break;
 766
 767         case tag_virt_autoscore:
 768             if (idx->tag_seek[tag_length] == 0
 769                 || idx->tag_seek[tag_playcount] == 0)
 770             {
 771                 data = 0;
 772             }
 773             else
 774             {
 775                 data = 100 * idx->tag_seek[tag_playtime]
 776                     / idx->tag_seek[tag_length]
 777                     / idx->tag_seek[tag_playcount];
 778             }
 779             break;
 780
 781         /* How many commits before the file has been added to the DB. */
 782         case tag_virt_entryage:
 783             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 784             break;
 785
 786         default:
 787             data = idx->tag_seek[tag];
 788     }
 789
 790     return data;
 791 }
 792
 793 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 794 {
 795     struct index_entry idx;
 796
 797     if (!tc_stat.ready)
 798         return false;
 799
 800     if (!TAGCACHE_IS_NUMERIC(tag))
 801         return -1;
 802
 803     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 804         return -2;
 805
 806     return check_virtual_tags(tag, &idx);
 807 }
 808
 809 inline static bool str_ends_with(const char *str1, const char *str2)
 810 {
 811     int str_len = strlen(str1);
 812     int clause_len = strlen(str2);
 813
 814     if (clause_len > str_len)
 815         return false;
 816
 817     return !strcasecmp(&str1[str_len - clause_len], str2);
 818 }
 819
 820 inline static bool str_oneof(const char *str, const char *list)
 821 {
 822     const char *sep;
 823     int l, len = strlen(str);
 824
 825     while (*list)
 826     {
 827         sep = strchr(list, '|');
 828         l = sep ? (long)sep - (long)list : (int)strlen(list);
 829         if ((l==len) && !strncasecmp(str, list, len))
 830             return true;
 831         list += sep ? l + 1 : l;
 832     }
 833
 834     return false;
 835 }
 836
 837 static bool check_against_clause(long numeric, const char *str,
 838                                  const struct tagcache_search_clause *clause)
 839 {
 840     if (clause->numeric)
 841     {
 842         switch (clause->type)
 843         {
 844             case clause_is:
 845                 return numeric == clause->numeric_data;
 846             case clause_is_not:
 847                 return numeric != clause->numeric_data;
 848             case clause_gt:
 849                 return numeric > clause->numeric_data;
 850             case clause_gteq:
 851                 return numeric >= clause->numeric_data;
 852             case clause_lt:
 853                 return numeric < clause->numeric_data;
 854             case clause_lteq:
 855                 return numeric <= clause->numeric_data;
 856             default:
 857                 logf("Incorrect numeric tag: %d", clause->type);
 858         }
 859     }
 860     else
 861     {
 862         switch (clause->type)
 863         {
 864             case clause_is:
 865                 return !strcasecmp(clause->str, str);
 866             case clause_is_not:
 867                 return strcasecmp(clause->str, str);
 868             case clause_gt:
 869                 return 0>strcasecmp(clause->str, str);
 870             case clause_gteq:
 871                 return 0>=strcasecmp(clause->str, str);
 872             case clause_lt:
 873                 return 0<strcasecmp(clause->str, str);
 874             case clause_lteq:
 875                 return 0<=strcasecmp(clause->str, str);
 876             case clause_contains:
 877                 return (strcasestr(str, clause->str) != NULL);
 878             case clause_not_contains:
 879                 return (strcasestr(str, clause->str) == NULL);
 880             case clause_begins_with:
 881                 return (strcasestr(str, clause->str) == str);
 882             case clause_not_begins_with:
 883                 return (strcasestr(str, clause->str) != str);
 884             case clause_ends_with:
 885                 return str_ends_with(str, clause->str);
 886             case clause_not_ends_with:
 887                 return !str_ends_with(str, clause->str);
 888             case clause_oneof:
 889                 return str_oneof(str, clause->str);
 890
 891             default:
 892                 logf("Incorrect tag: %d", clause->type);
 893         }
 894     }
 895
 896     return false;
 897 }
 898
 899 static bool check_clauses(struct tagcache_search *tcs,
 900                           struct index_entry *idx,
 901                           struct tagcache_search_clause **clause, int count)
 902 {
 903     int i;
 904
 905 #ifdef HAVE_TC_RAMCACHE
 906     if (tcs->ramsearch)
 907     {
 908         /* Go through all conditional clauses. */
 909         for (i = 0; i < count; i++)
 910         {
 911             struct tagfile_entry *tfe;
 912             int seek;
 913             char buf[256];
 914             char *str = NULL;
 915
 916             seek = check_virtual_tags(clause[i]->tag, idx);
 917
 918             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 919             {
 920                 if (clause[i]->tag == tag_filename)
 921                 {
 922                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 923                     str = buf;
 924                 }
 925                 else
 926                 {
 927                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 928                     str = tfe->tag_data;
 929                 }
 930             }
 931
 932             if (!check_against_clause(seek, str, clause[i]))
 933                 return false;
 934         }
 935     }
 936     else
 937 #endif
 938     {
 939         /* Check for conditions. */
 940         for (i = 0; i < count; i++)
 941         {
 942             struct tagfile_entry tfe;
 943             int seek;
 944             char str[256];
 945
 946             seek = check_virtual_tags(clause[i]->tag, idx);
 947
 948             memset(str, 0, sizeof str);
 949             if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
 950             {
 951                 int fd = tcs->idxfd[clause[i]->tag];
 952                 lseek(fd, seek, SEEK_SET);
 953                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 954                 if (tfe.tag_length >= (int)sizeof(str))
 955                 {
 956                     logf("Too long tag read!");
 957                     break ;
 958                 }
 959
 960                 read(fd, str, tfe.tag_length);
 961
 962                 /* Check if entry has been deleted. */
 963                 if (str[0] == '\0')
 964                     break;
 965             }
 966
 967             if (!check_against_clause(seek, str, clause[i]))
 968                 return false;
 969         }
 970     }
 971
 972     return true;
 973 }
 974
 975 bool tagcache_check_clauses(struct tagcache_search *tcs,
 976                             struct tagcache_search_clause **clause, int count)
 977 {
 978     struct index_entry idx;
 979
 980     if (count == 0)
 981         return true;
 982
 983     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 984         return false;
 985
 986     return check_clauses(tcs, &idx, clause, count);
 987 }
 988
 989 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
 990 {
 991     int i;
 992
 993     /* If uniq buffer is not defined we must return true for search to work. */
 994     if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
 995         && !TAGCACHE_IS_NUMERIC(tcs->type)))
 996     {
 997         return true;
 998     }
 999
1000     for (i = 0; i < tcs->unique_list_count; i++)
1001     {
1002         /* Return false if entry is found. */
1003         if (tcs->unique_list[i] == id)
1004             return false;
1005     }
1006
1007     if (tcs->unique_list_count < tcs->unique_list_capacity)
1008     {
1009         tcs->unique_list[i] = id;
1010         tcs->unique_list_count++;
1011     }
1012
1013     return true;
1014 }
1015
1016 static bool build_lookup_list(struct tagcache_search *tcs)
1017 {
1018     struct index_entry entry;
1019     int i, j;
1020
1021     tcs->seek_list_count = 0;
1022
1023 #ifdef HAVE_TC_RAMCACHE
1024     if (tcs->ramsearch
1025 # ifdef HAVE_DIRCACHE
1026         && (tcs->type != tag_filename || is_dircache_intact())
1027 # endif
1028         )
1029     {
1030         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1031         {
1032             struct tagcache_seeklist_entry *seeklist;
1033             struct index_entry *idx = &hdr->indices[i];
1034             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1035                 break ;
1036
1037             /* Skip deleted files. */
1038             if (idx->flag & FLAG_DELETED)
1039                 continue;
1040
1041             /* Go through all filters.. */
1042             for (j = 0; j < tcs->filter_count; j++)
1043             {
1044                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1045                 {
1046                     break ;
1047                 }
1048             }
1049
1050             if (j < tcs->filter_count)
1051                 continue ;
1052
1053             /* Check for conditions. */
1054             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1055                 continue;
1056
1057             /* Add to the seek list if not already in uniq buffer. */
1058             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1059                 continue;
1060
1061             /* Lets add it. */
1062             seeklist = &tcs->seeklist[tcs->seek_list_count];
1063             seeklist->seek = idx->tag_seek[tcs->type];
1064             seeklist->flag = idx->flag;
1065             seeklist->idx_id = i;
1066             tcs->seek_list_count++;
1067         }
1068
1069         tcs->seek_pos = i;
1070
1071         return tcs->seek_list_count > 0;
1072     }
1073 #endif
1074
1075     if (tcs->masterfd < 0)
1076     {
1077         struct master_header tcmh;
1078         tcs->masterfd = open_master_fd(&tcmh, false);
1079     }
1080
1081     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1082             sizeof(struct master_header), SEEK_SET);
1083
1084     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1085            == sizeof(struct index_entry))
1086     {
1087         struct tagcache_seeklist_entry *seeklist;
1088
1089         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1090             break ;
1091
1092         i = tcs->seek_pos;
1093         tcs->seek_pos++;
1094
1095         /* Check if entry has been deleted. */
1096         if (entry.flag & FLAG_DELETED)
1097             continue;
1098
1099         /* Go through all filters.. */
1100         for (j = 0; j < tcs->filter_count; j++)
1101         {
1102             if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1103                 break ;
1104         }
1105
1106         if (j < tcs->filter_count)
1107             continue ;
1108
1109         /* Check for conditions. */
1110         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1111             continue;
1112
1113         /* Add to the seek list if not already in uniq buffer. */
1114         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1115             continue;
1116
1117         /* Lets add it. */
1118         seeklist = &tcs->seeklist[tcs->seek_list_count];
1119         seeklist->seek = entry.tag_seek[tcs->type];
1120         seeklist->flag = entry.flag;
1121         seeklist->idx_id = i;
1122         tcs->seek_list_count++;
1123
1124         yield();
1125     }
1126
1127     return tcs->seek_list_count > 0;
1128 }
1129
1130
1131 static void remove_files(void)
1132 {
1133     int i;
1134     char buf[MAX_PATH];
1135
1136     tc_stat.ready = false;
1137     tc_stat.ramcache = false;
1138     tc_stat.econ = false;
1139     remove(TAGCACHE_FILE_MASTER);
1140     for (i = 0; i < TAG_COUNT; i++)
1141     {
1142         if (TAGCACHE_IS_NUMERIC(i))
1143             continue;
1144
1145         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1146         remove(buf);
1147     }
1148 }
1149
1150
1151 static bool check_all_headers(void)
1152 {
1153     struct master_header myhdr;
1154     struct tagcache_header tch;
1155     int tag;
1156     int fd;
1157
1158     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1159         return false;
1160
1161     close(fd);
1162     if (myhdr.dirty)
1163     {
1164         logf("tagcache is dirty!");
1165         return false;
1166     }
1167
1168     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1169
1170     for (tag = 0; tag < TAG_COUNT; tag++)
1171     {
1172         if (TAGCACHE_IS_NUMERIC(tag))
1173             continue;
1174
1175         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1176             return false;
1177
1178         close(fd);
1179     }
1180
1181     return true;
1182 }
1183
1184 bool tagcache_is_busy(void)
1185 {
1186     return read_lock || write_lock;
1187 }
1188
1189 bool tagcache_search(struct tagcache_search *tcs, int tag)
1190 {
1191     struct tagcache_header tag_hdr;
1192     struct master_header   master_hdr;
1193     int i;
1194
1195     while (read_lock)
1196         sleep(1);
1197
1198     memset(tcs, 0, sizeof(struct tagcache_search));
1199     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1200         return false;
1201
1202     tcs->position = sizeof(struct tagcache_header);
1203     tcs->type = tag;
1204     tcs->seek_pos = 0;
1205     tcs->list_position = 0;
1206     tcs->seek_list_count = 0;
1207     tcs->filter_count = 0;
1208     tcs->masterfd = -1;
1209
1210     for (i = 0; i < TAG_COUNT; i++)
1211         tcs->idxfd[i] = -1;
1212
1213 #ifndef HAVE_TC_RAMCACHE
1214     tcs->ramsearch = false;
1215 #else
1216     tcs->ramsearch = tc_stat.ramcache;
1217     if (tcs->ramsearch)
1218     {
1219         tcs->entry_count = hdr->entry_count[tcs->type];
1220     }
1221     else
1222 #endif
1223     {
1224         /* Always open as R/W so we can pass tcs to functions that modify data also
1225          * without failing. */
1226         tcs->masterfd = open_master_fd(&master_hdr, true);
1227         if (tcs->masterfd < 0)
1228             return false;
1229
1230         if (!TAGCACHE_IS_NUMERIC(tcs->type))
1231         {
1232             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1233             if (tcs->idxfd[tcs->type] < 0)
1234                 return false;
1235
1236             tcs->entry_count = tag_hdr.entry_count;
1237         }
1238         else
1239         {
1240             tcs->entry_count = master_hdr.tch.entry_count;
1241         }
1242     }
1243
1244     tcs->valid = true;
1245     tcs->initialized = true;
1246     write_lock++;
1247
1248     return true;
1249 }
1250
1251 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1252                                  void *buffer, long length)
1253 {
1254     tcs->unique_list = (unsigned long *)buffer;
1255     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1256     tcs->unique_list_count = 0;
1257 }
1258
1259 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1260                                 int tag, int seek)
1261 {
1262     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1263         return false;
1264
1265     if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1266         return false;
1267
1268     tcs->filter_tag[tcs->filter_count] = tag;
1269     tcs->filter_seek[tcs->filter_count] = seek;
1270     tcs->filter_count++;
1271
1272     return true;
1273 }
1274
1275 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1276                                 struct tagcache_search_clause *clause)
1277 {
1278     int i;
1279
1280     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1281     {
1282         logf("Too many clauses");
1283         return false;
1284     }
1285
1286     /* Check if there is already a similar filter in present (filters are
1287      * much faster than clauses).
1288      */
1289     for (i = 0; i < tcs->filter_count; i++)
1290     {
1291         if (tcs->filter_tag[i] == clause->tag)
1292             return true;
1293     }
1294
1295     if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1296     {
1297         char buf[MAX_PATH];
1298
1299         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1300         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1301     }
1302
1303     tcs->clause[tcs->clause_count] = clause;
1304     tcs->clause_count++;
1305
1306     return true;
1307 }
1308
1309 static bool get_next(struct tagcache_search *tcs)
1310 {
1311     static char buf[TAG_MAXLEN+32];
1312     struct tagfile_entry entry;
1313     long flag = 0;
1314
1315     if (!tcs->valid || !tc_stat.ready)
1316         return false;
1317
1318     if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1319 #ifdef HAVE_TC_RAMCACHE
1320         && !tcs->ramsearch
1321 #endif
1322         )
1323         return false;
1324
1325     /* Relative fetch. */
1326     if (tcs->filter_count > 0 || tcs->clause_count > 0
1327         || TAGCACHE_IS_NUMERIC(tcs->type)
1328 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1329         /* We need to retrieve flag status for dircache. */
1330         || (tcs->ramsearch && tcs->type == tag_filename)
1331 #endif
1332         )
1333     {
1334         struct tagcache_seeklist_entry *seeklist;
1335
1336         /* Check for end of list. */
1337         if (tcs->list_position == tcs->seek_list_count)
1338         {
1339             tcs->list_position = 0;
1340
1341             /* Try to fetch more. */
1342             if (!build_lookup_list(tcs))
1343             {
1344                 tcs->valid = false;
1345                 return false;
1346             }
1347         }
1348
1349         seeklist = &tcs->seeklist[tcs->list_position];
1350         flag = seeklist->flag;
1351         tcs->position = seeklist->seek;
1352         tcs->idx_id = seeklist->idx_id;
1353         tcs->list_position++;
1354     }
1355     else
1356     {
1357         if (tcs->entry_count == 0)
1358         {
1359             tcs->valid = false;
1360             return false;
1361         }
1362
1363         tcs->entry_count--;
1364     }
1365
1366     tcs->result_seek = tcs->position;
1367
1368     if (TAGCACHE_IS_NUMERIC(tcs->type))
1369     {
1370         snprintf(buf, sizeof(buf), "%d", tcs->position);
1371         tcs->result = buf;
1372         tcs->result_len = strlen(buf) + 1;
1373         return true;
1374     }
1375
1376     /* Direct fetch. */
1377 #ifdef HAVE_TC_RAMCACHE
1378     if (tcs->ramsearch)
1379     {
1380
1381 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1382         if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1383             && is_dircache_intact())
1384         {
1385             dircache_copy_path((struct dirent *)tcs->position,
1386                                buf, sizeof buf);
1387             tcs->result = buf;
1388             tcs->result_len = strlen(buf) + 1;
1389             tcs->ramresult = false;
1390
1391             return true;
1392         }
1393         else
1394 #endif
1395         if (tcs->type != tag_filename)
1396         {
1397             struct tagfile_entry *ep;
1398
1399             ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1400             tcs->result = ep->tag_data;
1401             tcs->result_len = strlen(tcs->result) + 1;
1402             tcs->idx_id = ep->idx_id;
1403             tcs->ramresult = true;
1404
1405             /* Increase position for the next run. This may get overwritten. */
1406             tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1407
1408             return true;
1409         }
1410     }
1411 #endif
1412
1413     if (!open_files(tcs, tcs->type))
1414     {
1415         tcs->valid = false;
1416         return false;
1417     }
1418
1419     /* Seek stream to the correct position and continue to direct fetch. */
1420     lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1421
1422     if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1423                tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1424     {
1425         logf("read error #5");
1426         tcs->valid = false;
1427         return false;
1428     }
1429
1430     if (entry.tag_length > (long)sizeof(buf))
1431     {
1432         tcs->valid = false;
1433         logf("too long tag #2");
1434         logf("P:%X/%X", tcs->position, entry.tag_length);
1435         return false;
1436     }
1437
1438     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1439     {
1440         tcs->valid = false;
1441         logf("read error #4");
1442         return false;
1443     }
1444
1445     /**
1446      Update the position for the next read (this may be overridden
1447      if filters or clauses are being used).
1448      */
1449     tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1450     tcs->result = buf;
1451     tcs->result_len = strlen(tcs->result) + 1;
1452     tcs->idx_id = entry.idx_id;
1453     tcs->ramresult = false;
1454
1455     return true;
1456 }
1457
1458 bool tagcache_get_next(struct tagcache_search *tcs)
1459 {
1460     while (get_next(tcs))
1461     {
1462         if (tcs->result_len > 1)
1463             return true;
1464     }
1465
1466     return false;
1467 }
1468
1469 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1470                        int tag, char *buf, long size)
1471 {
1472     struct index_entry idx;
1473
1474     *buf = '\0';
1475     if (!get_index(tcs->masterfd, idxid, &idx, true))
1476         return false;
1477
1478     return retrieve(tcs, &idx, tag, buf, size);
1479 }
1480
1481 static bool update_master_header(void)
1482 {
1483     struct master_header myhdr;
1484     int fd;
1485
1486     if (!tc_stat.ready)
1487         return false;
1488
1489     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1490         return false;
1491
1492     myhdr.serial = current_tcmh.serial;
1493     myhdr.commitid = current_tcmh.commitid;
1494     myhdr.dirty = current_tcmh.dirty;
1495
1496     /* Write it back */
1497     lseek(fd, 0, SEEK_SET);
1498     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1499     close(fd);
1500
1501 #ifdef HAVE_TC_RAMCACHE
1502     if (hdr)
1503     {
1504         hdr->h.serial = current_tcmh.serial;
1505         hdr->h.commitid = current_tcmh.commitid;
1506         hdr->h.dirty = current_tcmh.dirty;
1507     }
1508 #endif
1509
1510     return true;
1511 }
1512
1513 #if 0
1514
1515 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1516 {
1517     struct tagentry *entry;
1518
1519     if (tcs->type != tag_title)
1520         return ;
1521
1522     /* We will need reserve buffer for this. */
1523     if (tcs->ramcache)
1524     {
1525         struct tagfile_entry *ep;
1526
1527         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1528         tcs->seek_list[tcs->seek_list_count];
1529     }
1530
1531     entry = find_entry_ram();
1532
1533 }
1534 #endif
1535
1536 void tagcache_search_finish(struct tagcache_search *tcs)
1537 {
1538     int i;
1539
1540     if (!tcs->initialized)
1541         return;
1542
1543     if (tcs->masterfd >= 0)
1544     {
1545         close(tcs->masterfd);
1546         tcs->masterfd = -1;
1547     }
1548
1549     for (i = 0; i < TAG_COUNT; i++)
1550     {
1551         if (tcs->idxfd[i] >= 0)
1552         {
1553             close(tcs->idxfd[i]);
1554             tcs->idxfd[i] = -1;
1555         }
1556     }
1557
1558     tcs->ramsearch = false;
1559     tcs->valid = false;
1560     tcs->initialized = 0;
1561     if (write_lock > 0)
1562         write_lock--;
1563 }
1564
1565 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1566 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1567 {
1568     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1569 }
1570
1571 static long get_tag_numeric(const struct index_entry *entry, int tag)
1572 {
1573     return check_virtual_tags(tag, entry);
1574 }
1575
1576 static char* get_tag_string(const struct index_entry *entry, int tag)
1577 {
1578     char* s = get_tag(entry, tag)->tag_data;
1579     return strcmp(s, UNTAGGED) ? s : NULL;
1580 }
1581
1582 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1583 {
1584     struct index_entry *entry;
1585     int idx_id;
1586
1587     if (!tc_stat.ready || !tc_stat.ramcache)
1588         return false;
1589
1590     /* Find the corresponding entry in tagcache. */
1591     idx_id = find_entry_ram(filename, NULL);
1592     if (idx_id < 0)
1593         return false;
1594
1595     entry = &hdr->indices[idx_id];
1596
1597     id3->title        = get_tag_string(entry, tag_title);
1598     id3->artist       = get_tag_string(entry, tag_artist);
1599     id3->album        = get_tag_string(entry, tag_album);
1600     id3->genre_string = get_tag_string(entry, tag_genre);
1601     id3->composer     = get_tag_string(entry, tag_composer);
1602     id3->comment      = get_tag_string(entry, tag_comment);
1603     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1604     id3->grouping     = get_tag_string(entry, tag_grouping);
1605
1606     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1607     id3->rating     = get_tag_numeric(entry, tag_rating);
1608     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1609     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1610     id3->year       = get_tag_numeric(entry, tag_year);
1611
1612     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1613     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1614     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1615     if (id3->bitrate == 0)
1616         id3->bitrate = 1;
1617
1618     return true;
1619 }
1620 #endif
1621
1622 static inline void write_item(const char *item)
1623 {
1624     int len = strlen(item) + 1;
1625
1626     data_size += len;
1627     write(cachefd, item, len);
1628 }
1629
1630 static int check_if_empty(char **tag)
1631 {
1632     int length;
1633
1634     if (*tag == NULL || **tag == '\0')
1635     {
1636         *tag = UNTAGGED;
1637         return sizeof(UNTAGGED); /* Tag length */
1638     }
1639
1640     length = strlen(*tag);
1641     if (length > TAG_MAXLEN)
1642     {
1643         logf("over length tag: %s", *tag);
1644         length = TAG_MAXLEN;
1645         (*tag)[length] = '\0';
1646     }
1647
1648     return length + 1;
1649 }
1650
1651 #define ADD_TAG(entry,tag,data) \
1652     /* Adding tag */ \
1653     entry.tag_offset[tag] = offset; \
1654     entry.tag_length[tag] = check_if_empty(data); \
1655     offset += entry.tag_length[tag]
1656 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1657  * idea, as it uses lots of stack and is called from a recursive function
1658  * (check_dir).
1659  */
1660 static void __attribute__ ((noinline)) add_tagcache(char *path,
1661                                                     unsigned long mtime
1662 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1663                                                     ,const struct dirent *dc
1664 #endif
1665                                                    )
1666 {
1667     struct mp3entry id3;
1668     struct temp_file_entry entry;
1669     bool ret;
1670     int fd;
1671     int idx_id = -1;
1672     char tracknumfix[3];
1673     int offset = 0;
1674     int path_length = strlen(path);
1675     bool has_albumartist;
1676     bool has_grouping;
1677
1678 #ifdef SIMULATOR
1679     /* Crude logging for the sim - to aid in debugging */
1680     int logfd = open(ROCKBOX_DIR "/database.log",
1681                      O_WRONLY | O_APPEND | O_CREAT);
1682     if (logfd >= 0) {
1683         write(logfd, path, strlen(path));
1684         write(logfd, "\n", 1);
1685         close(logfd);
1686     }
1687 #endif
1688
1689     if (cachefd < 0)
1690         return ;
1691
1692     /* Check for overlength file path. */
1693     if (path_length > TAG_MAXLEN)
1694     {
1695         /* Path can't be shortened. */
1696         logf("Too long path: %s", path);
1697         return ;
1698     }
1699
1700     /* Check if the file is supported. */
1701     if (probe_file_format(path) == AFMT_UNKNOWN)
1702         return ;
1703
1704     /* Check if the file is already cached. */
1705 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1706     if (tc_stat.ramcache && is_dircache_intact())
1707     {
1708         idx_id = find_entry_ram(path, dc);
1709     }
1710 #endif
1711
1712     /* Be sure the entry doesn't exist. */
1713     if (filenametag_fd >= 0 && idx_id < 0)
1714         idx_id = find_entry_disk(path, false);
1715
1716     /* Check if file has been modified. */
1717     if (idx_id >= 0)
1718     {
1719         struct index_entry idx;
1720
1721         /* TODO: Mark that the index exists (for fast reverse scan) */
1722         //found_idx[idx_id/8] |= idx_id%8;
1723
1724         if (!get_index(-1, idx_id, &idx, true))
1725         {
1726             logf("failed to retrieve index entry");
1727             return ;
1728         }
1729
1730         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1731         {
1732             /* No changes to file. */
1733             return ;
1734         }
1735
1736         /* Metadata might have been changed. Delete the entry. */
1737         logf("Re-adding: %s", path);
1738         if (!delete_entry(idx_id))
1739         {
1740             logf("delete_entry failed: %d", idx_id);
1741             return ;
1742         }
1743     }
1744
1745     fd = open(path, O_RDONLY);
1746     if (fd < 0)
1747     {
1748         logf("open fail: %s", path);
1749         return ;
1750     }
1751
1752     memset(&id3, 0, sizeof(struct mp3entry));
1753     memset(&entry, 0, sizeof(struct temp_file_entry));
1754     memset(&tracknumfix, 0, sizeof(tracknumfix));
1755     ret = get_metadata(&id3, fd, path);
1756     close(fd);
1757
1758     if (!ret)
1759         return ;
1760
1761     logf("-> %s", path);
1762
1763     /* Generate track number if missing. */
1764     if (id3.tracknum <= 0)
1765     {
1766         const char *p = strrchr(path, '.');
1767
1768         if (p == NULL)
1769             p = &path[strlen(path)-1];
1770
1771         while (*p != '/')
1772         {
1773             if (isdigit(*p) && isdigit(*(p-1)))
1774             {
1775                 tracknumfix[1] = *p--;
1776                 tracknumfix[0] = *p;
1777                 break;
1778             }
1779             p--;
1780         }
1781
1782         if (tracknumfix[0] != '\0')
1783         {
1784             id3.tracknum = atoi(tracknumfix);
1785             /* Set a flag to indicate track number has been generated. */
1786             entry.flag |= FLAG_TRKNUMGEN;
1787         }
1788         else
1789         {
1790             /* Unable to generate track number. */
1791             id3.tracknum = -1;
1792         }
1793     }
1794
1795     /* Numeric tags */
1796     entry.tag_offset[tag_year] = id3.year;
1797     entry.tag_offset[tag_discnumber] = id3.discnum;
1798     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1799     entry.tag_offset[tag_length] = id3.length;
1800     entry.tag_offset[tag_bitrate] = id3.bitrate;
1801     entry.tag_offset[tag_mtime] = mtime;
1802
1803     /* String tags. */
1804     has_albumartist = id3.albumartist != NULL
1805         && strlen(id3.albumartist) > 0;
1806     has_grouping = id3.grouping != NULL
1807         && strlen(id3.grouping) > 0;
1808
1809     ADD_TAG(entry, tag_filename, &path);
1810     ADD_TAG(entry, tag_title, &id3.title);
1811     ADD_TAG(entry, tag_artist, &id3.artist);
1812     ADD_TAG(entry, tag_album, &id3.album);
1813     ADD_TAG(entry, tag_genre, &id3.genre_string);
1814     ADD_TAG(entry, tag_composer, &id3.composer);
1815     ADD_TAG(entry, tag_comment, &id3.comment);
1816     if (has_albumartist)
1817     {
1818         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1819     }
1820     else
1821     {
1822         ADD_TAG(entry, tag_albumartist, &id3.artist);
1823     }
1824     if (has_grouping)
1825     {
1826         ADD_TAG(entry, tag_grouping, &id3.grouping);
1827     }
1828     else
1829     {
1830         ADD_TAG(entry, tag_grouping, &id3.title);
1831     }
1832     entry.data_length = offset;
1833
1834     /* Write the header */
1835     write(cachefd, &entry, sizeof(struct temp_file_entry));
1836
1837     /* And tags also... Correct order is critical */
1838     write_item(path);
1839     write_item(id3.title);
1840     write_item(id3.artist);
1841     write_item(id3.album);
1842     write_item(id3.genre_string);
1843     write_item(id3.composer);
1844     write_item(id3.comment);
1845     if (has_albumartist)
1846     {
1847         write_item(id3.albumartist);
1848     }
1849     else
1850     {
1851         write_item(id3.artist);
1852     }
1853     if (has_grouping)
1854     {
1855         write_item(id3.grouping);
1856     }
1857     else
1858     {
1859         write_item(id3.title);
1860     }
1861     total_entry_count++;
1862 }
1863
1864 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1865 {
1866     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1867     int len = strlen(str)+1;
1868     int i;
1869     unsigned crc32;
1870     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1871     char buf[TAG_MAXLEN+32];
1872
1873     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1874         buf[i] = tolower(str[i]);
1875     buf[i] = '\0';
1876
1877     crc32 = crc_32(buf, i, 0xffffffff);
1878
1879     if (unique)
1880     {
1881         /* Check if the crc does not exist -> entry does not exist for sure. */
1882         for (i = 0; i < tempbufidx; i++)
1883         {
1884             if (crcbuf[-i] != crc32)
1885                 continue;
1886
1887             if (!strcasecmp(str, index[i].str))
1888             {
1889                 if (id < 0 || id >= lookup_buffer_depth)
1890                 {
1891                     logf("lookup buf overf.: %d", id);
1892                     return false;
1893                 }
1894
1895                 lookup[id] = &index[i];
1896                 return true;
1897             }
1898         }
1899     }
1900
1901     /* Insert to CRC buffer. */
1902     crcbuf[-tempbufidx] = crc32;
1903     tempbuf_left -= 4;
1904
1905     /* Insert it to the buffer. */
1906     tempbuf_left -= len;
1907     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1908         return false;
1909
1910     if (id >= lookup_buffer_depth)
1911     {
1912         logf("lookup buf overf. #2: %d", id);
1913         return false;
1914     }
1915
1916     if (id >= 0)
1917     {
1918         lookup[id] = &index[tempbufidx];
1919         index[tempbufidx].idlist.id = id;
1920     }
1921     else
1922         index[tempbufidx].idlist.id = -1;
1923
1924     index[tempbufidx].idlist.next = NULL;
1925     index[tempbufidx].idx_id = idx_id;
1926     index[tempbufidx].seek = -1;
1927     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1928     memcpy(index[tempbufidx].str, str, len);
1929     tempbuf_pos += len;
1930     tempbufidx++;
1931
1932     return true;
1933 }
1934
1935 static int compare(const void *p1, const void *p2)
1936 {
1937     do_timed_yield();
1938
1939     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1940     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1941
1942     if (strcmp(e1->str, UNTAGGED) == 0)
1943     {
1944         if (strcmp(e2->str, UNTAGGED) == 0)
1945             return 0;
1946         return -1;
1947     }
1948     else if (strcmp(e2->str, UNTAGGED) == 0)
1949         return 1;
1950
1951     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1952 }
1953
1954 static int tempbuf_sort(int fd)
1955 {
1956     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1957     struct tagfile_entry fe;
1958     int i;
1959     int length;
1960
1961     /* Generate reverse lookup entries. */
1962     for (i = 0; i < lookup_buffer_depth; i++)
1963     {
1964         struct tempbuf_id_list *idlist;
1965
1966         if (!lookup[i])
1967             continue;
1968
1969         if (lookup[i]->idlist.id == i)
1970             continue;
1971
1972         idlist = &lookup[i]->idlist;
1973         while (idlist->next != NULL)
1974             idlist = idlist->next;
1975
1976         tempbuf_left -= sizeof(struct tempbuf_id_list);
1977         if (tempbuf_left - 4 < 0)
1978             return -1;
1979
1980         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1981         if (tempbuf_pos & 0x03)
1982         {
1983             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1984             tempbuf_left -= 3;
1985             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1986         }
1987         tempbuf_pos += sizeof(struct tempbuf_id_list);
1988
1989         idlist = idlist->next;
1990         idlist->id = i;
1991         idlist->next = NULL;
1992
1993         do_timed_yield();
1994     }
1995
1996     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1997     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1998
1999     for (i = 0; i < tempbufidx; i++)
2000     {
2001         struct tempbuf_id_list *idlist = &index[i].idlist;
2002
2003         /* Fix the lookup list. */
2004         while (idlist != NULL)
2005         {
2006             if (idlist->id >= 0)
2007                 lookup[idlist->id] = &index[i];
2008             idlist = idlist->next;
2009         }
2010
2011         index[i].seek = lseek(fd, 0, SEEK_CUR);
2012         length = strlen(index[i].str) + 1;
2013         fe.tag_length = length;
2014         fe.idx_id = index[i].idx_id;
2015
2016         /* Check the chunk alignment. */
2017         if ((fe.tag_length + sizeof(struct tagfile_entry))
2018             % TAGFILE_ENTRY_CHUNK_LENGTH)
2019         {
2020             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2021                 ((fe.tag_length + sizeof(struct tagfile_entry))
2022                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2023         }
2024
2025 #ifdef TAGCACHE_STRICT_ALIGN
2026         /* Make sure the entry is long aligned. */
2027         if (index[i].seek & 0x03)
2028         {
2029             logf("tempbuf_sort: alignment error!");
2030             return -3;
2031         }
2032 #endif
2033
2034         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2035             sizeof(struct tagfile_entry))
2036         {
2037             logf("tempbuf_sort: write error #1");
2038             return -1;
2039         }
2040
2041         if (write(fd, index[i].str, length) != length)
2042         {
2043             logf("tempbuf_sort: write error #2");
2044             return -2;
2045         }
2046
2047         /* Write some padding. */
2048         if (fe.tag_length - length > 0)
2049             write(fd, "XXXXXXXX", fe.tag_length - length);
2050     }
2051
2052     return i;
2053 }
2054
2055 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2056 {
2057     if (id < 0 || id >= lookup_buffer_depth)
2058         return NULL;
2059
2060     return lookup[id];
2061 }
2062
2063
2064 inline static int tempbuf_find_location(int id)
2065 {
2066     struct tempbuf_searchidx *entry;
2067
2068     entry = tempbuf_locate(id);
2069     if (entry == NULL)
2070         return -1;
2071
2072     return entry->seek;
2073 }
2074
2075 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2076 {
2077     struct master_header tcmh;
2078     struct index_entry idx;
2079     int masterfd;
2080     int masterfd_pos;
2081     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2082     int max_entries;
2083     int entries_processed = 0;
2084     int i, j;
2085     char buf[TAG_MAXLEN];
2086
2087     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2088
2089     logf("Building numeric indices...");
2090     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2091
2092     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2093         return false;
2094
2095     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2096                          SEEK_CUR);
2097     if (masterfd_pos == filesize(masterfd))
2098     {
2099         logf("we can't append!");
2100         close(masterfd);
2101         return false;
2102     }
2103
2104     while (entries_processed < h->entry_count)
2105     {
2106         int count = MIN(h->entry_count - entries_processed, max_entries);
2107
2108         /* Read in as many entries as possible. */
2109         for (i = 0; i < count; i++)
2110         {
2111             struct temp_file_entry *tfe = &entrybuf[i];
2112             int datastart;
2113
2114             /* Read in numeric data. */
2115             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2116                 sizeof(struct temp_file_entry))
2117             {
2118                 logf("read fail #1");
2119                 close(masterfd);
2120                 return false;
2121             }
2122
2123             datastart = lseek(tmpfd, 0, SEEK_CUR);
2124
2125             /**
2126              * Read string data from the following tags:
2127              * - tag_filename
2128              * - tag_artist
2129              * - tag_album
2130              * - tag_title
2131              *
2132              * A crc32 hash is calculated from the read data
2133              * and stored back to the data offset field kept in memory.
2134              */
2135 #define tmpdb_read_string_tag(tag) \
2136     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2137     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2138     { \
2139         logf("read fail: buffer overflow"); \
2140         close(masterfd); \
2141         return false; \
2142     } \
2143     \
2144     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2145         tfe->tag_length[tag]) \
2146     { \
2147         logf("read fail #2"); \
2148         close(masterfd); \
2149         return false; \
2150     } \
2151     \
2152     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2153     lseek(tmpfd, datastart, SEEK_SET)
2154
2155             tmpdb_read_string_tag(tag_filename);
2156             tmpdb_read_string_tag(tag_artist);
2157             tmpdb_read_string_tag(tag_album);
2158             tmpdb_read_string_tag(tag_title);
2159
2160             /* Seek to the end of the string data. */
2161             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2162         }
2163
2164         /* Backup the master index position. */
2165         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2166         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2167
2168         /* Check if we can resurrect some deleted runtime statistics data. */
2169         for (i = 0; i < tcmh.tch.entry_count; i++)
2170         {
2171             /* Read the index entry. */
2172             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2173                 != sizeof(struct index_entry))
2174             {
2175                 logf("read fail #3");
2176                 close(masterfd);
2177                 return false;
2178             }
2179
2180             /**
2181              * Skip unless the entry is marked as being deleted
2182              * or the data has already been resurrected.
2183              */
2184             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2185                 continue;
2186
2187             /* Now try to match the entry. */
2188             /**
2189              * To succesfully match a song, the following conditions
2190              * must apply:
2191              *
2192              * For numeric fields: tag_length
2193              * - Full identical match is required
2194              *
2195              * If tag_filename matches, no further checking necessary.
2196              *
2197              * For string hashes: tag_artist, tag_album, tag_title
2198              * - Two of these must match
2199              */
2200             for (j = 0; j < count; j++)
2201             {
2202                 struct temp_file_entry *tfe = &entrybuf[j];
2203
2204                 /* Try to match numeric fields first. */
2205                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2206                     continue;
2207
2208                 /* Now it's time to do the hash matching. */
2209                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2210                 {
2211                     int match_count = 0;
2212
2213                     /* No filename match, check if we can match two other tags. */
2214 #define tmpdb_match(tag) \
2215     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2216         match_count++
2217
2218                     tmpdb_match(tag_artist);
2219                     tmpdb_match(tag_album);
2220                     tmpdb_match(tag_title);
2221
2222                     if (match_count < 2)
2223                     {
2224                         /* Still no match found, give up. */
2225                         continue;
2226                     }
2227                 }
2228
2229                 /* A match found, now copy & resurrect the statistical data. */
2230 #define tmpdb_copy_tag(tag) \
2231     tfe->tag_offset[tag] = idx.tag_seek[tag]
2232
2233                 tmpdb_copy_tag(tag_playcount);
2234                 tmpdb_copy_tag(tag_rating);
2235                 tmpdb_copy_tag(tag_playtime);
2236                 tmpdb_copy_tag(tag_lastplayed);
2237                 tmpdb_copy_tag(tag_commitid);
2238
2239                 /* Avoid processing this entry again. */
2240                 idx.flag |= FLAG_RESURRECTED;
2241
2242                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2243                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2244                     != sizeof(struct index_entry))
2245                 {
2246                     logf("masterfd writeback fail #1");
2247                     close(masterfd);
2248                     return false;
2249                 }
2250
2251                 logf("Entry resurrected");
2252             }
2253         }
2254
2255
2256         /* Restore the master index position. */
2257         lseek(masterfd, masterfd_pos, SEEK_SET);
2258
2259         /* Commit the data to the index. */
2260         for (i = 0; i < count; i++)
2261         {
2262             int loc = lseek(masterfd, 0, SEEK_CUR);
2263
2264             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2265                 != sizeof(struct index_entry))
2266             {
2267                 logf("read fail #3");
2268                 close(masterfd);
2269                 return false;
2270             }
2271
2272             for (j = 0; j < TAG_COUNT; j++)
2273             {
2274                 if (!TAGCACHE_IS_NUMERIC(j))
2275                     continue;
2276
2277                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2278             }
2279             idx.flag = entrybuf[i].flag;
2280
2281             if (idx.tag_seek[tag_commitid])
2282             {
2283                 /* Data has been resurrected. */
2284                 idx.flag |= FLAG_DIRTYNUM;
2285             }
2286             else if (tc_stat.ready && current_tcmh.commitid > 0)
2287             {
2288                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2289                 idx.flag |= FLAG_DIRTYNUM;
2290             }
2291
2292             /* Write back the updated index. */
2293             lseek(masterfd, loc, SEEK_SET);
2294             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2295                 != sizeof(struct index_entry))
2296             {
2297                 logf("write fail");
2298                 close(masterfd);
2299                 return false;
2300             }
2301         }
2302
2303         entries_processed += count;
2304         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2305     }
2306
2307     close(masterfd);
2308
2309     return true;
2310 }
2311
2312 /**
2313  * Return values:
2314  *     > 0   success
2315  *    == 0   temporary failure
2316  *     < 0   fatal error
2317  */
2318 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2319 {
2320     int i;
2321     struct tagcache_header tch;
2322     struct master_header   tcmh;
2323     struct index_entry idxbuf[IDX_BUF_DEPTH];
2324     int idxbuf_pos;
2325     char buf[TAG_MAXLEN+32];
2326     int fd = -1, masterfd;
2327     bool error = false;
2328     int init;
2329     int masterfd_pos;
2330
2331     logf("Building index: %d", index_type);
2332
2333     /* Check the number of entries we need to allocate ram for. */
2334     commit_entry_count = h->entry_count + 1;
2335
2336     masterfd = open_master_fd(&tcmh, false);
2337     if (masterfd >= 0)
2338     {
2339         commit_entry_count += tcmh.tch.entry_count;
2340         close(masterfd);
2341     }
2342     else
2343         remove_files(); /* Just to be sure we are clean. */
2344
2345     /* Open the index file, which contains the tag names. */
2346     fd = open_tag_fd(&tch, index_type, true);
2347     if (fd >= 0)
2348     {
2349         logf("tch.datasize=%ld", tch.datasize);
2350         lookup_buffer_depth = 1 +
2351         /* First part */ commit_entry_count +
2352         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2353     }
2354     else
2355     {
2356         lookup_buffer_depth = 1 +
2357         /* First part */ commit_entry_count +
2358         /* Second part */ 0;
2359     }
2360
2361     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2362     logf("commit_entry_count=%ld", commit_entry_count);
2363
2364     /* Allocate buffer for all index entries from both old and new
2365      * tag files. */
2366     tempbufidx = 0;
2367     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2368
2369     /* Allocate lookup buffer. The first portion of commit_entry_count
2370      * contains the new tags in the temporary file and the second
2371      * part for locating entries already in the db.
2372      *
2373      *  New tags  Old tags
2374      * +---------+---------------------------+
2375      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2376      * +---------+---------------------------+
2377      *
2378      * Old tags are inserted to a temporary buffer with position:
2379      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2380      * And new tags with index:
2381      *     tempbuf_insert(idx, ...);
2382      *
2383      * The buffer is sorted and written into tag file:
2384      *     tempbuf_sort(...);
2385      * leaving master index locations messed up.
2386      *
2387      * That is fixed using the lookup buffer for old tags:
2388      *     new_seek = tempbuf_find_location(old_seek, ...);
2389      * and for new tags:
2390      *     new_seek = tempbuf_find_location(idx);
2391      */
2392     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2393     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2394     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2395
2396     /* And calculate the remaining data space used mainly for storing
2397      * tag data (strings). */
2398     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2399     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2400     {
2401         logf("Buffer way too small!");
2402         return 0;
2403     }
2404
2405     if (fd >= 0)
2406     {
2407         /**
2408          * If tag file contains unique tags (sorted index), we will load
2409          * it entirely into memory so we can resort it later for use with
2410          * chunked browsing.
2411          */
2412         if (TAGCACHE_IS_SORTED(index_type))
2413         {
2414             logf("loading tags...");
2415             for (i = 0; i < tch.entry_count; i++)
2416             {
2417                 struct tagfile_entry entry;
2418                 int loc = lseek(fd, 0, SEEK_CUR);
2419                 bool ret;
2420
2421                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2422                     != sizeof(struct tagfile_entry))
2423                 {
2424                     logf("read error #7");
2425                     close(fd);
2426                     return -2;
2427                 }
2428
2429                 if (entry.tag_length >= (int)sizeof(buf))
2430                 {
2431                     logf("too long tag #3");
2432                     close(fd);
2433                     return -2;
2434                 }
2435
2436                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2437                 {
2438                     logf("read error #8");
2439                     close(fd);
2440                     return -2;
2441                 }
2442
2443                 /* Skip deleted entries. */
2444                 if (buf[0] == '\0')
2445                     continue;
2446
2447                 /**
2448                  * Save the tag and tag id in the memory buffer. Tag id
2449                  * is saved so we can later reindex the master lookup
2450                  * table when the index gets resorted.
2451                  */
2452                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2453                                      + commit_entry_count, entry.idx_id,
2454                                      TAGCACHE_IS_UNIQUE(index_type));
2455                 if (!ret)
2456                 {
2457                     close(fd);
2458                     return -3;
2459                 }
2460                 do_timed_yield();
2461             }
2462             logf("done");
2463         }
2464         else
2465             tempbufidx = tch.entry_count;
2466     }
2467     else
2468     {
2469         /**
2470          * Creating new index file to store the tags. No need to preload
2471          * anything whether the index type is sorted or not.
2472          */
2473         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2474         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2475         if (fd < 0)
2476         {
2477             logf("%s open fail", buf);
2478             return -2;
2479         }
2480
2481         tch.magic = TAGCACHE_MAGIC;
2482         tch.entry_count = 0;
2483         tch.datasize = 0;
2484
2485         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2486             != sizeof(struct tagcache_header))
2487         {
2488             logf("header write failed");
2489             close(fd);
2490             return -2;
2491         }
2492     }
2493
2494     /* Loading the tag lookup file as "master file". */
2495     logf("Loading index file");
2496     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2497
2498     if (masterfd < 0)
2499     {
2500         logf("Creating new DB");
2501         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2502
2503         if (masterfd < 0)
2504         {
2505             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2506             close(fd);
2507             return -2;
2508         }
2509
2510         /* Write the header (write real values later). */
2511         memset(&tcmh, 0, sizeof(struct master_header));
2512         tcmh.tch = *h;
2513         tcmh.tch.entry_count = 0;
2514         tcmh.tch.datasize = 0;
2515         tcmh.dirty = true;
2516         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2517         init = true;
2518         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2519     }
2520     else
2521     {
2522         /**
2523          * Master file already exists so we need to process the current
2524          * file first.
2525          */
2526         init = false;
2527
2528         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2529             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2530         {
2531             logf("header error");
2532             close(fd);
2533             close(masterfd);
2534             return -2;
2535         }
2536
2537         /**
2538          * If we reach end of the master file, we need to expand it to
2539          * hold new tags. If the current index is not sorted, we can
2540          * simply append new data to end of the file.
2541          * However, if the index is sorted, we need to update all tag
2542          * pointers in the master file for the current index.
2543          */
2544         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2545             SEEK_CUR);
2546         if (masterfd_pos == filesize(masterfd))
2547         {
2548             logf("appending...");
2549             init = true;
2550         }
2551     }
2552
2553     /**
2554      * Load new unique tags in memory to be sorted later and added
2555      * to the master lookup file.
2556      */
2557     if (TAGCACHE_IS_SORTED(index_type))
2558     {
2559         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2560         /* h is the header of the temporary file containing new tags. */
2561         logf("inserting new tags...");
2562         for (i = 0; i < h->entry_count; i++)
2563         {
2564             struct temp_file_entry entry;
2565
2566             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2567                 sizeof(struct temp_file_entry))
2568             {
2569                 logf("read fail #3");
2570                 error = true;
2571                 goto error_exit;
2572             }
2573
2574             /* Read data. */
2575             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2576             {
2577                 logf("too long entry!");
2578                 error = true;
2579                 goto error_exit;
2580             }
2581
2582             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2583             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2584                 entry.tag_length[index_type])
2585             {
2586                 logf("read fail #4");
2587                 error = true;
2588                 goto error_exit;
2589             }
2590
2591             if (TAGCACHE_IS_UNIQUE(index_type))
2592                 error = !tempbuf_insert(buf, i, -1, true);
2593             else
2594                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2595
2596             if (error)
2597             {
2598                 logf("insert error");
2599                 goto error_exit;
2600             }
2601
2602             /* Skip to next. */
2603             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2604                     entry.tag_length[index_type], SEEK_CUR);
2605             do_timed_yield();
2606         }
2607         logf("done");
2608
2609         /* Sort the buffer data and write it to the index file. */
2610         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2611         /**
2612          * We need to truncate the index file now. There can be junk left
2613          * at the end of file (however, we _should_ always follow the
2614          * entry_count and don't crash with that).
2615          */
2616         ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2617
2618         i = tempbuf_sort(fd);
2619         if (i < 0)
2620             goto error_exit;
2621         logf("sorted %d tags", i);
2622
2623         /**
2624          * Now update all indexes in the master lookup file.
2625          */
2626         logf("updating indices...");
2627         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2628         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2629         {
2630             int j;
2631             int loc = lseek(masterfd, 0, SEEK_CUR);
2632
2633             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2634
2635             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2636                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2637             {
2638                 logf("read fail #5");
2639                 error = true;
2640                 goto error_exit ;
2641             }
2642             lseek(masterfd, loc, SEEK_SET);
2643
2644             for (j = 0; j < idxbuf_pos; j++)
2645             {
2646                 if (idxbuf[j].flag & FLAG_DELETED)
2647                 {
2648                     /* We can just ignore deleted entries. */
2649                     // idxbuf[j].tag_seek[index_type] = 0;
2650                     continue;
2651                 }
2652
2653                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2654                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2655                     + commit_entry_count);
2656
2657                 if (idxbuf[j].tag_seek[index_type] < 0)
2658                 {
2659                     logf("update error: %d/%d/%d",
2660                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2661                     error = true;
2662                     goto error_exit;
2663                 }
2664
2665                 do_timed_yield();
2666             }
2667
2668             /* Write back the updated index. */
2669             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2670                         index_entry_ec, tc_stat.econ) !=
2671                 (int)sizeof(struct index_entry)*idxbuf_pos)
2672             {
2673                 logf("write fail");
2674                 error = true;
2675                 goto error_exit;
2676             }
2677         }
2678         logf("done");
2679     }
2680
2681     /**
2682      * Walk through the temporary file containing the new tags.
2683      */
2684     // build_normal_index(h, tmpfd, masterfd, idx);
2685     logf("updating new indices...");
2686     lseek(masterfd, masterfd_pos, SEEK_SET);
2687     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2688     lseek(fd, 0, SEEK_END);
2689     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2690     {
2691         int j;
2692
2693         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2694         if (init)
2695         {
2696             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2697         }
2698         else
2699         {
2700             int loc = lseek(masterfd, 0, SEEK_CUR);
2701
2702             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2703                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2704             {
2705                 logf("read fail #6");
2706                 error = true;
2707                 break ;
2708             }
2709             lseek(masterfd, loc, SEEK_SET);
2710         }
2711
2712         /* Read entry headers. */
2713         for (j = 0; j < idxbuf_pos; j++)
2714         {
2715             if (!TAGCACHE_IS_SORTED(index_type))
2716             {
2717                 struct temp_file_entry entry;
2718                 struct tagfile_entry fe;
2719
2720                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2721                     sizeof(struct temp_file_entry))
2722                 {
2723                     logf("read fail #7");
2724                     error = true;
2725                     break ;
2726                 }
2727
2728                 /* Read data. */
2729                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2730                 {
2731                     logf("too long entry!");
2732                     logf("length=%d", entry.tag_length[index_type]);
2733                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2734                     error = true;
2735                     break ;
2736                 }
2737
2738                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2739                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2740                     entry.tag_length[index_type])
2741                 {
2742                     logf("read fail #8");
2743                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2744                     logf("length=0x%02x", entry.tag_length[index_type]);
2745                     error = true;
2746                     break ;
2747                 }
2748
2749                 /* Write to index file. */
2750                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2751                 fe.tag_length = entry.tag_length[index_type];
2752                 fe.idx_id = tcmh.tch.entry_count + i + j;
2753                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2754                 write(fd, buf, fe.tag_length);
2755                 tempbufidx++;
2756
2757                 /* Skip to next. */
2758                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2759                       entry.tag_length[index_type], SEEK_CUR);
2760             }
2761             else
2762             {
2763                 /* Locate the correct entry from the sorted array. */
2764                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2765                 if (idxbuf[j].tag_seek[index_type] < 0)
2766                 {
2767                     logf("entry not found (%d)", j);
2768                     error = true;
2769                     break ;
2770                 }
2771             }
2772         }
2773
2774         /* Write index. */
2775         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2776                   index_entry_ec, tc_stat.econ) !=
2777             (int)sizeof(struct index_entry)*idxbuf_pos)
2778         {
2779             logf("tagcache: write fail #4");
2780             error = true;
2781             break ;
2782         }
2783
2784         do_timed_yield();
2785     }
2786     logf("done");
2787
2788     /* Finally write the header. */
2789     tch.magic = TAGCACHE_MAGIC;
2790     tch.entry_count = tempbufidx;
2791     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2792     lseek(fd, 0, SEEK_SET);
2793     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2794
2795     if (index_type != tag_filename)
2796         h->datasize += tch.datasize;
2797     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2798     error_exit:
2799
2800     close(fd);
2801     close(masterfd);
2802
2803     if (error)
2804         return -2;
2805
2806     return 1;
2807 }
2808
2809 static bool commit(void)
2810 {
2811     struct tagcache_header tch;
2812     struct master_header   tcmh;
2813     int i, len, rc;
2814     int tmpfd;
2815     int masterfd;
2816 #ifdef HAVE_DIRCACHE
2817     bool dircache_buffer_stolen = false;
2818 #endif
2819     bool local_allocation = false;
2820
2821     logf("committing tagcache");
2822
2823     while (write_lock)
2824         sleep(1);
2825
2826     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2827     if (tmpfd < 0)
2828     {
2829         logf("nothing to commit");
2830         return true;
2831     }
2832
2833
2834     /* Load the header. */
2835     len = sizeof(struct tagcache_header);
2836     rc = read(tmpfd, &tch, len);
2837
2838     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2839     {
2840         logf("incorrect header");
2841         close(tmpfd);
2842         remove(TAGCACHE_FILE_TEMP);
2843         return false;
2844     }
2845
2846     if (tch.entry_count == 0)
2847     {
2848         logf("nothing to commit");
2849         close(tmpfd);
2850         remove(TAGCACHE_FILE_TEMP);
2851         return true;
2852     }
2853
2854 #ifdef HAVE_EEPROM_SETTINGS
2855     remove(TAGCACHE_STATEFILE);
2856 #endif
2857
2858     /* At first be sure to unload the ramcache! */
2859 #ifdef HAVE_TC_RAMCACHE
2860     tc_stat.ramcache = false;
2861 #endif
2862
2863     read_lock++;
2864
2865     /* Try to steal every buffer we can :) */
2866     if (tempbuf_size == 0)
2867         local_allocation = true;
2868
2869 #ifdef HAVE_DIRCACHE
2870     if (tempbuf_size == 0)
2871     {
2872         /* Try to steal the dircache buffer. */
2873         tempbuf = dircache_steal_buffer(&tempbuf_size);
2874         tempbuf_size &= ~0x03;
2875
2876         if (tempbuf_size > 0)
2877         {
2878             dircache_buffer_stolen = true;
2879         }
2880     }
2881 #endif
2882
2883 #ifdef HAVE_TC_RAMCACHE
2884     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2885     {
2886         tempbuf = (char *)(hdr + 1);
2887         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2888         tempbuf_size &= ~0x03;
2889     }
2890 #endif
2891
2892     /* And finally fail if there are no buffers available. */
2893     if (tempbuf_size == 0)
2894     {
2895         logf("delaying commit until next boot");
2896         tc_stat.commit_delayed = true;
2897         close(tmpfd);
2898         read_lock--;
2899         return false;
2900     }
2901
2902     logf("commit %ld entries...", tch.entry_count);
2903
2904     /* Mark DB dirty so it will stay disabled if commit fails. */
2905     current_tcmh.dirty = true;
2906     update_master_header();
2907
2908     /* Now create the index files. */
2909     tc_stat.commit_step = 0;
2910     tch.datasize = 0;
2911     tc_stat.commit_delayed = false;
2912
2913     for (i = 0; i < TAG_COUNT; i++)
2914     {
2915         int ret;
2916
2917         if (TAGCACHE_IS_NUMERIC(i))
2918             continue;
2919
2920         tc_stat.commit_step++;
2921         ret = build_index(i, &tch, tmpfd);
2922         if (ret <= 0)
2923         {
2924             close(tmpfd);
2925             logf("tagcache failed init");
2926             if (ret == 0)
2927                 tc_stat.commit_delayed = true;
2928
2929             tc_stat.commit_step = 0;
2930             read_lock--;
2931             return false;
2932         }
2933     }
2934
2935     if (!build_numeric_indices(&tch, tmpfd))
2936     {
2937         logf("Failure to commit numeric indices");
2938         close(tmpfd);
2939         tc_stat.commit_step = 0;
2940         read_lock--;
2941         return false;
2942     }
2943
2944     close(tmpfd);
2945     remove(TAGCACHE_FILE_TEMP);
2946
2947     tc_stat.commit_step = 0;
2948
2949     /* Update the master index headers. */
2950     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2951     {
2952         read_lock--;
2953         return false;
2954     }
2955
2956     tcmh.tch.entry_count += tch.entry_count;
2957     tcmh.tch.datasize = sizeof(struct master_header)
2958         + sizeof(struct index_entry) * tcmh.tch.entry_count
2959         + tch.datasize;
2960     tcmh.dirty = false;
2961     tcmh.commitid++;
2962
2963     lseek(masterfd, 0, SEEK_SET);
2964     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2965     close(masterfd);
2966
2967     logf("tagcache committed");
2968     tc_stat.ready = check_all_headers();
2969     tc_stat.readyvalid = true;
2970
2971     if (local_allocation)
2972     {
2973         tempbuf = NULL;
2974         tempbuf_size = 0;
2975     }
2976
2977 #ifdef HAVE_DIRCACHE
2978     /* Rebuild the dircache, if we stole the buffer. */
2979     if (dircache_buffer_stolen)
2980         dircache_build(0);
2981 #endif
2982
2983 #ifdef HAVE_TC_RAMCACHE
2984     /* Reload tagcache. */
2985     if (tc_stat.ramcache_allocated > 0)
2986         tagcache_start_scan();
2987 #endif
2988
2989     read_lock--;
2990
2991     return true;
2992 }
2993
2994 static void allocate_tempbuf(void)
2995 {
2996     /* Yeah, malloc would be really nice now :) */
2997 #ifdef __PCTOOL__
2998     tempbuf_size = 32*1024*1024;
2999     tempbuf = malloc(tempbuf_size);
3000 #else
3001     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3002     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3003     audiobuf += tempbuf_size;
3004 #endif
3005 }
3006
3007 static void free_tempbuf(void)
3008 {
3009     if (tempbuf_size == 0)
3010         return ;
3011
3012 #ifdef __PCTOOL__
3013     free(tempbuf);
3014 #else
3015     audiobuf -= tempbuf_size;
3016 #endif
3017     tempbuf = NULL;
3018     tempbuf_size = 0;
3019 }
3020
3021 #ifndef __PCTOOL__
3022
3023 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3024 {
3025     struct index_entry idx;
3026
3027     if (!tc_stat.ready)
3028         return false;
3029
3030     if (!TAGCACHE_IS_NUMERIC(tag))
3031         return false;
3032
3033     if (!get_index(masterfd, idx_id, &idx, false))
3034         return false;
3035
3036     idx.tag_seek[tag] = data;
3037     idx.flag |= FLAG_DIRTYNUM;
3038
3039     return write_index(masterfd, idx_id, &idx);
3040 }
3041
3042 #if 0
3043 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3044                                    int tag, long data)
3045 {
3046     struct master_header myhdr;
3047
3048     if (tcs->masterfd < 0)
3049     {
3050         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3051             return false;
3052     }
3053
3054     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3055 }
3056 #endif
3057
3058 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3059
3060 static bool command_queue_is_full(void)
3061 {
3062     int next;
3063
3064     next = command_queue_widx + 1;
3065     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3066         next = 0;
3067
3068     return (next == command_queue_ridx);
3069 }
3070
3071 static void command_queue_sync_callback(void *data)
3072 {
3073     (void)data;
3074     struct master_header myhdr;
3075     int masterfd;
3076
3077     mutex_lock(&command_queue_mutex);
3078
3079     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3080         return;
3081
3082     while (command_queue_ridx != command_queue_widx)
3083     {
3084         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3085
3086         switch (ce->command)
3087         {
3088             case CMD_UPDATE_MASTER_HEADER:
3089             {
3090                 close(masterfd);
3091                 update_master_header();
3092
3093                 /* Re-open the masterfd. */
3094                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3095                     return;
3096
3097                 break;
3098             }
3099             case CMD_UPDATE_NUMERIC:
3100             {
3101                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3102                 break;
3103             }
3104         }
3105
3106         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3107             command_queue_ridx = 0;
3108     }
3109
3110     close(masterfd);
3111
3112     tc_stat.queue_length = 0;
3113     mutex_unlock(&command_queue_mutex);
3114 }
3115
3116 static void run_command_queue(bool force)
3117 {
3118     if (COMMAND_QUEUE_IS_EMPTY)
3119         return;
3120
3121     if (force || command_queue_is_full())
3122         command_queue_sync_callback(NULL);
3123     else
3124         register_storage_idle_func(command_queue_sync_callback);
3125 }
3126
3127 static void queue_command(int cmd, long idx_id, int tag, long data)
3128 {
3129     while (1)
3130     {
3131         int next;
3132
3133         mutex_lock(&command_queue_mutex);
3134         next = command_queue_widx + 1;
3135         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3136             next = 0;
3137
3138         /* Make sure queue is not full. */
3139         if (next != command_queue_ridx)
3140         {
3141             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3142
3143             ce->command = cmd;
3144             ce->idx_id = idx_id;
3145             ce->tag = tag;
3146             ce->data = data;
3147
3148             command_queue_widx = next;
3149
3150             tc_stat.queue_length++;
3151
3152             mutex_unlock(&command_queue_mutex);
3153             break;
3154         }
3155
3156         /* Queue is full, try again later... */
3157         mutex_unlock(&command_queue_mutex);
3158         sleep(1);
3159     }
3160 }
3161
3162 long tagcache_increase_serial(void)
3163 {
3164     long old;
3165
3166     if (!tc_stat.ready)
3167         return -2;
3168
3169     while (read_lock)
3170         sleep(1);
3171
3172     old = current_tcmh.serial++;
3173     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3174
3175     return old;
3176 }
3177
3178 void tagcache_update_numeric(int idx_id, int tag, long data)
3179 {
3180     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3181 }
3182 #endif /* !__PCTOOL__ */
3183
3184 long tagcache_get_serial(void)
3185 {
3186     return current_tcmh.serial;
3187 }
3188
3189 long tagcache_get_commitid(void)
3190 {
3191     return current_tcmh.commitid;
3192 }
3193
3194 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3195 {
3196     char buf[512];
3197     int i;
3198
3199     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3200     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3201     {
3202         if (*datastr == '\0')
3203             break;
3204
3205         if (*datastr == '"' || *datastr == '\\')
3206             buf[i++] = '\\';
3207
3208         buf[i] = *(datastr++);
3209     }
3210
3211     strcpy(&buf[i], "\" ");
3212
3213     write(fd, buf, i + 2);
3214
3215     return true;
3216 }
3217
3218 #ifndef __PCTOOL__
3219
3220 static bool read_tag(char *dest, long size,
3221                      const char *src, const char *tagstr)
3222 {
3223     int pos;
3224     char current_tag[32];
3225
3226     while (*src != '\0')
3227     {
3228         /* Skip all whitespace */
3229         while (*src == ' ')
3230             src++;
3231
3232         if (*src == '\0')
3233             break;
3234
3235         pos = 0;
3236         /* Read in tag name */
3237         while (*src != '=' && *src != ' ')
3238         {
3239             current_tag[pos] = *src;
3240             src++;
3241             pos++;
3242
3243             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3244                 return false;
3245         }
3246         current_tag[pos] = '\0';
3247
3248         /* Read in tag data */
3249
3250         /* Find the start. */
3251         while (*src != '"' && *src != '\0')
3252             src++;
3253
3254         if (*src == '\0' || *(++src) == '\0')
3255             return false;
3256
3257         /* Read the data. */
3258         for (pos = 0; pos < size; pos++)
3259         {
3260             if (*src == '\0')
3261                 break;
3262
3263             if (*src == '\\')
3264             {
3265                 dest[pos] = *(src+1);
3266                 src += 2;
3267                 continue;
3268             }
3269
3270             dest[pos] = *src;
3271
3272             if (*src == '"')
3273             {
3274                 src++;
3275                 break;
3276             }
3277
3278             if (*src == '\0')
3279                 break;
3280
3281             src++;
3282         }
3283         dest[pos] = '\0';
3284
3285         if (!strcasecmp(tagstr, current_tag))
3286             return true;
3287     }
3288
3289     return false;
3290 }
3291
3292 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3293 {
3294     struct index_entry idx;
3295     char tag_data[TAG_MAXLEN+32];
3296     int idx_id;
3297     long masterfd = (long)parameters;
3298     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3299         tag_commitid };
3300     int i;
3301     (void)line_n;
3302
3303     if (*buf == '#')
3304         return 0;
3305
3306     logf("%d/%s", line_n, buf);
3307     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3308     {
3309         logf("filename missing");
3310         logf("-> %s", buf);
3311         return 0;
3312     }
3313
3314     idx_id = find_index(tag_data);
3315     if (idx_id < 0)
3316     {
3317         logf("entry not found");
3318         return 0;
3319     }
3320
3321     if (!get_index(masterfd, idx_id, &idx, false))
3322     {
3323         logf("failed to retrieve index entry");
3324         return 0;
3325     }
3326
3327     /* Stop if tag has already been modified. */
3328     if (idx.flag & FLAG_DIRTYNUM)
3329         return 0;
3330
3331     logf("import: %s", tag_data);
3332
3333     idx.flag |= FLAG_DIRTYNUM;
3334     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3335     {
3336         int data;
3337
3338         if (!read_tag(tag_data, sizeof tag_data, buf,
3339                       tagcache_tag_to_str(import_tags[i])))
3340         {
3341             continue;
3342         }
3343
3344         data = atoi(tag_data);
3345         if (data < 0)
3346             continue;
3347
3348         idx.tag_seek[import_tags[i]] = data;
3349
3350         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3351             current_tcmh.serial = data + 1;
3352         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3353             current_tcmh.commitid = data + 1;
3354     }
3355
3356     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3357 }
3358
3359 bool tagcache_import_changelog(void)
3360 {
3361     struct master_header myhdr;
3362     struct tagcache_header tch;
3363     int clfd;
3364     long masterfd;
3365     char buf[2048];
3366
3367     if (!tc_stat.ready)
3368         return false;
3369
3370     while (read_lock)
3371         sleep(1);
3372
3373     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3374     if (clfd < 0)
3375     {
3376         logf("failure to open changelog");
3377         return false;
3378     }
3379
3380     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3381     {
3382         close(clfd);
3383         return false;
3384     }
3385
3386     write_lock++;
3387
3388     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3389
3390     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3391                   parse_changelog_line);
3392
3393     close(clfd);
3394     close(masterfd);
3395
3396     if (filenametag_fd >= 0)
3397     {
3398         close(filenametag_fd);
3399         filenametag_fd = -1;
3400     }
3401
3402     write_lock--;
3403
3404     update_master_header();
3405
3406     return true;
3407 }
3408
3409 #endif /* !__PCTOOL__ */
3410
3411 bool tagcache_create_changelog(struct tagcache_search *tcs)
3412 {
3413     struct master_header myhdr;
3414     struct index_entry idx;
3415     char buf[TAG_MAXLEN+32];
3416     char temp[32];
3417     int clfd;
3418     int i, j;
3419
3420     if (!tc_stat.ready)
3421         return false;
3422
3423     if (!tagcache_search(tcs, tag_filename))
3424         return false;
3425
3426     /* Initialize the changelog */
3427     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3428     if (clfd < 0)
3429     {
3430         logf("failure to open changelog");
3431         return false;
3432     }
3433
3434     if (tcs->masterfd < 0)
3435     {
3436         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3437             return false;
3438     }
3439     else
3440     {
3441         lseek(tcs->masterfd, 0, SEEK_SET);
3442         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3443     }
3444
3445     write(clfd, "## Changelog version 1\n", 23);
3446
3447     for (i = 0; i < myhdr.tch.entry_count; i++)
3448     {
3449         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3450             != sizeof(struct index_entry))
3451         {
3452             logf("read error #9");
3453             tagcache_search_finish(tcs);
3454             close(clfd);
3455             return false;
3456         }
3457
3458         /* Skip until the entry found has been modified. */
3459         if (! (idx.flag & FLAG_DIRTYNUM) )
3460             continue;
3461
3462         /* Skip deleted entries too. */
3463         if (idx.flag & FLAG_DELETED)
3464             continue;
3465
3466         /* Now retrieve all tags. */
3467         for (j = 0; j < TAG_COUNT; j++)
3468         {
3469             if (TAGCACHE_IS_NUMERIC(j))
3470             {
3471                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3472                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3473                 continue;
3474             }
3475
3476             tcs->type = j;
3477             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3478             write_tag(clfd, tagcache_tag_to_str(j), buf);
3479         }
3480
3481         write(clfd, "\n", 1);
3482         do_timed_yield();
3483     }
3484
3485     close(clfd);
3486
3487     tagcache_search_finish(tcs);
3488
3489     return true;
3490 }
3491
3492 static bool delete_entry(long idx_id)
3493 {
3494     int fd = -1;
3495     int masterfd = -1;
3496     int tag, i;
3497     struct index_entry idx, myidx;
3498     struct master_header myhdr;
3499     char buf[TAG_MAXLEN+32];
3500     int in_use[TAG_COUNT];
3501
3502     logf("delete_entry(): %ld", idx_id);
3503
3504 #ifdef HAVE_TC_RAMCACHE
3505     /* At first mark the entry removed from ram cache. */
3506     if (tc_stat.ramcache)
3507         hdr->indices[idx_id].flag |= FLAG_DELETED;
3508 #endif
3509
3510     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3511         return false;
3512
3513     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3514     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3515         != sizeof(struct index_entry))
3516     {
3517         logf("delete_entry(): read error");
3518         goto cleanup;
3519     }
3520
3521     if (myidx.flag & FLAG_DELETED)
3522     {
3523         logf("delete_entry(): already deleted!");
3524         goto cleanup;
3525     }
3526
3527     myidx.flag |= FLAG_DELETED;
3528     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3529     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3530         != sizeof(struct index_entry))
3531     {
3532         logf("delete_entry(): write_error #1");
3533         goto cleanup;
3534     }
3535
3536     /* Now check which tags are no longer in use (if any) */
3537     for (tag = 0; tag < TAG_COUNT; tag++)
3538         in_use[tag] = 0;
3539
3540     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3541     for (i = 0; i < myhdr.tch.entry_count; i++)
3542     {
3543         struct index_entry *idxp;
3544
3545 #ifdef HAVE_TC_RAMCACHE
3546         /* Use RAM DB if available for greater speed */
3547         if (tc_stat.ramcache)
3548             idxp = &hdr->indices[i];
3549         else
3550 #endif
3551         {
3552             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3553                 != sizeof(struct index_entry))
3554             {
3555                 logf("delete_entry(): read error #2");
3556                 goto cleanup;
3557             }
3558             idxp = &idx;
3559         }
3560
3561         if (idxp->flag & FLAG_DELETED)
3562             continue;
3563
3564         for (tag = 0; tag < TAG_COUNT; tag++)
3565         {
3566             if (TAGCACHE_IS_NUMERIC(tag))
3567                 continue;
3568
3569             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3570                 in_use[tag]++;
3571         }
3572     }
3573
3574     /* Now delete all tags no longer in use. */
3575     for (tag = 0; tag < TAG_COUNT; tag++)
3576     {
3577         struct tagcache_header tch;
3578         int oldseek = myidx.tag_seek[tag];
3579
3580         if (TAGCACHE_IS_NUMERIC(tag))
3581             continue;
3582
3583         /**
3584          * Replace tag seek with a hash value of the field string data.
3585          * That way runtime statistics of moved or altered files can be
3586          * resurrected.
3587          */
3588 #ifdef HAVE_TC_RAMCACHE
3589         if (tc_stat.ramcache && tag != tag_filename)
3590         {
3591             struct tagfile_entry *tfe;
3592             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3593
3594             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3595             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3596             myidx.tag_seek[tag] = *seek;
3597         }
3598         else
3599 #endif
3600         {
3601             struct tagfile_entry tfe;
3602
3603             /* Open the index file, which contains the tag names. */
3604             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3605                 goto cleanup;
3606
3607             /* Skip the header block */
3608             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3609             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3610                 != sizeof(struct tagfile_entry))
3611             {
3612                 logf("delete_entry(): read error #3");
3613                 goto cleanup;
3614             }
3615
3616             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3617             {
3618                 logf("delete_entry(): read error #4");
3619                 goto cleanup;
3620             }
3621
3622             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3623         }
3624
3625         if (in_use[tag])
3626         {
3627             logf("in use: %d/%d", tag, in_use[tag]);
3628             if (fd >= 0)
3629             {
3630                 close(fd);
3631                 fd = -1;
3632             }
3633             continue;
3634         }
3635
3636 #ifdef HAVE_TC_RAMCACHE
3637         /* Delete from ram. */
3638         if (tc_stat.ramcache && tag != tag_filename)
3639         {
3640             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3641             tagentry->tag_data[0] = '\0';
3642         }
3643 #endif
3644
3645         /* Open the index file, which contains the tag names. */
3646         if (fd < 0)
3647         {
3648             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3649                 goto cleanup;
3650         }
3651
3652         /* Skip the header block */
3653         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3654
3655         /* Debug, print 10 first characters of the tag
3656         read(fd, buf, 10);
3657         buf[10]='\0';
3658         logf("TAG:%s", buf);
3659         lseek(fd, -10, SEEK_CUR);
3660         */
3661
3662         /* Write first data byte in tag as \0 */
3663         write(fd, "", 1);
3664
3665         /* Now tag data has been removed */
3666         close(fd);
3667         fd = -1;
3668     }
3669
3670     /* Write index entry back into master index. */
3671     lseek(masterfd, sizeof(struct master_header) +
3672           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3673     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3674         != sizeof(struct index_entry))
3675     {
3676         logf("delete_entry(): write_error #2");
3677         goto cleanup;
3678     }
3679
3680     close(masterfd);
3681
3682     return true;
3683
3684     cleanup:
3685     if (fd >= 0)
3686         close(fd);
3687     if (masterfd >= 0)
3688         close(masterfd);
3689
3690     return false;
3691 }
3692
3693 #ifndef __PCTOOL__
3694 /**
3695  * Returns true if there is an event waiting in the queue
3696  * that requires the current operation to be aborted.
3697  */
3698 static bool check_event_queue(void)
3699 {
3700     struct queue_event ev;
3701
3702     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3703     switch (ev.id)
3704     {
3705         case Q_STOP_SCAN:
3706         case SYS_POWEROFF:
3707         case SYS_USB_CONNECTED:
3708             /* Put the event back into the queue. */
3709             queue_post(&tagcache_queue, ev.id, ev.data);
3710             return true;
3711     }
3712
3713     return false;
3714 }
3715 #endif
3716
3717 #ifdef HAVE_TC_RAMCACHE
3718 static bool allocate_tagcache(void)
3719 {
3720     struct master_header tcmh;
3721     int fd;
3722
3723     /* Load the header. */
3724     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3725     {
3726         hdr = NULL;
3727         return false;
3728     }
3729
3730     close(fd);
3731
3732     /**
3733      * Now calculate the required cache size plus
3734      * some extra space for alignment fixes.
3735      */
3736     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3737         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3738     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3739     memset(hdr, 0, sizeof(struct ramcache_header));
3740     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3741     hdr->indices = (struct index_entry *)(hdr + 1);
3742     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3743
3744     return true;
3745 }
3746
3747 # ifdef HAVE_EEPROM_SETTINGS
3748 static bool tagcache_dumpload(void)
3749 {
3750     struct statefile_header shdr;
3751     int fd, rc;
3752     long offpos;
3753     int i;
3754
3755     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3756     if (fd < 0)
3757     {
3758         logf("no tagcache statedump");
3759         return false;
3760     }
3761
3762     /* Check the statefile memory placement */
3763     hdr = buffer_alloc(0);
3764     rc = read(fd, &shdr, sizeof(struct statefile_header));
3765     if (rc != sizeof(struct statefile_header)
3766         /* || (long)hdr != (long)shdr.hdr */)
3767     {
3768         logf("incorrect statefile");
3769         hdr = NULL;
3770         close(fd);
3771         return false;
3772     }
3773
3774     offpos = (long)hdr - (long)shdr.hdr;
3775
3776     /* Lets allocate real memory and load it */
3777     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3778     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3779     close(fd);
3780
3781     if (rc != shdr.tc_stat.ramcache_allocated)
3782     {
3783         logf("read failure!");
3784         hdr = NULL;
3785         return false;
3786     }
3787
3788     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3789
3790     /* Now fix the pointers */
3791     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3792     for (i = 0; i < TAG_COUNT; i++)
3793         hdr->tags[i] += offpos;
3794
3795     return true;
3796 }
3797
3798 static bool tagcache_dumpsave(void)
3799 {
3800     struct statefile_header shdr;
3801     int fd;
3802
3803     if (!tc_stat.ramcache)
3804         return false;
3805
3806     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3807     if (fd < 0)
3808     {
3809         logf("failed to create a statedump");
3810         return false;
3811     }
3812
3813     /* Create the header */
3814     shdr.hdr = hdr;
3815     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3816     write(fd, &shdr, sizeof(struct statefile_header));
3817
3818     /* And dump the data too */
3819     write(fd, hdr, tc_stat.ramcache_allocated);
3820     close(fd);
3821
3822     return true;
3823 }
3824 # endif
3825
3826 static bool load_tagcache(void)
3827 {
3828     struct tagcache_header *tch;
3829     long bytesleft = tc_stat.ramcache_allocated;
3830     struct index_entry *idx;
3831     int rc, fd;
3832     char *p;
3833     int i, tag;
3834
3835 # ifdef HAVE_DIRCACHE
3836     while (dircache_is_initializing())
3837         sleep(1);
3838
3839     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3840 # endif
3841
3842     logf("loading tagcache to ram...");
3843
3844     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3845     if (fd < 0)
3846     {
3847         logf("tagcache open failed");
3848         return false;
3849     }
3850
3851     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3852         != sizeof(struct master_header)
3853         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3854     {
3855         logf("incorrect header");
3856         return false;
3857     }
3858
3859     idx = hdr->indices;
3860
3861     /* Load the master index table. */
3862     for (i = 0; i < hdr->h.tch.entry_count; i++)
3863     {
3864         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3865         if (rc != sizeof(struct index_entry))
3866         {
3867             logf("read error #10");
3868             close(fd);
3869             return false;
3870         }
3871
3872         bytesleft -= sizeof(struct index_entry);
3873         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3874         {
3875             logf("too big tagcache.");
3876             close(fd);
3877             return false;
3878         }
3879
3880         idx++;
3881     }
3882
3883     close(fd);
3884
3885     /* Load the tags. */
3886     p = (char *)idx;
3887     for (tag = 0; tag < TAG_COUNT; tag++)
3888     {
3889         struct tagfile_entry *fe;
3890         char buf[TAG_MAXLEN+32];
3891
3892         if (TAGCACHE_IS_NUMERIC(tag))
3893             continue ;
3894
3895         //p = ((void *)p+1);
3896         p = (char *)((long)p & ~0x03) + 0x04;
3897         hdr->tags[tag] = p;
3898
3899         /* Check the header. */
3900         tch = (struct tagcache_header *)p;
3901         p += sizeof(struct tagcache_header);
3902
3903         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3904             return false;
3905
3906         for (hdr->entry_count[tag] = 0;
3907              hdr->entry_count[tag] < tch->entry_count;
3908              hdr->entry_count[tag]++)
3909         {
3910             long pos;
3911
3912             if (do_timed_yield())
3913             {
3914                 /* Abort if we got a critical event in queue */
3915                 if (check_event_queue())
3916                     return false;
3917             }
3918
3919             fe = (struct tagfile_entry *)p;
3920             pos = lseek(fd, 0, SEEK_CUR);
3921             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3922             if (rc != sizeof(struct tagfile_entry))
3923             {
3924                 /* End of lookup table. */
3925                 logf("read error #11");
3926                 close(fd);
3927                 return false;
3928             }
3929
3930             /* We have a special handling for the filename tags. */
3931             if (tag == tag_filename)
3932             {
3933 # ifdef HAVE_DIRCACHE
3934                 const struct dirent *dc;
3935 # endif
3936
3937                 // FIXME: This is wrong!
3938                 // idx = &hdr->indices[hdr->entry_count[i]];
3939                 idx = &hdr->indices[fe->idx_id];
3940
3941                 if (fe->tag_length >= (long)sizeof(buf)-1)
3942                 {
3943                     read(fd, buf, 10);
3944                     buf[10] = '\0';
3945                     logf("TAG:%s", buf);
3946                     logf("too long filename");
3947                     close(fd);
3948                     return false;
3949                 }
3950
3951                 rc = read(fd, buf, fe->tag_length);
3952                 if (rc != fe->tag_length)
3953                 {
3954                     logf("read error #12");
3955                     close(fd);
3956                     return false;
3957                 }
3958
3959                 /* Check if the entry has already been removed */
3960                 if (idx->flag & FLAG_DELETED)
3961                     continue;
3962
3963                 /* This flag must not be used yet. */
3964                 if (idx->flag & FLAG_DIRCACHE)
3965                 {
3966                     logf("internal error!");
3967                     close(fd);
3968                     return false;
3969                 }
3970
3971                 if (idx->tag_seek[tag] != pos)
3972                 {
3973                     logf("corrupt data structures!");
3974                     close(fd);
3975                     return false;
3976                 }
3977
3978 # ifdef HAVE_DIRCACHE
3979                 if (dircache_is_enabled())
3980                 {
3981                     dc = dircache_get_entry_ptr(buf);
3982                     if (dc == NULL)
3983                     {
3984                         logf("Entry no longer valid.");
3985                         logf("-> %s", buf);
3986                         if (global_settings.tagcache_autoupdate)
3987                             delete_entry(fe->idx_id);
3988                         continue ;
3989                     }
3990
3991                     idx->flag |= FLAG_DIRCACHE;
3992                     idx->tag_seek[tag_filename] = (long)dc;
3993                 }
3994                 else
3995 # endif
3996                 {
3997                     /* This will be very slow unless dircache is enabled
3998                        or target is flash based, but do it anyway for
3999                        consistency. */
4000                     /* Check if entry has been removed. */
4001                     if (global_settings.tagcache_autoupdate)
4002                     {
4003                         if (!file_exists(buf))
4004                         {
4005                             logf("Entry no longer valid.");
4006                             logf("-> %s", buf);
4007                             delete_entry(fe->idx_id);
4008                             continue;
4009                         }
4010                     }
4011                 }
4012
4013                 continue ;
4014             }
4015
4016             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4017             if (bytesleft < 0)
4018             {
4019                 logf("too big tagcache #2");
4020                 logf("tl: %d", fe->tag_length);
4021                 logf("bl: %ld", bytesleft);
4022                 close(fd);
4023                 return false;
4024             }
4025
4026             p = fe->tag_data;
4027             rc = read(fd, fe->tag_data, fe->tag_length);
4028             p += rc;
4029
4030             if (rc != fe->tag_length)
4031             {
4032                 logf("read error #13");
4033                 logf("rc=0x%04x", rc); // 0x431
4034                 logf("len=0x%04x", fe->tag_length); // 0x4000
4035                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4036                 logf("tag=0x%02x", tag); // 0x00
4037                 close(fd);
4038                 return false;
4039             }
4040         }
4041         close(fd);
4042     }
4043
4044     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4045     logf("tagcache loaded into ram!");
4046
4047     return true;
4048 }
4049 #endif /* HAVE_TC_RAMCACHE */
4050
4051 static bool check_deleted_files(void)
4052 {
4053     int fd;
4054     char buf[TAG_MAXLEN+32];
4055     struct tagfile_entry tfe;
4056
4057     logf("reverse scan...");
4058     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4059     fd = open(buf, O_RDONLY);
4060
4061     if (fd < 0)
4062     {
4063         logf("%s open fail", buf);
4064         return false;
4065     }
4066
4067     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4068     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4069            == sizeof(struct tagfile_entry)
4070 #ifndef __PCTOOL__
4071            && !check_event_queue()
4072 #endif
4073            )
4074     {
4075         if (tfe.tag_length >= (long)sizeof(buf)-1)
4076         {
4077             logf("too long tag");
4078             close(fd);
4079             return false;
4080         }
4081
4082         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4083         {
4084             logf("read error #14");
4085             close(fd);
4086             return false;
4087         }
4088
4089         /* Check if the file has already deleted from the db. */
4090         if (*buf == '\0')
4091             continue;
4092
4093         /* Now check if the file exists. */
4094         if (!file_exists(buf))
4095         {
4096             logf("Entry no longer valid.");
4097             logf("-> %s / %d", buf, tfe.tag_length);
4098             delete_entry(tfe.idx_id);
4099         }
4100     }
4101
4102     close(fd);
4103
4104     logf("done");
4105
4106     return true;
4107 }
4108
4109
4110 /* Note that this function must not be inlined, otherwise the whole point
4111  * of having the code in a separate function is lost.
4112  */
4113 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4114     int *ignore, int *unignore)
4115 {
4116     char newpath[MAX_PATH];
4117
4118     /* check for a database.ignore file */
4119     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4120     *ignore = file_exists(newpath);
4121     /* check for a database.unignore file */
4122     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4123     *unignore = file_exists(newpath);
4124 }
4125
4126
4127 static bool check_dir(const char *dirname, int add_files)
4128 {
4129     DIR *dir;
4130     int len;
4131     int success = false;
4132     int ignore, unignore;
4133
4134     dir = opendir(dirname);
4135     if (!dir)
4136     {
4137         logf("tagcache: opendir() failed");
4138         return false;
4139     }
4140
4141     /* check for a database.ignore and database.unignore */
4142     check_ignore(dirname, &ignore, &unignore);
4143
4144     /* don't do anything if both ignore and unignore are there */
4145     if (ignore != unignore)
4146         add_files = unignore;
4147
4148     /* Recursively scan the dir. */
4149 #ifdef __PCTOOL__
4150     while (1)
4151 #else
4152     while (!check_event_queue())
4153 #endif
4154     {
4155         struct dirent *entry;
4156
4157         entry = readdir(dir);
4158
4159         if (entry == NULL)
4160         {
4161             success = true;
4162             break ;
4163         }
4164
4165         if (!strcmp((char *)entry->d_name, ".") ||
4166             !strcmp((char *)entry->d_name, ".."))
4167             continue;
4168
4169         yield();
4170
4171         len = strlen(curpath);
4172         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4173                  entry->d_name);
4174
4175         processed_dir_count++;
4176         if (entry->attribute & ATTR_DIRECTORY)
4177             check_dir(curpath, add_files);
4178         else if (add_files)
4179         {
4180             tc_stat.curentry = curpath;
4181
4182             /* Add a new entry to the temporary db file. */
4183             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4184 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4185                          , dir->internal_entry
4186 #endif
4187                          );
4188
4189             /* Wait until current path for debug screen is read and unset. */
4190             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4191                 yield();
4192
4193             tc_stat.curentry = NULL;
4194         }
4195
4196         curpath[len] = '\0';
4197     }
4198
4199     closedir(dir);
4200
4201     return success;
4202 }
4203
4204 void tagcache_screensync_event(void)
4205 {
4206     tc_stat.curentry = NULL;
4207 }
4208
4209 void tagcache_screensync_enable(bool state)
4210 {
4211     tc_stat.syncscreen = state;
4212 }
4213
4214 void tagcache_build(const char *path)
4215 {
4216     struct tagcache_header header;
4217     bool ret;
4218
4219     curpath[0] = '\0';
4220     data_size = 0;
4221     total_entry_count = 0;
4222     processed_dir_count = 0;
4223
4224 #ifdef HAVE_DIRCACHE
4225     while (dircache_is_initializing())
4226         sleep(1);
4227 #endif
4228
4229     logf("updating tagcache");
4230
4231     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4232     if (cachefd >= 0)
4233     {
4234         logf("skipping, cache already waiting for commit");
4235         close(cachefd);
4236         return ;
4237     }
4238
4239     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4240     if (cachefd < 0)
4241     {
4242         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4243         return ;
4244     }
4245
4246     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4247
4248     cpu_boost(true);
4249
4250     logf("Scanning files...");
4251     /* Scan for new files. */
4252     memset(&header, 0, sizeof(struct tagcache_header));
4253     write(cachefd, &header, sizeof(struct tagcache_header));
4254
4255     if (strcmp("/", path) != 0)
4256         strcpy(curpath, path);
4257     ret = check_dir(path, true);
4258
4259     /* Write the header. */
4260     header.magic = TAGCACHE_MAGIC;
4261     header.datasize = data_size;
4262     header.entry_count = total_entry_count;
4263     lseek(cachefd, 0, SEEK_SET);
4264     write(cachefd, &header, sizeof(struct tagcache_header));
4265     close(cachefd);
4266
4267     if (filenametag_fd >= 0)
4268     {
4269         close(filenametag_fd);
4270         filenametag_fd = -1;
4271     }
4272
4273     if (!ret)
4274     {
4275         logf("Aborted.");
4276         cpu_boost(false);
4277         return ;
4278     }
4279
4280     /* Commit changes to the database. */
4281 #ifdef __PCTOOL__
4282     allocate_tempbuf();
4283 #endif
4284     if (commit())
4285     {
4286         remove(TAGCACHE_FILE_TEMP);
4287         logf("tagcache built!");
4288     }
4289 #ifdef __PCTOOL__
4290     free_tempbuf();
4291 #endif
4292
4293 #ifdef HAVE_TC_RAMCACHE
4294     if (hdr)
4295     {
4296         /* Import runtime statistics if we just initialized the db. */
4297         if (hdr->h.serial == 0)
4298             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4299     }
4300 #endif
4301
4302     cpu_boost(false);
4303 }
4304
4305 #ifdef HAVE_TC_RAMCACHE
4306 static void load_ramcache(void)
4307 {
4308     if (!hdr)
4309         return ;
4310
4311     cpu_boost(true);
4312
4313     /* At first we should load the cache (if exists). */
4314     tc_stat.ramcache = load_tagcache();
4315
4316     if (!tc_stat.ramcache)
4317     {
4318         /* If loading failed, it must indicate some problem with the db
4319          * so disable it entirely to prevent further issues. */
4320         tc_stat.ready = false;
4321         hdr = NULL;
4322     }
4323
4324     cpu_boost(false);
4325 }
4326
4327 void tagcache_unload_ramcache(void)
4328 {
4329     tc_stat.ramcache = false;
4330     /* Just to make sure there is no statefile present. */
4331     // remove(TAGCACHE_STATEFILE);
4332 }
4333 #endif
4334
4335 #ifndef __PCTOOL__
4336 static void tagcache_thread(void)
4337 {
4338     struct queue_event ev;
4339     bool check_done = false;
4340
4341     /* If the previous cache build/update was interrupted, commit
4342      * the changes first in foreground. */
4343     cpu_boost(true);
4344     allocate_tempbuf();
4345     commit();
4346     free_tempbuf();
4347
4348 #ifdef HAVE_TC_RAMCACHE
4349 # ifdef HAVE_EEPROM_SETTINGS
4350     if (firmware_settings.initialized && firmware_settings.disk_clean)
4351         check_done = tagcache_dumpload();
4352
4353     remove(TAGCACHE_STATEFILE);
4354 # endif
4355
4356     /* Allocate space for the tagcache if found on disk. */
4357     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4358         allocate_tagcache();
4359 #endif
4360
4361     cpu_boost(false);
4362     tc_stat.initialized = true;
4363
4364     /* Don't delay bootup with the header check but do it on background. */
4365     sleep(HZ);
4366     tc_stat.ready = check_all_headers();
4367     tc_stat.readyvalid = true;
4368
4369     while (1)
4370     {
4371         run_command_queue(false);
4372
4373         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4374
4375         switch (ev.id)
4376         {
4377             case Q_IMPORT_CHANGELOG:
4378                 tagcache_import_changelog();
4379                 break;
4380
4381             case Q_REBUILD:
4382                 remove_files();
4383                 remove(TAGCACHE_FILE_TEMP);
4384                 tagcache_build("/");
4385                 break;
4386
4387             case Q_UPDATE:
4388                 tagcache_build("/");
4389 #ifdef HAVE_TC_RAMCACHE
4390                 load_ramcache();
4391 #endif
4392                 check_deleted_files();
4393                 break ;
4394
4395             case Q_START_SCAN:
4396                 check_done = false;
4397             case SYS_TIMEOUT:
4398                 if (check_done || !tc_stat.ready)
4399                     break ;
4400
4401 #ifdef HAVE_TC_RAMCACHE
4402                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4403                 {
4404                     load_ramcache();
4405                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4406                         tagcache_build("/");
4407                 }
4408                 else
4409 #endif
4410                 if (global_settings.tagcache_autoupdate)
4411                 {
4412                     tagcache_build("/");
4413
4414                     /* This will be very slow unless dircache is enabled
4415                        or target is flash based, but do it anyway for
4416                        consistency. */
4417                     check_deleted_files();
4418                 }
4419
4420                 logf("tagcache check done");
4421
4422                 check_done = true;
4423                 break ;
4424
4425             case Q_STOP_SCAN:
4426                 break ;
4427
4428             case SYS_POWEROFF:
4429                 break ;
4430
4431 #ifndef SIMULATOR
4432             case SYS_USB_CONNECTED:
4433                 logf("USB: TagCache");
4434                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4435                 usb_wait_for_disconnect(&tagcache_queue);
4436                 break ;
4437 #endif
4438         }
4439     }
4440 }
4441
4442 bool tagcache_prepare_shutdown(void)
4443 {
4444     if (tagcache_get_commit_step() > 0)
4445         return false;
4446
4447     tagcache_stop_scan();
4448     while (read_lock || write_lock)
4449         sleep(1);
4450
4451     return true;
4452 }
4453
4454 void tagcache_shutdown(void)
4455 {
4456     /* Flush the command queue. */
4457     run_command_queue(true);
4458
4459 #ifdef HAVE_EEPROM_SETTINGS
4460     if (tc_stat.ramcache)
4461         tagcache_dumpsave();
4462 #endif
4463 }
4464
4465 static int get_progress(void)
4466 {
4467     int total_count = -1;
4468
4469 #ifdef HAVE_DIRCACHE
4470     if (dircache_is_enabled())
4471     {
4472         total_count = dircache_get_entry_count();
4473     }
4474     else
4475 #endif
4476 #ifdef HAVE_TC_RAMCACHE
4477     {
4478         if (hdr && tc_stat.ramcache)
4479             total_count = hdr->h.tch.entry_count;
4480     }
4481 #endif
4482
4483     if (total_count < 0)
4484         return -1;
4485
4486     return processed_dir_count * 100 / total_count;
4487 }
4488
4489 struct tagcache_stat* tagcache_get_stat(void)
4490 {
4491     tc_stat.progress = get_progress();
4492     tc_stat.processed_entries = processed_dir_count;
4493
4494     return &tc_stat;
4495 }
4496
4497 void tagcache_start_scan(void)
4498 {
4499     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4500 }
4501
4502 bool tagcache_update(void)
4503 {
4504     if (!tc_stat.ready)
4505         return false;
4506
4507     queue_post(&tagcache_queue, Q_UPDATE, 0);
4508     return false;
4509 }
4510
4511 bool tagcache_rebuild()
4512 {
4513     queue_post(&tagcache_queue, Q_REBUILD, 0);
4514     return false;
4515 }
4516
4517 void tagcache_stop_scan(void)
4518 {
4519     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4520 }
4521
4522 #ifdef HAVE_TC_RAMCACHE
4523 bool tagcache_is_ramcache(void)
4524 {
4525     return tc_stat.ramcache;
4526 }
4527 #endif
4528
4529 #endif /* !__PCTOOL__ */
4530
4531
4532 void tagcache_init(void)
4533 {
4534     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4535     memset(&current_tcmh, 0, sizeof(struct master_header));
4536     filenametag_fd = -1;
4537     write_lock = read_lock = 0;
4538
4539 #ifndef __PCTOOL__
4540     mutex_init(&command_queue_mutex);
4541     queue_init(&tagcache_queue, true);
4542     create_thread(tagcache_thread, tagcache_stack,
4543                   sizeof(tagcache_stack), 0, tagcache_thread_name
4544                   IF_PRIO(, PRIORITY_BACKGROUND)
4545                           IF_COP(, CPU));
4546 #else
4547     tc_stat.initialized = true;
4548     allocate_tempbuf();
4549     commit();
4550     free_tempbuf();
4551     tc_stat.ready = check_all_headers();
4552 #endif
4553 }
4554
4555 #ifdef __PCTOOL__
4556 void tagcache_reverse_scan(void)
4557 {
4558     logf("Checking for deleted files");
4559     check_deleted_files();
4560 }
4561 #endif
4562
4563 bool tagcache_is_initialized(void)
4564 {
4565     return tc_stat.initialized;
4566 }
4567 bool tagcache_is_usable(void)
4568 {
4569     return tc_stat.initialized && tc_stat.ready;
4570 }
4571 int tagcache_get_commit_step(void)
4572 {
4573     return tc_stat.commit_step;
4574 }
4575 int tagcache_get_max_commit_step(void)
4576 {
4577     return (int)(SORTED_TAGS_COUNT)+1;
4578 }
4579