apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /* #define LOGF_ENABLE */
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "metadata.h"
  73 #include "tagcache.h"
  74 #include "buffer.h"
  75 #include "crc32.h"
  76 #include "misc.h"
  77 #include "settings.h"
  78 #include "dir.h"
  79 #include "structec.h"
  80 #include "tagcache.h"
  81
  82 #ifndef __PCTOOL__
  83 #include "splash.h"
  84 #include "lang.h"
  85 #include "eeprom_settings.h"
  86 #endif
  87
  88 #ifdef __PCTOOL__
  89 #define yield() do { } while(0)
  90 #define sim_sleep(timeout) do { } while(0)
  91 #define do_timed_yield() do { } while(0)
  92 #endif
  93
  94 #ifndef __PCTOOL__
  95 /* Tag Cache thread. */
  96 static struct event_queue tagcache_queue;
  97 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  98 static const char tagcache_thread_name[] = "tagcache";
  99 #endif
 100
 101 #define UNTAGGED "<Untagged>"
 102
 103 /* Previous path when scanning directory tree recursively. */
 104 static char curpath[TAG_MAXLEN+32];
 105
 106 /* Used when removing duplicates. */
 107 static char *tempbuf;     /* Allocated when needed. */
 108 static long tempbufidx;   /* Current location in buffer. */
 109 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 110 static long tempbuf_left; /* Buffer space left. */
 111 static long tempbuf_pos;
 112
 113 /* Tags we want to get sorted (loaded to the tempbuf). */
 114 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
 115     tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
 116
 117 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 118 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
 119     tag_composer, tag_comment, tag_albumartist, tag_grouping };
 120
 121 /* Numeric tags (we can use these tags with conditional clauses). */
 122 static const int numeric_tags[] = { tag_year, tag_discnumber,
 123     tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
 124     tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
 125     tag_virt_length_min, tag_virt_length_sec,
 126     tag_virt_playtime_min, tag_virt_playtime_sec,
 127     tag_virt_entryage, tag_virt_autoscore };
 128
 129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 130 static const char *tags_str[] = { "artist", "album", "genre", "title",
 131     "filename", "composer", "comment", "albumartist", "grouping", "year",
 132     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 133     "playtime", "lastplayed", "commitid", "mtime" };
 134
 135 /* Status information of the tagcache. */
 136 static struct tagcache_stat tc_stat;
 137
 138 /* Queue commands. */
 139 enum tagcache_queue {
 140     Q_STOP_SCAN = 0,
 141     Q_START_SCAN,
 142     Q_IMPORT_CHANGELOG,
 143     Q_UPDATE,
 144     Q_REBUILD,
 145
 146     /* Internal tagcache command queue. */
 147     CMD_UPDATE_MASTER_HEADER,
 148     CMD_UPDATE_NUMERIC,
 149 };
 150
 151 struct tagcache_command_entry {
 152     int32_t command;
 153     int32_t idx_id;
 154     int32_t tag;
 155     int32_t data;
 156 };
 157
 158 #ifndef __PCTOOL__
 159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 160 static volatile int command_queue_widx = 0;
 161 static volatile int command_queue_ridx = 0;
 162 static struct mutex command_queue_mutex;
 163 #endif
 164
 165 /* Tag database structures. */
 166
 167 /* Variable-length tag entry in tag files. */
 168 struct tagfile_entry {
 169     short tag_length;  /* Length of the data in bytes including '\0' */
 170     short idx_id;      /* Corresponding entry location in index file of not unique tags */
 171     char tag_data[0];  /* Begin of the tag data */
 172 };
 173
 174 /* Fixed-size tag entry in master db index. */
 175 struct index_entry {
 176     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 177     int32_t flag;                /* Status flags */
 178 };
 179
 180 /* Header is the same in every file. */
 181 struct tagcache_header {
 182     int32_t magic;       /* Header version number */
 183     int32_t datasize;    /* Data size in bytes */
 184     int32_t entry_count; /* Number of entries in this file */
 185 };
 186
 187 struct master_header {
 188     struct tagcache_header tch;
 189     int32_t serial; /* Increasing counting number */
 190     int32_t commitid; /* Number of commits so far */
 191     int32_t dirty;
 192 };
 193
 194 /* For the endianess correction */
 195 static const char *tagfile_entry_ec   = "ss";
 196 /**
 197  Note: This should be (1 + TAG_COUNT) amount of l's.
 198  */
 199 static const char *index_entry_ec     = "lllllllllllllllllllll";
 200
 201 static const char *tagcache_header_ec = "lll";
 202 static const char *master_header_ec   = "llllll";
 203
 204 static struct master_header current_tcmh;
 205
 206 #ifdef HAVE_TC_RAMCACHE
 207 /* Header is created when loading database to ram. */
 208 struct ramcache_header {
 209     struct master_header h;      /* Header from the master index */
 210     struct index_entry *indices; /* Master index file content */
 211     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 212     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 213 };
 214
 215 # ifdef HAVE_EEPROM_SETTINGS
 216 struct statefile_header {
 217     struct ramcache_header *hdr;
 218     struct tagcache_stat tc_stat;
 219 };
 220 # endif
 221
 222 /* Pointer to allocated ramcache_header */
 223 static struct ramcache_header *hdr;
 224 #endif
 225
 226 /**
 227  * Full tag entries stored in a temporary file waiting
 228  * for commit to the cache. */
 229 struct temp_file_entry {
 230     long tag_offset[TAG_COUNT];
 231     short tag_length[TAG_COUNT];
 232     long flag;
 233
 234     long data_length;
 235 };
 236
 237 struct tempbuf_id_list {
 238     long id;
 239     struct tempbuf_id_list *next;
 240 };
 241
 242 struct tempbuf_searchidx {
 243     long idx_id;
 244     char *str;
 245     int seek;
 246     struct tempbuf_id_list idlist;
 247 };
 248
 249 /* Lookup buffer for fixing messed up index while after sorting. */
 250 static long commit_entry_count;
 251 static long lookup_buffer_depth;
 252 static struct tempbuf_searchidx **lookup;
 253
 254 /* Used when building the temporary file. */
 255 static int cachefd = -1, filenametag_fd;
 256 static int total_entry_count = 0;
 257 static int data_size = 0;
 258 static int processed_dir_count;
 259
 260 /* Thread safe locking */
 261 static volatile int write_lock;
 262 static volatile int read_lock;
 263
 264 static bool delete_entry(long idx_id);
 265
 266 const char* tagcache_tag_to_str(int tag)
 267 {
 268     return tags_str[tag];
 269 }
 270
 271 bool tagcache_is_numeric_tag(int type)
 272 {
 273     int i;
 274
 275     for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
 276     {
 277         if (type == numeric_tags[i])
 278             return true;
 279     }
 280
 281     return false;
 282 }
 283
 284 bool tagcache_is_unique_tag(int type)
 285 {
 286     int i;
 287
 288     for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
 289     {
 290         if (type == unique_tags[i])
 291             return true;
 292     }
 293
 294     return false;
 295 }
 296
 297 bool tagcache_is_sorted_tag(int type)
 298 {
 299     int i;
 300
 301     for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
 302     {
 303         if (type == sorted_tags[i])
 304             return true;
 305     }
 306
 307     return false;
 308 }
 309
 310 #ifdef HAVE_DIRCACHE
 311 /**
 312  * Returns true if specified flag is still present, i.e., dircache
 313  * has not been reloaded.
 314  */
 315 static bool is_dircache_intact(void)
 316 {
 317     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 318 }
 319 #endif
 320
 321 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 322 {
 323     int fd;
 324     char buf[MAX_PATH];
 325     int rc;
 326
 327     if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
 328         return -1;
 329
 330     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 331
 332     fd = open(buf, write ? O_RDWR : O_RDONLY);
 333     if (fd < 0)
 334     {
 335         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 336         tc_stat.ready = false;
 337         return fd;
 338     }
 339
 340     /* Check the header. */
 341     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 342     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 343     {
 344         logf("header error");
 345         tc_stat.ready = false;
 346         close(fd);
 347         return -2;
 348     }
 349
 350     return fd;
 351 }
 352
 353 static int open_master_fd(struct master_header *hdr, bool write)
 354 {
 355     int fd;
 356     int rc;
 357
 358     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 359     if (fd < 0)
 360     {
 361         logf("master file open failed for R/W");
 362         tc_stat.ready = false;
 363         return fd;
 364     }
 365
 366     tc_stat.econ = false;
 367
 368     /* Check the header. */
 369     rc = read(fd, hdr, sizeof(struct master_header));
 370     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 371     {
 372         /* Success. */
 373         return fd;
 374     }
 375
 376     /* Trying to read again, this time with endianess correction enabled. */
 377     lseek(fd, 0, SEEK_SET);
 378
 379     rc = ecread(fd, hdr, 1, master_header_ec, true);
 380     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 381     {
 382         logf("header error");
 383         tc_stat.ready = false;
 384         close(fd);
 385         return -2;
 386     }
 387
 388     tc_stat.econ = true;
 389
 390     return fd;
 391 }
 392
 393 #ifndef __PCTOOL__
 394 static bool do_timed_yield(void)
 395 {
 396     /* Sorting can lock up for quite a while, so yield occasionally */
 397     static long wakeup_tick = 0;
 398     if (current_tick >= wakeup_tick)
 399     {
 400         wakeup_tick = current_tick + (HZ/4);
 401         yield();
 402         return true;
 403     }
 404     return false;
 405 }
 406 #endif
 407
 408 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 409 static long find_entry_ram(const char *filename,
 410                            const struct dirent *dc)
 411 {
 412     static long last_pos = 0;
 413     int i;
 414
 415     /* Check if we tagcache is loaded into ram. */
 416     if (!tc_stat.ramcache)
 417         return -1;
 418
 419     if (dc == NULL)
 420         dc = dircache_get_entry_ptr(filename);
 421
 422     if (dc == NULL)
 423     {
 424         logf("tagcache: file not found.");
 425         return -1;
 426     }
 427
 428     try_again:
 429
 430     if (last_pos > 0)
 431         i = last_pos;
 432     else
 433         i = 0;
 434
 435     for (; i < hdr->h.tch.entry_count; i++)
 436     {
 437         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 438         {
 439             last_pos = MAX(0, i - 3);
 440             return i;
 441         }
 442
 443         do_timed_yield();
 444     }
 445
 446     if (last_pos > 0)
 447     {
 448         last_pos = 0;
 449         goto try_again;
 450     }
 451
 452     return -1;
 453 }
 454 #endif
 455
 456 static long find_entry_disk(const char *filename)
 457 {
 458     struct tagcache_header tch;
 459     static long last_pos = -1;
 460     long pos_history[POS_HISTORY_COUNT];
 461     long pos_history_idx = 0;
 462     bool found = false;
 463     struct tagfile_entry tfe;
 464     int fd;
 465     char buf[TAG_MAXLEN+32];
 466     int i;
 467     int pos = -1;
 468
 469     if (!tc_stat.ready)
 470         return -2;
 471
 472     fd = filenametag_fd;
 473     if (fd < 0)
 474     {
 475         last_pos = -1;
 476         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 477             return -1;
 478     }
 479
 480     check_again:
 481
 482     if (last_pos > 0)
 483         lseek(fd, last_pos, SEEK_SET);
 484     else
 485         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 486
 487     while (true)
 488     {
 489         pos = lseek(fd, 0, SEEK_CUR);
 490         for (i = pos_history_idx-1; i >= 0; i--)
 491             pos_history[i+1] = pos_history[i];
 492         pos_history[0] = pos;
 493
 494         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 495             != sizeof(struct tagfile_entry))
 496         {
 497             break ;
 498         }
 499
 500         if (tfe.tag_length >= (long)sizeof(buf))
 501         {
 502             logf("too long tag #1");
 503             close(fd);
 504             last_pos = -1;
 505             return -2;
 506         }
 507
 508         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 509         {
 510             logf("read error #2");
 511             close(fd);
 512             last_pos = -1;
 513             return -3;
 514         }
 515
 516         if (!strcasecmp(filename, buf))
 517         {
 518             last_pos = pos_history[pos_history_idx];
 519             found = true;
 520             break ;
 521         }
 522
 523         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 524             pos_history_idx++;
 525     }
 526
 527     /* Not found? */
 528     if (!found)
 529     {
 530         if (last_pos > 0)
 531         {
 532             last_pos = -1;
 533             logf("seek again");
 534             goto check_again;
 535         }
 536
 537         if (fd != filenametag_fd)
 538             close(fd);
 539         return -4;
 540     }
 541
 542     if (fd != filenametag_fd)
 543         close(fd);
 544
 545     return tfe.idx_id;
 546 }
 547
 548 static int find_index(const char *filename)
 549 {
 550     long idx_id = -1;
 551
 552 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 553     if (tc_stat.ramcache && is_dircache_intact())
 554         idx_id = find_entry_ram(filename, NULL);
 555 #endif
 556
 557     if (idx_id < 0)
 558         idx_id = find_entry_disk(filename);
 559
 560     return idx_id;
 561 }
 562
 563 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 564 {
 565     int idx_id;
 566
 567     if (!tc_stat.ready)
 568         return false;
 569
 570     idx_id = find_index(filename);
 571     if (idx_id < 0)
 572         return false;
 573
 574     if (!tagcache_search(tcs, tag_filename))
 575         return false;
 576
 577     tcs->entry_count = 0;
 578     tcs->idx_id = idx_id;
 579
 580     return true;
 581 }
 582
 583 static bool get_index(int masterfd, int idxid,
 584                       struct index_entry *idx, bool use_ram)
 585 {
 586     bool localfd = false;
 587
 588     if (idxid < 0)
 589     {
 590         logf("Incorrect idxid: %d", idxid);
 591         return false;
 592     }
 593
 594 #ifdef HAVE_TC_RAMCACHE
 595     if (tc_stat.ramcache && use_ram)
 596     {
 597         if (hdr->indices[idxid].flag & FLAG_DELETED)
 598             return false;
 599
 600         memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 601         return true;
 602     }
 603 #else
 604     (void)use_ram;
 605 #endif
 606
 607     if (masterfd < 0)
 608     {
 609         struct master_header tcmh;
 610
 611         localfd = true;
 612         masterfd = open_master_fd(&tcmh, false);
 613         if (masterfd < 0)
 614             return false;
 615     }
 616
 617     lseek(masterfd, idxid * sizeof(struct index_entry)
 618           + sizeof(struct master_header), SEEK_SET);
 619     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 620         != sizeof(struct index_entry))
 621     {
 622         logf("read error #3");
 623         if (localfd)
 624             close(masterfd);
 625
 626         return false;
 627     }
 628
 629     if (localfd)
 630         close(masterfd);
 631
 632     if (idx->flag & FLAG_DELETED)
 633         return false;
 634
 635     return true;
 636 }
 637
 638 #ifndef __PCTOOL__
 639
 640 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 641 {
 642     /* We need to exclude all memory only flags & tags when writing to disk. */
 643     if (idx->flag & FLAG_DIRCACHE)
 644     {
 645         logf("memory only flags!");
 646         return false;
 647     }
 648
 649 #ifdef HAVE_TC_RAMCACHE
 650     /* Only update numeric data. Writing the whole index to RAM by memcpy
 651      * destroys dircache pointers!
 652      */
 653     if (tc_stat.ramcache)
 654     {
 655         int tag;
 656         struct index_entry *idx_ram = &hdr->indices[idxid];
 657
 658         for (tag = 0; tag < TAG_COUNT; tag++)
 659         {
 660             if (tagcache_is_numeric_tag(tag))
 661             {
 662                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 663             }
 664         }
 665
 666         /* Don't touch the dircache flag or attributes. */
 667         idx_ram->flag = (idx->flag & 0x0000ffff)
 668             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 669     }
 670 #endif
 671
 672     lseek(masterfd, idxid * sizeof(struct index_entry)
 673           + sizeof(struct master_header), SEEK_SET);
 674     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 675         != sizeof(struct index_entry))
 676     {
 677         logf("write error #3");
 678         logf("idxid: %d", idxid);
 679         return false;
 680     }
 681
 682     return true;
 683 }
 684
 685 #endif /* !__PCTOOL__ */
 686
 687 static bool open_files(struct tagcache_search *tcs, int tag)
 688 {
 689     if (tcs->idxfd[tag] < 0)
 690     {
 691         char fn[MAX_PATH];
 692
 693         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 694         tcs->idxfd[tag] = open(fn, O_RDONLY);
 695     }
 696
 697     if (tcs->idxfd[tag] < 0)
 698     {
 699         logf("File not open!");
 700         return false;
 701     }
 702
 703     return true;
 704 }
 705
 706 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 707                      int tag, char *buf, long size)
 708 {
 709     struct tagfile_entry tfe;
 710     long seek;
 711
 712     *buf = '\0';
 713
 714     if (tagcache_is_numeric_tag(tag))
 715         return false;
 716
 717     seek = idx->tag_seek[tag];
 718     if (seek < 0)
 719     {
 720         logf("Retrieve failed");
 721         return false;
 722     }
 723
 724 #ifdef HAVE_TC_RAMCACHE
 725     if (tcs->ramsearch)
 726     {
 727         struct tagfile_entry *ep;
 728
 729 # ifdef HAVE_DIRCACHE
 730         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 731             && is_dircache_intact())
 732         {
 733             dircache_copy_path((struct dirent *)seek,
 734                                buf, size);
 735             return true;
 736         }
 737         else
 738 # endif
 739         if (tag != tag_filename)
 740         {
 741             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 742             strncpy(buf, ep->tag_data, size-1);
 743
 744             return true;
 745         }
 746     }
 747 #endif
 748
 749     if (!open_files(tcs, tag))
 750         return false;
 751
 752     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 753     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 754         != sizeof(struct tagfile_entry))
 755     {
 756         logf("read error #5");
 757         return false;
 758     }
 759
 760     if (tfe.tag_length >= size)
 761     {
 762         logf("too small buffer");
 763         return false;
 764     }
 765
 766     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 767         tfe.tag_length)
 768     {
 769         logf("read error #6");
 770         return false;
 771     }
 772
 773     buf[tfe.tag_length] = '\0';
 774
 775     return true;
 776 }
 777
 778 static long check_virtual_tags(int tag, const struct index_entry *idx)
 779 {
 780     long data = 0;
 781
 782     switch (tag)
 783     {
 784         case tag_virt_length_sec:
 785             data = (idx->tag_seek[tag_length]/1000) % 60;
 786             break;
 787
 788         case tag_virt_length_min:
 789             data = (idx->tag_seek[tag_length]/1000) / 60;
 790             break;
 791
 792         case tag_virt_playtime_sec:
 793             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 794             break;
 795
 796         case tag_virt_playtime_min:
 797             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 798             break;
 799
 800         case tag_virt_autoscore:
 801             if (idx->tag_seek[tag_length] == 0
 802                 || idx->tag_seek[tag_playcount] == 0)
 803             {
 804                 data = 0;
 805             }
 806             else
 807             {
 808                 data = 100 * idx->tag_seek[tag_playtime]
 809                     / idx->tag_seek[tag_length]
 810                     / idx->tag_seek[tag_playcount];
 811             }
 812             break;
 813
 814         /* How many commits before the file has been added to the DB. */
 815         case tag_virt_entryage:
 816             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 817             break;
 818
 819         default:
 820             data = idx->tag_seek[tag];
 821     }
 822
 823     return data;
 824 }
 825
 826 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 827 {
 828     struct index_entry idx;
 829
 830     if (!tc_stat.ready)
 831         return false;
 832
 833     if (!tagcache_is_numeric_tag(tag))
 834         return -1;
 835
 836     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 837         return -2;
 838
 839     return check_virtual_tags(tag, &idx);
 840 }
 841
 842 inline static bool str_ends_with(const char *str1, const char *str2)
 843 {
 844     int str_len = strlen(str1);
 845     int clause_len = strlen(str2);
 846
 847     if (clause_len > str_len)
 848         return false;
 849
 850     return !strcasecmp(&str1[str_len - clause_len], str2);
 851 }
 852
 853 inline static bool str_oneof(const char *str, const char *list)
 854 {
 855     const char *sep;
 856     int l, len = strlen(str);
 857
 858     while (*list)
 859     {
 860         sep = strchr(list, '|');
 861         l = sep ? (long)sep - (long)list : (int)strlen(list);
 862         if ((l==len) && !strncasecmp(str, list, len))
 863             return true;
 864         list += sep ? l + 1 : l;
 865     }
 866
 867     return false;
 868 }
 869
 870 static bool check_against_clause(long numeric, const char *str,
 871                                  const struct tagcache_search_clause *clause)
 872 {
 873     if (clause->numeric)
 874     {
 875         switch (clause->type)
 876         {
 877             case clause_is:
 878                 return numeric == clause->numeric_data;
 879             case clause_is_not:
 880                 return numeric != clause->numeric_data;
 881             case clause_gt:
 882                 return numeric > clause->numeric_data;
 883             case clause_gteq:
 884                 return numeric >= clause->numeric_data;
 885             case clause_lt:
 886                 return numeric < clause->numeric_data;
 887             case clause_lteq:
 888                 return numeric <= clause->numeric_data;
 889             default:
 890                 logf("Incorrect numeric tag: %d", clause->type);
 891         }
 892     }
 893     else
 894     {
 895         switch (clause->type)
 896         {
 897             case clause_is:
 898                 return !strcasecmp(clause->str, str);
 899             case clause_is_not:
 900                 return strcasecmp(clause->str, str);
 901             case clause_gt:
 902                 return 0>strcasecmp(clause->str, str);
 903             case clause_gteq:
 904                 return 0>=strcasecmp(clause->str, str);
 905             case clause_lt:
 906                 return 0<strcasecmp(clause->str, str);
 907             case clause_lteq:
 908                 return 0<=strcasecmp(clause->str, str);
 909             case clause_contains:
 910                 return (strcasestr(str, clause->str) != NULL);
 911             case clause_not_contains:
 912                 return (strcasestr(str, clause->str) == NULL);
 913             case clause_begins_with:
 914                 return (strcasestr(str, clause->str) == str);
 915             case clause_not_begins_with:
 916                 return (strcasestr(str, clause->str) != str);
 917             case clause_ends_with:
 918                 return str_ends_with(str, clause->str);
 919             case clause_not_ends_with:
 920                 return !str_ends_with(str, clause->str);
 921             case clause_oneof:
 922                 return str_oneof(str, clause->str);
 923
 924             default:
 925                 logf("Incorrect tag: %d", clause->type);
 926         }
 927     }
 928
 929     return false;
 930 }
 931
 932 static bool check_clauses(struct tagcache_search *tcs,
 933                           struct index_entry *idx,
 934                           struct tagcache_search_clause **clause, int count)
 935 {
 936     int i;
 937
 938 #ifdef HAVE_TC_RAMCACHE
 939     if (tcs->ramsearch)
 940     {
 941         /* Go through all conditional clauses. */
 942         for (i = 0; i < count; i++)
 943         {
 944             struct tagfile_entry *tfe;
 945             int seek;
 946             char buf[256];
 947             char *str = NULL;
 948
 949             seek = check_virtual_tags(clause[i]->tag, idx);
 950
 951             if (!tagcache_is_numeric_tag(clause[i]->tag))
 952             {
 953                 if (clause[i]->tag == tag_filename)
 954                 {
 955                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 956                     str = buf;
 957                 }
 958                 else
 959                 {
 960                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 961                     str = tfe->tag_data;
 962                 }
 963             }
 964
 965             if (!check_against_clause(seek, str, clause[i]))
 966                 return false;
 967         }
 968     }
 969     else
 970 #endif
 971     {
 972         /* Check for conditions. */
 973         for (i = 0; i < count; i++)
 974         {
 975             struct tagfile_entry tfe;
 976             int seek;
 977             char str[256];
 978
 979             seek = check_virtual_tags(clause[i]->tag, idx);
 980
 981             memset(str, 0, sizeof str);
 982             if (!tagcache_is_numeric_tag(clause[i]->tag))
 983             {
 984                 int fd = tcs->idxfd[clause[i]->tag];
 985                 lseek(fd, seek, SEEK_SET);
 986                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 987                 if (tfe.tag_length >= (int)sizeof(str))
 988                 {
 989                     logf("Too long tag read!");
 990                     break ;
 991                 }
 992
 993                 read(fd, str, tfe.tag_length);
 994
 995                 /* Check if entry has been deleted. */
 996                 if (str[0] == '\0')
 997                     break;
 998             }
 999
1000             if (!check_against_clause(seek, str, clause[i]))
1001                 return false;
1002         }
1003     }
1004
1005     return true;
1006 }
1007
1008 bool tagcache_check_clauses(struct tagcache_search *tcs,
1009                             struct tagcache_search_clause **clause, int count)
1010 {
1011     struct index_entry idx;
1012
1013     if (count == 0)
1014         return true;
1015
1016     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1017         return false;
1018
1019     return check_clauses(tcs, &idx, clause, count);
1020 }
1021
1022 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1023 {
1024     int i;
1025
1026     /* If uniq buffer is not defined we must return true for search to work. */
1027     if (tcs->unique_list == NULL
1028         || (!tagcache_is_unique_tag(tcs->type)
1029             && !tagcache_is_numeric_tag(tcs->type)))
1030     {
1031         return true;
1032     }
1033
1034     for (i = 0; i < tcs->unique_list_count; i++)
1035     {
1036         /* Return false if entry is found. */
1037         if (tcs->unique_list[i] == id)
1038             return false;
1039     }
1040
1041     if (tcs->unique_list_count < tcs->unique_list_capacity)
1042     {
1043         tcs->unique_list[i] = id;
1044         tcs->unique_list_count++;
1045     }
1046
1047     return true;
1048 }
1049
1050 static bool build_lookup_list(struct tagcache_search *tcs)
1051 {
1052     struct index_entry entry;
1053     int i;
1054
1055     tcs->seek_list_count = 0;
1056
1057 #ifdef HAVE_TC_RAMCACHE
1058     if (tcs->ramsearch)
1059     {
1060         int j;
1061
1062         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1063         {
1064             struct index_entry *idx = &hdr->indices[i];
1065             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1066                 break ;
1067
1068             /* Skip deleted files. */
1069             if (idx->flag & FLAG_DELETED)
1070                 continue;
1071
1072             /* Go through all filters.. */
1073             for (j = 0; j < tcs->filter_count; j++)
1074             {
1075                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1076                 {
1077                     break ;
1078                 }
1079             }
1080
1081             if (j < tcs->filter_count)
1082                 continue ;
1083
1084             /* Check for conditions. */
1085             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1086                 continue;
1087
1088             /* Add to the seek list if not already in uniq buffer. */
1089             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1090                 continue;
1091
1092             /* Lets add it. */
1093             tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1094             tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1095             tcs->seek_list_count++;
1096         }
1097
1098         tcs->seek_pos = i;
1099
1100         return tcs->seek_list_count > 0;
1101     }
1102 #endif
1103
1104     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1105             sizeof(struct master_header), SEEK_SET);
1106
1107     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1108            == sizeof(struct index_entry))
1109     {
1110         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1111             break ;
1112
1113         tcs->seek_pos++;
1114
1115         /* Check if entry has been deleted. */
1116         if (entry.flag & FLAG_DELETED)
1117             continue;
1118
1119         /* Go through all filters.. */
1120         for (i = 0; i < tcs->filter_count; i++)
1121         {
1122             if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1123                 break ;
1124         }
1125
1126         if (i < tcs->filter_count)
1127             continue ;
1128
1129         /* Check for conditions. */
1130         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1131             continue;
1132
1133         /* Add to the seek list if not already in uniq buffer. */
1134         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1135             continue;
1136
1137         /* Lets add it. */
1138         tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1139         tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1140         tcs->seek_list_count++;
1141
1142         yield();
1143     }
1144
1145     return tcs->seek_list_count > 0;
1146 }
1147
1148
1149 static void remove_files(void)
1150 {
1151     int i;
1152     char buf[MAX_PATH];
1153
1154     tc_stat.ready = false;
1155     tc_stat.ramcache = false;
1156     tc_stat.econ = false;
1157     remove(TAGCACHE_FILE_MASTER);
1158     for (i = 0; i < TAG_COUNT; i++)
1159     {
1160         if (tagcache_is_numeric_tag(i))
1161             continue;
1162
1163         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1164         remove(buf);
1165     }
1166 }
1167
1168
1169 static bool check_all_headers(void)
1170 {
1171     struct master_header myhdr;
1172     struct tagcache_header tch;
1173     int tag;
1174     int fd;
1175
1176     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1177         return false;
1178
1179     close(fd);
1180     if (myhdr.dirty)
1181     {
1182         logf("tagcache is dirty!");
1183         return false;
1184     }
1185
1186     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1187
1188     for (tag = 0; tag < TAG_COUNT; tag++)
1189     {
1190         if (tagcache_is_numeric_tag(tag))
1191             continue;
1192
1193         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1194             return false;
1195
1196         close(fd);
1197     }
1198
1199     return true;
1200 }
1201
1202 bool tagcache_search(struct tagcache_search *tcs, int tag)
1203 {
1204     struct tagcache_header tag_hdr;
1205     struct master_header   master_hdr;
1206     int i;
1207
1208     if (tcs->initialized)
1209         tagcache_search_finish(tcs);
1210
1211     while (read_lock)
1212         sleep(1);
1213
1214     memset(tcs, 0, sizeof(struct tagcache_search));
1215     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1216         return false;
1217
1218     tcs->position = sizeof(struct tagcache_header);
1219     tcs->type = tag;
1220     tcs->seek_pos = 0;
1221     tcs->seek_list_count = 0;
1222     tcs->filter_count = 0;
1223     tcs->masterfd = -1;
1224
1225     for (i = 0; i < TAG_COUNT; i++)
1226         tcs->idxfd[i] = -1;
1227
1228 #ifndef HAVE_TC_RAMCACHE
1229     tcs->ramsearch = false;
1230 #else
1231     tcs->ramsearch = tc_stat.ramcache;
1232     if (tcs->ramsearch)
1233     {
1234         tcs->entry_count = hdr->entry_count[tcs->type];
1235     }
1236     else
1237 #endif
1238     {
1239         if (!tagcache_is_numeric_tag(tcs->type))
1240         {
1241             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1242             if (tcs->idxfd[tcs->type] < 0)
1243                 return false;
1244         }
1245
1246         /* Always open as R/W so we can pass tcs to functions that modify data also
1247          * without failing. */
1248         tcs->masterfd = open_master_fd(&master_hdr, true);
1249
1250         if (tcs->masterfd < 0)
1251             return false;
1252     }
1253
1254     tcs->valid = true;
1255     tcs->initialized = true;
1256     write_lock++;
1257
1258     return true;
1259 }
1260
1261 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1262                                  void *buffer, long length)
1263 {
1264     tcs->unique_list = (unsigned long *)buffer;
1265     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1266     tcs->unique_list_count = 0;
1267 }
1268
1269 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1270                                 int tag, int seek)
1271 {
1272     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1273         return false;
1274
1275     if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1276         return false;
1277
1278     tcs->filter_tag[tcs->filter_count] = tag;
1279     tcs->filter_seek[tcs->filter_count] = seek;
1280     tcs->filter_count++;
1281
1282     return true;
1283 }
1284
1285 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1286                                 struct tagcache_search_clause *clause)
1287 {
1288     int i;
1289
1290     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1291     {
1292         logf("Too many clauses");
1293         return false;
1294     }
1295
1296     /* Check if there is already a similar filter in present (filters are
1297      * much faster than clauses).
1298      */
1299     for (i = 0; i < tcs->filter_count; i++)
1300     {
1301         if (tcs->filter_tag[i] == clause->tag)
1302             return true;
1303     }
1304
1305     if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1306     {
1307         char buf[MAX_PATH];
1308
1309         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1310         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1311     }
1312
1313     tcs->clause[tcs->clause_count] = clause;
1314     tcs->clause_count++;
1315
1316     return true;
1317 }
1318
1319 /* TODO: Remove this mess. */
1320 #ifdef HAVE_DIRCACHE
1321 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1322     ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1323 #else
1324 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1325 #endif
1326
1327 static bool get_next(struct tagcache_search *tcs)
1328 {
1329     static char buf[TAG_MAXLEN+32];
1330     struct tagfile_entry entry;
1331     long flag = 0;
1332
1333     if (!tcs->valid || !tc_stat.ready)
1334         return false;
1335
1336     if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1337 #ifdef HAVE_TC_RAMCACHE
1338         && !tcs->ramsearch
1339 #endif
1340         )
1341         return false;
1342
1343     /* Relative fetch. */
1344     if (tcs->filter_count > 0 || tcs->clause_count > 0
1345         || tagcache_is_numeric_tag(tcs->type))
1346     {
1347         /* Check for end of list. */
1348         if (tcs->seek_list_count == 0)
1349         {
1350             /* Try to fetch more. */
1351             if (!build_lookup_list(tcs))
1352             {
1353                 tcs->valid = false;
1354                 return false;
1355             }
1356         }
1357
1358         tcs->seek_list_count--;
1359         flag = tcs->seek_flags[tcs->seek_list_count];
1360
1361         /* Seek stream to the correct position and continue to direct fetch. */
1362         if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1363             && !tagcache_is_numeric_tag(tcs->type))
1364         {
1365             if (!open_files(tcs, tcs->type))
1366                 return false;
1367
1368             lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1369         }
1370         else
1371             tcs->position = tcs->seek_list[tcs->seek_list_count];
1372     }
1373
1374     if (tagcache_is_numeric_tag(tcs->type))
1375     {
1376         snprintf(buf, sizeof(buf), "%d", tcs->position);
1377         tcs->result_seek = tcs->position;
1378         tcs->result = buf;
1379         tcs->result_len = strlen(buf) + 1;
1380         return true;
1381     }
1382
1383     /* Direct fetch. */
1384 #ifdef HAVE_TC_RAMCACHE
1385     if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1386     {
1387         struct tagfile_entry *ep;
1388
1389         if (tcs->entry_count == 0)
1390         {
1391             tcs->valid = false;
1392             return false;
1393         }
1394         tcs->entry_count--;
1395
1396         tcs->result_seek = tcs->position;
1397
1398 # ifdef HAVE_DIRCACHE
1399         if (tcs->type == tag_filename)
1400         {
1401             dircache_copy_path((struct dirent *)tcs->position,
1402                                buf, sizeof buf);
1403             tcs->result = buf;
1404             tcs->result_len = strlen(buf) + 1;
1405             tcs->idx_id = FLAG_GET_ATTR(flag);
1406             tcs->ramresult = false;
1407
1408             return true;
1409         }
1410 # endif
1411
1412         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1413         tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1414         tcs->result = ep->tag_data;
1415         tcs->result_len = strlen(tcs->result) + 1;
1416         tcs->idx_id = ep->idx_id;
1417         tcs->ramresult = true;
1418
1419         return true;
1420     }
1421     else
1422 #endif
1423     {
1424         if (!open_files(tcs, tcs->type))
1425             return false;
1426
1427         tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1428         if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1429                  tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1430         {
1431             /* End of data. */
1432             tcs->valid = false;
1433             return false;
1434         }
1435     }
1436
1437     if (entry.tag_length > (long)sizeof(buf))
1438     {
1439         tcs->valid = false;
1440         logf("too long tag #2");
1441         return false;
1442     }
1443
1444     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1445     {
1446         tcs->valid = false;
1447         logf("read error #4");
1448         return false;
1449     }
1450
1451     tcs->result = buf;
1452     tcs->result_len = strlen(tcs->result) + 1;
1453     tcs->idx_id = entry.idx_id;
1454     tcs->ramresult = false;
1455
1456     return true;
1457 }
1458
1459 bool tagcache_get_next(struct tagcache_search *tcs)
1460 {
1461     while (get_next(tcs))
1462     {
1463         if (tcs->result_len > 1)
1464             return true;
1465     }
1466
1467     return false;
1468 }
1469
1470 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1471                        int tag, char *buf, long size)
1472 {
1473     struct index_entry idx;
1474
1475     *buf = '\0';
1476     if (!get_index(tcs->masterfd, idxid, &idx, true))
1477         return false;
1478
1479     return retrieve(tcs, &idx, tag, buf, size);
1480 }
1481
1482 static bool update_master_header(void)
1483 {
1484     struct master_header myhdr;
1485     int fd;
1486
1487     if (!tc_stat.ready)
1488         return false;
1489
1490     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1491         return false;
1492
1493     myhdr.serial = current_tcmh.serial;
1494     myhdr.commitid = current_tcmh.commitid;
1495
1496     /* Write it back */
1497     lseek(fd, 0, SEEK_SET);
1498     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1499     close(fd);
1500
1501 #ifdef HAVE_TC_RAMCACHE
1502     if (hdr)
1503     {
1504         hdr->h.serial = current_tcmh.serial;
1505         hdr->h.commitid = current_tcmh.commitid;
1506     }
1507 #endif
1508
1509     return true;
1510 }
1511
1512 #if 0
1513
1514 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1515 {
1516     struct tagentry *entry;
1517
1518     if (tcs->type != tag_title)
1519         return ;
1520
1521     /* We will need reserve buffer for this. */
1522     if (tcs->ramcache)
1523     {
1524         struct tagfile_entry *ep;
1525
1526         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1527         tcs->seek_list[tcs->seek_list_count];
1528     }
1529
1530     entry = find_entry_ram();
1531
1532 }
1533 #endif
1534
1535 void tagcache_search_finish(struct tagcache_search *tcs)
1536 {
1537     int i;
1538
1539     if (!tcs->initialized)
1540         return;
1541
1542     if (tcs->masterfd >= 0)
1543     {
1544         close(tcs->masterfd);
1545         tcs->masterfd = -1;
1546     }
1547
1548     for (i = 0; i < TAG_COUNT; i++)
1549     {
1550         if (tcs->idxfd[i] >= 0)
1551         {
1552             close(tcs->idxfd[i]);
1553             tcs->idxfd[i] = -1;
1554         }
1555     }
1556
1557     tcs->ramsearch = false;
1558     tcs->valid = false;
1559     tcs->initialized = 0;
1560     if (write_lock > 0)
1561         write_lock--;
1562 }
1563
1564 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1565 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1566 {
1567     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1568 }
1569
1570 static long get_tag_numeric(const struct index_entry *entry, int tag)
1571 {
1572     return check_virtual_tags(tag, entry);
1573 }
1574
1575 static char* get_tag_string(const struct index_entry *entry, int tag)
1576 {
1577     char* s = get_tag(entry, tag)->tag_data;
1578     return strcmp(s, UNTAGGED) ? s : NULL;
1579 }
1580
1581 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1582 {
1583     struct index_entry *entry;
1584     int idx_id;
1585
1586     if (!tc_stat.ready)
1587         return false;
1588
1589     /* Find the corresponding entry in tagcache. */
1590     idx_id = find_entry_ram(filename, NULL);
1591     if (idx_id < 0 || !tc_stat.ramcache)
1592         return false;
1593
1594     entry = &hdr->indices[idx_id];
1595
1596     id3->title        = get_tag_string(entry, tag_title);
1597     id3->artist       = get_tag_string(entry, tag_artist);
1598     id3->album        = get_tag_string(entry, tag_album);
1599     id3->genre_string = get_tag_string(entry, tag_genre);
1600     id3->composer     = get_tag_string(entry, tag_composer);
1601     id3->comment      = get_tag_string(entry, tag_comment);
1602     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1603     id3->grouping     = get_tag_string(entry, tag_grouping);
1604
1605     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1606     id3->rating     = get_tag_numeric(entry, tag_rating);
1607     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1608     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1609     id3->year       = get_tag_numeric(entry, tag_year);
1610
1611     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1612     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1613     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1614     if (id3->bitrate == 0)
1615         id3->bitrate = 1;
1616
1617     return true;
1618 }
1619 #endif
1620
1621 static inline void write_item(const char *item)
1622 {
1623     int len = strlen(item) + 1;
1624
1625     data_size += len;
1626     write(cachefd, item, len);
1627 }
1628
1629 static int check_if_empty(char **tag)
1630 {
1631     int length;
1632
1633     if (*tag == NULL || **tag == '\0')
1634     {
1635         *tag = UNTAGGED;
1636         return sizeof(UNTAGGED); /* Tag length */
1637     }
1638
1639     length = strlen(*tag);
1640     if (length > TAG_MAXLEN)
1641     {
1642         logf("over length tag: %s", *tag);
1643         length = TAG_MAXLEN;
1644         (*tag)[length] = '\0';
1645     }
1646
1647     return length + 1;
1648 }
1649
1650 #define ADD_TAG(entry,tag,data) \
1651     /* Adding tag */ \
1652     entry.tag_offset[tag] = offset; \
1653     entry.tag_length[tag] = check_if_empty(data); \
1654     offset += entry.tag_length[tag]
1655
1656 static void add_tagcache(char *path, unsigned long mtime
1657 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1658                          ,const struct dirent *dc
1659 #endif
1660                          )
1661 {
1662     struct mp3entry id3;
1663     struct temp_file_entry entry;
1664     bool ret;
1665     int fd;
1666     int idx_id = -1;
1667     char tracknumfix[3];
1668     int offset = 0;
1669     int path_length = strlen(path);
1670     bool has_albumartist;
1671     bool has_grouping;
1672
1673     if (cachefd < 0)
1674         return ;
1675
1676     /* Check for overlength file path. */
1677     if (path_length > TAG_MAXLEN)
1678     {
1679         /* Path can't be shortened. */
1680         logf("Too long path: %s", path);
1681         return ;
1682     }
1683
1684     /* Check if the file is supported. */
1685     if (probe_file_format(path) == AFMT_UNKNOWN)
1686         return ;
1687
1688     /* Check if the file is already cached. */
1689 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1690     if (tc_stat.ramcache && is_dircache_intact())
1691     {
1692         idx_id = find_entry_ram(path, dc);
1693     }
1694     else
1695 #endif
1696     {
1697         if (filenametag_fd >= 0)
1698         {
1699             idx_id = find_entry_disk(path);
1700         }
1701     }
1702
1703     /* Check if file has been modified. */
1704     if (idx_id >= 0)
1705     {
1706         struct index_entry idx;
1707
1708         /* TODO: Mark that the index exists (for fast reverse scan) */
1709         //found_idx[idx_id/8] |= idx_id%8;
1710
1711         if (!get_index(-1, idx_id, &idx, true))
1712         {
1713             logf("failed to retrieve index entry");
1714             return ;
1715         }
1716
1717         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1718         {
1719             /* No changes to file. */
1720             return ;
1721         }
1722
1723         /* Metadata might have been changed. Delete the entry. */
1724         logf("Re-adding: %s", path);
1725         if (!delete_entry(idx_id))
1726         {
1727             logf("delete_entry failed: %d", idx_id);
1728             return ;
1729         }
1730     }
1731
1732     fd = open(path, O_RDONLY);
1733     if (fd < 0)
1734     {
1735         logf("open fail: %s", path);
1736         return ;
1737     }
1738
1739     memset(&id3, 0, sizeof(struct mp3entry));
1740     memset(&entry, 0, sizeof(struct temp_file_entry));
1741     memset(&tracknumfix, 0, sizeof(tracknumfix));
1742     ret = get_metadata(&id3, fd, path);
1743     close(fd);
1744
1745     if (!ret)
1746         return ;
1747
1748     logf("-> %s", path);
1749
1750     /* Generate track number if missing. */
1751     if (id3.tracknum <= 0)
1752     {
1753         const char *p = strrchr(path, '.');
1754
1755         if (p == NULL)
1756             p = &path[strlen(path)-1];
1757
1758         while (*p != '/')
1759         {
1760             if (isdigit(*p) && isdigit(*(p-1)))
1761             {
1762                 tracknumfix[1] = *p--;
1763                 tracknumfix[0] = *p;
1764                 break;
1765             }
1766             p--;
1767         }
1768
1769         if (tracknumfix[0] != '\0')
1770         {
1771             id3.tracknum = atoi(tracknumfix);
1772             /* Set a flag to indicate track number has been generated. */
1773             entry.flag |= FLAG_TRKNUMGEN;
1774         }
1775         else
1776         {
1777             /* Unable to generate track number. */
1778             id3.tracknum = -1;
1779         }
1780     }
1781
1782     /* Numeric tags */
1783     entry.tag_offset[tag_year] = id3.year;
1784     entry.tag_offset[tag_discnumber] = id3.discnum;
1785     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1786     entry.tag_offset[tag_length] = id3.length;
1787     entry.tag_offset[tag_bitrate] = id3.bitrate;
1788     entry.tag_offset[tag_mtime] = mtime;
1789
1790     /* String tags. */
1791     has_albumartist = id3.albumartist != NULL
1792         && strlen(id3.albumartist) > 0;
1793     has_grouping = id3.grouping != NULL
1794         && strlen(id3.grouping) > 0;
1795
1796     ADD_TAG(entry, tag_filename, &path);
1797     ADD_TAG(entry, tag_title, &id3.title);
1798     ADD_TAG(entry, tag_artist, &id3.artist);
1799     ADD_TAG(entry, tag_album, &id3.album);
1800     ADD_TAG(entry, tag_genre, &id3.genre_string);
1801     ADD_TAG(entry, tag_composer, &id3.composer);
1802     ADD_TAG(entry, tag_comment, &id3.comment);
1803     if (has_albumartist)
1804     {
1805         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1806     }
1807     else
1808     {
1809         ADD_TAG(entry, tag_albumartist, &id3.artist);
1810     }
1811     if (has_grouping)
1812     {
1813         ADD_TAG(entry, tag_grouping, &id3.grouping);
1814     }
1815     else
1816     {
1817         ADD_TAG(entry, tag_grouping, &id3.title);
1818     }
1819     entry.data_length = offset;
1820
1821     /* Write the header */
1822     write(cachefd, &entry, sizeof(struct temp_file_entry));
1823
1824     /* And tags also... Correct order is critical */
1825     write_item(path);
1826     write_item(id3.title);
1827     write_item(id3.artist);
1828     write_item(id3.album);
1829     write_item(id3.genre_string);
1830     write_item(id3.composer);
1831     write_item(id3.comment);
1832     if (has_albumartist)
1833     {
1834         write_item(id3.albumartist);
1835     }
1836     else
1837     {
1838         write_item(id3.artist);
1839     }
1840     if (has_grouping)
1841     {
1842         write_item(id3.grouping);
1843     }
1844     else
1845     {
1846         write_item(id3.title);
1847     }
1848     total_entry_count++;
1849 }
1850
1851 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1852 {
1853     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1854     int len = strlen(str)+1;
1855     int i;
1856     unsigned crc32;
1857     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1858     char buf[TAG_MAXLEN+32];
1859
1860     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1861         buf[i] = tolower(str[i]);
1862     buf[i] = '\0';
1863
1864     crc32 = crc_32(buf, i, 0xffffffff);
1865
1866     if (unique)
1867     {
1868         /* Check if the crc does not exist -> entry does not exist for sure. */
1869         for (i = 0; i < tempbufidx; i++)
1870         {
1871             if (crcbuf[-i] != crc32)
1872                 continue;
1873
1874             if (!strcasecmp(str, index[i].str))
1875             {
1876                 if (id < 0 || id >= lookup_buffer_depth)
1877                 {
1878                     logf("lookup buf overf.: %d", id);
1879                     return false;
1880                 }
1881
1882                 lookup[id] = &index[i];
1883                 return true;
1884             }
1885         }
1886     }
1887
1888     /* Insert to CRC buffer. */
1889     crcbuf[-tempbufidx] = crc32;
1890     tempbuf_left -= 4;
1891
1892     /* Insert it to the buffer. */
1893     tempbuf_left -= len;
1894     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1895         return false;
1896
1897     if (id >= lookup_buffer_depth)
1898     {
1899         logf("lookup buf overf. #2: %d", id);
1900         return false;
1901     }
1902
1903     if (id >= 0)
1904     {
1905         lookup[id] = &index[tempbufidx];
1906         index[tempbufidx].idlist.id = id;
1907     }
1908     else
1909         index[tempbufidx].idlist.id = -1;
1910
1911     index[tempbufidx].idlist.next = NULL;
1912     index[tempbufidx].idx_id = idx_id;
1913     index[tempbufidx].seek = -1;
1914     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1915     memcpy(index[tempbufidx].str, str, len);
1916     tempbuf_pos += len;
1917     tempbufidx++;
1918
1919     return true;
1920 }
1921
1922 static int compare(const void *p1, const void *p2)
1923 {
1924     do_timed_yield();
1925
1926     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1927     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1928
1929     if (strcmp(e1->str, UNTAGGED) == 0)
1930     {
1931         if (strcmp(e2->str, UNTAGGED) == 0)
1932             return 0;
1933         return -1;
1934     }
1935     else if (strcmp(e2->str, UNTAGGED) == 0)
1936         return 1;
1937
1938     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1939 }
1940
1941 static int tempbuf_sort(int fd)
1942 {
1943     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1944     struct tagfile_entry fe;
1945     int i;
1946     int length;
1947
1948     /* Generate reverse lookup entries. */
1949     for (i = 0; i < lookup_buffer_depth; i++)
1950     {
1951         struct tempbuf_id_list *idlist;
1952
1953         if (!lookup[i])
1954             continue;
1955
1956         if (lookup[i]->idlist.id == i)
1957             continue;
1958
1959         idlist = &lookup[i]->idlist;
1960         while (idlist->next != NULL)
1961             idlist = idlist->next;
1962
1963         tempbuf_left -= sizeof(struct tempbuf_id_list);
1964         if (tempbuf_left - 4 < 0)
1965             return -1;
1966
1967         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1968         if (tempbuf_pos & 0x03)
1969         {
1970             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1971             tempbuf_left -= 3;
1972             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1973         }
1974         tempbuf_pos += sizeof(struct tempbuf_id_list);
1975
1976         idlist = idlist->next;
1977         idlist->id = i;
1978         idlist->next = NULL;
1979
1980         do_timed_yield();
1981     }
1982
1983     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1984     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1985
1986     for (i = 0; i < tempbufidx; i++)
1987     {
1988         struct tempbuf_id_list *idlist = &index[i].idlist;
1989
1990         /* Fix the lookup list. */
1991         while (idlist != NULL)
1992         {
1993             if (idlist->id >= 0)
1994                 lookup[idlist->id] = &index[i];
1995             idlist = idlist->next;
1996         }
1997
1998         index[i].seek = lseek(fd, 0, SEEK_CUR);
1999         length = strlen(index[i].str) + 1;
2000         fe.tag_length = length;
2001         fe.idx_id = index[i].idx_id;
2002
2003         /* Check the chunk alignment. */
2004         if ((fe.tag_length + sizeof(struct tagfile_entry))
2005             % TAGFILE_ENTRY_CHUNK_LENGTH)
2006         {
2007             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2008                 ((fe.tag_length + sizeof(struct tagfile_entry))
2009                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2010         }
2011
2012 #ifdef TAGCACHE_STRICT_ALIGN
2013         /* Make sure the entry is long aligned. */
2014         if (index[i].seek & 0x03)
2015         {
2016             logf("tempbuf_sort: alignment error!");
2017             return -3;
2018         }
2019 #endif
2020
2021         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2022             sizeof(struct tagfile_entry))
2023         {
2024             logf("tempbuf_sort: write error #1");
2025             return -1;
2026         }
2027
2028         if (write(fd, index[i].str, length) != length)
2029         {
2030             logf("tempbuf_sort: write error #2");
2031             return -2;
2032         }
2033
2034         /* Write some padding. */
2035         if (fe.tag_length - length > 0)
2036             write(fd, "XXXXXXXX", fe.tag_length - length);
2037     }
2038
2039     return i;
2040 }
2041
2042 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2043 {
2044     if (id < 0 || id >= lookup_buffer_depth)
2045         return NULL;
2046
2047     return lookup[id];
2048 }
2049
2050
2051 inline static int tempbuf_find_location(int id)
2052 {
2053     struct tempbuf_searchidx *entry;
2054
2055     entry = tempbuf_locate(id);
2056     if (entry == NULL)
2057         return -1;
2058
2059     return entry->seek;
2060 }
2061
2062 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2063 {
2064     struct master_header tcmh;
2065     struct index_entry idx;
2066     int masterfd;
2067     int masterfd_pos;
2068     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2069     int max_entries;
2070     int entries_processed = 0;
2071     int i, j;
2072     char buf[TAG_MAXLEN];
2073
2074     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2075
2076     logf("Building numeric indices...");
2077     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2078
2079     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2080         return false;
2081
2082     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2083                          SEEK_CUR);
2084     if (masterfd_pos == filesize(masterfd))
2085     {
2086         logf("we can't append!");
2087         close(masterfd);
2088         return false;
2089     }
2090
2091     while (entries_processed < h->entry_count)
2092     {
2093         int count = MIN(h->entry_count - entries_processed, max_entries);
2094
2095         /* Read in as many entries as possible. */
2096         for (i = 0; i < count; i++)
2097         {
2098             struct temp_file_entry *tfe = &entrybuf[i];
2099             int datastart;
2100
2101             /* Read in numeric data. */
2102             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2103                 sizeof(struct temp_file_entry))
2104             {
2105                 logf("read fail #1");
2106                 close(masterfd);
2107                 return false;
2108             }
2109
2110             datastart = lseek(tmpfd, 0, SEEK_CUR);
2111
2112             /**
2113              * Read string data from the following tags:
2114              * - tag_filename
2115              * - tag_artist
2116              * - tag_album
2117              * - tag_title
2118              *
2119              * A crc32 hash is calculated from the read data
2120              * and stored back to the data offset field kept in memory.
2121              */
2122 #define tmpdb_read_string_tag(tag) \
2123     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2124     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2125     { \
2126         logf("read fail: buffer overflow"); \
2127         close(masterfd); \
2128         return false; \
2129     } \
2130     \
2131     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2132         tfe->tag_length[tag]) \
2133     { \
2134         logf("read fail #2"); \
2135         close(masterfd); \
2136         return false; \
2137     } \
2138     \
2139     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2140     lseek(tmpfd, datastart, SEEK_SET)
2141
2142             tmpdb_read_string_tag(tag_filename);
2143             tmpdb_read_string_tag(tag_artist);
2144             tmpdb_read_string_tag(tag_album);
2145             tmpdb_read_string_tag(tag_title);
2146
2147             /* Seek to the end of the string data. */
2148             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2149         }
2150
2151         /* Backup the master index position. */
2152         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2153         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2154
2155         /* Check if we can resurrect some deleted runtime statistics data. */
2156         for (i = 0; i < tcmh.tch.entry_count; i++)
2157         {
2158             /* Read the index entry. */
2159             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2160                 != sizeof(struct index_entry))
2161             {
2162                 logf("read fail #3");
2163                 close(masterfd);
2164                 return false;
2165             }
2166
2167             /**
2168              * Skip unless the entry is marked as being deleted
2169              * or the data has already been resurrected.
2170              */
2171             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2172                 continue;
2173
2174             /* Now try to match the entry. */
2175             /**
2176              * To succesfully match a song, the following conditions
2177              * must apply:
2178              *
2179              * For numeric fields: tag_length
2180              * - Full identical match is required
2181              *
2182              * If tag_filename matches, no further checking necessary.
2183              *
2184              * For string hashes: tag_artist, tag_album, tag_title
2185              * - Two of these must match
2186              */
2187             for (j = 0; j < count; j++)
2188             {
2189                 struct temp_file_entry *tfe = &entrybuf[j];
2190
2191                 /* Try to match numeric fields first. */
2192                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2193                     continue;
2194
2195                 /* Now it's time to do the hash matching. */
2196                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2197                 {
2198                     int match_count = 0;
2199
2200                     /* No filename match, check if we can match two other tags. */
2201 #define tmpdb_match(tag) \
2202     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2203         match_count++
2204
2205                     tmpdb_match(tag_artist);
2206                     tmpdb_match(tag_album);
2207                     tmpdb_match(tag_title);
2208
2209                     if (match_count < 2)
2210                     {
2211                         /* Still no match found, give up. */
2212                         continue;
2213                     }
2214                 }
2215
2216                 /* A match found, now copy & resurrect the statistical data. */
2217 #define tmpdb_copy_tag(tag) \
2218     tfe->tag_offset[tag] = idx.tag_seek[tag]
2219
2220                 tmpdb_copy_tag(tag_playcount);
2221                 tmpdb_copy_tag(tag_rating);
2222                 tmpdb_copy_tag(tag_playtime);
2223                 tmpdb_copy_tag(tag_lastplayed);
2224                 tmpdb_copy_tag(tag_commitid);
2225
2226                 /* Avoid processing this entry again. */
2227                 idx.flag |= FLAG_RESURRECTED;
2228
2229                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2230                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2231                     != sizeof(struct index_entry))
2232                 {
2233                     logf("masterfd writeback fail #1");
2234                     close(masterfd);
2235                     return false;
2236                 }
2237
2238                 logf("Entry resurrected");
2239             }
2240         }
2241
2242
2243         /* Restore the master index position. */
2244         lseek(masterfd, masterfd_pos, SEEK_SET);
2245
2246         /* Commit the data to the index. */
2247         for (i = 0; i < count; i++)
2248         {
2249             int loc = lseek(masterfd, 0, SEEK_CUR);
2250
2251             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2252                 != sizeof(struct index_entry))
2253             {
2254                 logf("read fail #3");
2255                 close(masterfd);
2256                 return false;
2257             }
2258
2259             for (j = 0; j < TAG_COUNT; j++)
2260             {
2261                 if (!tagcache_is_numeric_tag(j))
2262                     continue;
2263
2264                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2265             }
2266             idx.flag = entrybuf[i].flag;
2267
2268             if (idx.tag_seek[tag_commitid])
2269             {
2270                 /* Data has been resurrected. */
2271                 idx.flag |= FLAG_DIRTYNUM;
2272             }
2273             else if (tc_stat.ready && current_tcmh.commitid > 0)
2274             {
2275                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2276                 idx.flag |= FLAG_DIRTYNUM;
2277             }
2278
2279             /* Write back the updated index. */
2280             lseek(masterfd, loc, SEEK_SET);
2281             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2282                 != sizeof(struct index_entry))
2283             {
2284                 logf("write fail");
2285                 close(masterfd);
2286                 return false;
2287             }
2288         }
2289
2290         entries_processed += count;
2291         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2292     }
2293
2294     close(masterfd);
2295
2296     return true;
2297 }
2298
2299 /**
2300  * Return values:
2301  *     > 0   success
2302  *    == 0   temporary failure
2303  *     < 0   fatal error
2304  */
2305 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2306 {
2307     int i;
2308     struct tagcache_header tch;
2309     struct master_header   tcmh;
2310     struct index_entry idxbuf[IDX_BUF_DEPTH];
2311     int idxbuf_pos;
2312     char buf[TAG_MAXLEN+32];
2313     int fd = -1, masterfd;
2314     bool error = false;
2315     int init;
2316     int masterfd_pos;
2317
2318     logf("Building index: %d", index_type);
2319
2320     /* Check the number of entries we need to allocate ram for. */
2321     commit_entry_count = h->entry_count + 1;
2322
2323     masterfd = open_master_fd(&tcmh, false);
2324     if (masterfd >= 0)
2325     {
2326         commit_entry_count += tcmh.tch.entry_count;
2327         close(masterfd);
2328     }
2329     else
2330         remove_files(); /* Just to be sure we are clean. */
2331
2332     /* Open the index file, which contains the tag names. */
2333     fd = open_tag_fd(&tch, index_type, true);
2334     if (fd >= 0)
2335     {
2336         logf("tch.datasize=%ld", tch.datasize);
2337         lookup_buffer_depth = 1 +
2338         /* First part */ commit_entry_count +
2339         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2340     }
2341     else
2342     {
2343         lookup_buffer_depth = 1 +
2344         /* First part */ commit_entry_count +
2345         /* Second part */ 0;
2346     }
2347
2348     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2349     logf("commit_entry_count=%ld", commit_entry_count);
2350
2351     /* Allocate buffer for all index entries from both old and new
2352      * tag files. */
2353     tempbufidx = 0;
2354     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2355
2356     /* Allocate lookup buffer. The first portion of commit_entry_count
2357      * contains the new tags in the temporary file and the second
2358      * part for locating entries already in the db.
2359      *
2360      *  New tags  Old tags
2361      * +---------+---------------------------+
2362      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2363      * +---------+---------------------------+
2364      *
2365      * Old tags are inserted to a temporary buffer with position:
2366      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2367      * And new tags with index:
2368      *     tempbuf_insert(idx, ...);
2369      *
2370      * The buffer is sorted and written into tag file:
2371      *     tempbuf_sort(...);
2372      * leaving master index locations messed up.
2373      *
2374      * That is fixed using the lookup buffer for old tags:
2375      *     new_seek = tempbuf_find_location(old_seek, ...);
2376      * and for new tags:
2377      *     new_seek = tempbuf_find_location(idx);
2378      */
2379     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2380     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2381     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2382
2383     /* And calculate the remaining data space used mainly for storing
2384      * tag data (strings). */
2385     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2386     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2387     {
2388         logf("Buffer way too small!");
2389         return 0;
2390     }
2391
2392     if (fd >= 0)
2393     {
2394         /**
2395          * If tag file contains unique tags (sorted index), we will load
2396          * it entirely into memory so we can resort it later for use with
2397          * chunked browsing.
2398          */
2399         if (tagcache_is_sorted_tag(index_type))
2400         {
2401             logf("loading tags...");
2402             for (i = 0; i < tch.entry_count; i++)
2403             {
2404                 struct tagfile_entry entry;
2405                 int loc = lseek(fd, 0, SEEK_CUR);
2406                 bool ret;
2407
2408                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2409                     != sizeof(struct tagfile_entry))
2410                 {
2411                     logf("read error #7");
2412                     close(fd);
2413                     return -2;
2414                 }
2415
2416                 if (entry.tag_length >= (int)sizeof(buf))
2417                 {
2418                     logf("too long tag #3");
2419                     close(fd);
2420                     return -2;
2421                 }
2422
2423                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2424                 {
2425                     logf("read error #8");
2426                     close(fd);
2427                     return -2;
2428                 }
2429
2430                 /* Skip deleted entries. */
2431                 if (buf[0] == '\0')
2432                     continue;
2433
2434                 /**
2435                  * Save the tag and tag id in the memory buffer. Tag id
2436                  * is saved so we can later reindex the master lookup
2437                  * table when the index gets resorted.
2438                  */
2439                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2440                                      + commit_entry_count, entry.idx_id,
2441                                      tagcache_is_unique_tag(index_type));
2442                 if (!ret)
2443                 {
2444                     close(fd);
2445                     return -3;
2446                 }
2447                 do_timed_yield();
2448             }
2449             logf("done");
2450         }
2451         else
2452             tempbufidx = tch.entry_count;
2453     }
2454     else
2455     {
2456         /**
2457          * Creating new index file to store the tags. No need to preload
2458          * anything whether the index type is sorted or not.
2459          */
2460         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2461         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2462         if (fd < 0)
2463         {
2464             logf("%s open fail", buf);
2465             return -2;
2466         }
2467
2468         tch.magic = TAGCACHE_MAGIC;
2469         tch.entry_count = 0;
2470         tch.datasize = 0;
2471
2472         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2473             != sizeof(struct tagcache_header))
2474         {
2475             logf("header write failed");
2476             close(fd);
2477             return -2;
2478         }
2479     }
2480
2481     /* Loading the tag lookup file as "master file". */
2482     logf("Loading index file");
2483     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2484
2485     if (masterfd < 0)
2486     {
2487         logf("Creating new DB");
2488         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2489
2490         if (masterfd < 0)
2491         {
2492             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2493             close(fd);
2494             return -2;
2495         }
2496
2497         /* Write the header (write real values later). */
2498         memset(&tcmh, 0, sizeof(struct master_header));
2499         tcmh.tch = *h;
2500         tcmh.tch.entry_count = 0;
2501         tcmh.tch.datasize = 0;
2502         tcmh.dirty = true;
2503         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2504         init = true;
2505         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2506     }
2507     else
2508     {
2509         /**
2510          * Master file already exists so we need to process the current
2511          * file first.
2512          */
2513         init = false;
2514
2515         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2516             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2517         {
2518             logf("header error");
2519             close(fd);
2520             close(masterfd);
2521             return -2;
2522         }
2523
2524         /**
2525          * If we reach end of the master file, we need to expand it to
2526          * hold new tags. If the current index is not sorted, we can
2527          * simply append new data to end of the file.
2528          * However, if the index is sorted, we need to update all tag
2529          * pointers in the master file for the current index.
2530          */
2531         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2532             SEEK_CUR);
2533         if (masterfd_pos == filesize(masterfd))
2534         {
2535             logf("appending...");
2536             init = true;
2537         }
2538     }
2539
2540     /**
2541      * Load new unique tags in memory to be sorted later and added
2542      * to the master lookup file.
2543      */
2544     if (tagcache_is_sorted_tag(index_type))
2545     {
2546         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2547         /* h is the header of the temporary file containing new tags. */
2548         logf("inserting new tags...");
2549         for (i = 0; i < h->entry_count; i++)
2550         {
2551             struct temp_file_entry entry;
2552
2553             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2554                 sizeof(struct temp_file_entry))
2555             {
2556                 logf("read fail #3");
2557                 error = true;
2558                 goto error_exit;
2559             }
2560
2561             /* Read data. */
2562             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2563             {
2564                 logf("too long entry!");
2565                 error = true;
2566                 goto error_exit;
2567             }
2568
2569             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2570             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2571                 entry.tag_length[index_type])
2572             {
2573                 logf("read fail #4");
2574                 error = true;
2575                 goto error_exit;
2576             }
2577
2578             if (tagcache_is_unique_tag(index_type))
2579                 error = !tempbuf_insert(buf, i, -1, true);
2580             else
2581                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2582
2583             if (error)
2584             {
2585                 logf("insert error");
2586                 goto error_exit;
2587             }
2588
2589             /* Skip to next. */
2590             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2591                     entry.tag_length[index_type], SEEK_CUR);
2592             do_timed_yield();
2593         }
2594         logf("done");
2595
2596         /* Sort the buffer data and write it to the index file. */
2597         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2598         i = tempbuf_sort(fd);
2599         if (i < 0)
2600             goto error_exit;
2601         logf("sorted %d tags", i);
2602
2603         /**
2604          * Now update all indexes in the master lookup file.
2605          */
2606         logf("updating indices...");
2607         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2608         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2609         {
2610             int j;
2611             int loc = lseek(masterfd, 0, SEEK_CUR);
2612
2613             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2614
2615             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2616                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2617             {
2618                 logf("read fail #5");
2619                 error = true;
2620                 goto error_exit ;
2621             }
2622             lseek(masterfd, loc, SEEK_SET);
2623
2624             for (j = 0; j < idxbuf_pos; j++)
2625             {
2626                 if (idxbuf[j].flag & FLAG_DELETED)
2627                 {
2628                     /* We can just ignore deleted entries. */
2629                     // idxbuf[j].tag_seek[index_type] = 0;
2630                     continue;
2631                 }
2632
2633                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2634                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2635                     + commit_entry_count);
2636
2637                 if (idxbuf[j].tag_seek[index_type] < 0)
2638                 {
2639                     logf("update error: %d/%d/%d",
2640                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2641                     error = true;
2642                     goto error_exit;
2643                 }
2644
2645                 do_timed_yield();
2646             }
2647
2648             /* Write back the updated index. */
2649             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2650                         index_entry_ec, tc_stat.econ) !=
2651                 (int)sizeof(struct index_entry)*idxbuf_pos)
2652             {
2653                 logf("write fail");
2654                 error = true;
2655                 goto error_exit;
2656             }
2657         }
2658         logf("done");
2659     }
2660
2661     /**
2662      * Walk through the temporary file containing the new tags.
2663      */
2664     // build_normal_index(h, tmpfd, masterfd, idx);
2665     logf("updating new indices...");
2666     lseek(masterfd, masterfd_pos, SEEK_SET);
2667     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2668     lseek(fd, 0, SEEK_END);
2669     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2670     {
2671         int j;
2672
2673         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2674         if (init)
2675         {
2676             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2677         }
2678         else
2679         {
2680             int loc = lseek(masterfd, 0, SEEK_CUR);
2681
2682             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2683                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2684             {
2685                 logf("read fail #6");
2686                 error = true;
2687                 break ;
2688             }
2689             lseek(masterfd, loc, SEEK_SET);
2690         }
2691
2692         /* Read entry headers. */
2693         for (j = 0; j < idxbuf_pos; j++)
2694         {
2695             if (!tagcache_is_sorted_tag(index_type))
2696             {
2697                 struct temp_file_entry entry;
2698                 struct tagfile_entry fe;
2699
2700                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2701                     sizeof(struct temp_file_entry))
2702                 {
2703                     logf("read fail #7");
2704                     error = true;
2705                     break ;
2706                 }
2707
2708                 /* Read data. */
2709                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2710                 {
2711                     logf("too long entry!");
2712                     logf("length=%d", entry.tag_length[index_type]);
2713                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2714                     error = true;
2715                     break ;
2716                 }
2717
2718                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2719                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2720                     entry.tag_length[index_type])
2721                 {
2722                     logf("read fail #8");
2723                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2724                     logf("length=0x%02x", entry.tag_length[index_type]);
2725                     error = true;
2726                     break ;
2727                 }
2728
2729                 /* Write to index file. */
2730                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2731                 fe.tag_length = entry.tag_length[index_type];
2732                 fe.idx_id = tcmh.tch.entry_count + i + j;
2733                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2734                 write(fd, buf, fe.tag_length);
2735                 tempbufidx++;
2736
2737                 /* Skip to next. */
2738                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2739                       entry.tag_length[index_type], SEEK_CUR);
2740             }
2741             else
2742             {
2743                 /* Locate the correct entry from the sorted array. */
2744                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2745                 if (idxbuf[j].tag_seek[index_type] < 0)
2746                 {
2747                     logf("entry not found (%d)", j);
2748                     error = true;
2749                     break ;
2750                 }
2751             }
2752         }
2753
2754         /* Write index. */
2755         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2756                   index_entry_ec, tc_stat.econ) !=
2757             (int)sizeof(struct index_entry)*idxbuf_pos)
2758         {
2759             logf("tagcache: write fail #4");
2760             error = true;
2761             break ;
2762         }
2763
2764         do_timed_yield();
2765     }
2766     logf("done");
2767
2768     /* Finally write the header. */
2769     tch.magic = TAGCACHE_MAGIC;
2770     tch.entry_count = tempbufidx;
2771     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2772     lseek(fd, 0, SEEK_SET);
2773     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2774
2775     if (index_type != tag_filename)
2776         h->datasize += tch.datasize;
2777     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2778     error_exit:
2779
2780     close(fd);
2781     close(masterfd);
2782
2783     if (error)
2784         return -2;
2785
2786     return 1;
2787 }
2788
2789 static bool commit(void)
2790 {
2791     struct tagcache_header tch;
2792     struct master_header   tcmh;
2793     int i, len, rc;
2794     int tmpfd;
2795     int masterfd;
2796 #ifdef HAVE_DIRCACHE
2797     bool dircache_buffer_stolen = false;
2798 #endif
2799     bool local_allocation = false;
2800
2801     logf("committing tagcache");
2802
2803     while (write_lock)
2804         sleep(1);
2805
2806     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2807     if (tmpfd < 0)
2808     {
2809         logf("nothing to commit");
2810         return true;
2811     }
2812
2813
2814     /* Load the header. */
2815     len = sizeof(struct tagcache_header);
2816     rc = read(tmpfd, &tch, len);
2817
2818     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2819     {
2820         logf("incorrect header");
2821         close(tmpfd);
2822         remove(TAGCACHE_FILE_TEMP);
2823         return false;
2824     }
2825
2826     if (tch.entry_count == 0)
2827     {
2828         logf("nothing to commit");
2829         close(tmpfd);
2830         remove(TAGCACHE_FILE_TEMP);
2831         return true;
2832     }
2833
2834 #ifdef HAVE_EEPROM_SETTINGS
2835     remove(TAGCACHE_STATEFILE);
2836 #endif
2837
2838     /* At first be sure to unload the ramcache! */
2839 #ifdef HAVE_TC_RAMCACHE
2840     tc_stat.ramcache = false;
2841 #endif
2842
2843     read_lock++;
2844
2845     /* Try to steal every buffer we can :) */
2846     if (tempbuf_size == 0)
2847         local_allocation = true;
2848
2849 #ifdef HAVE_DIRCACHE
2850     if (tempbuf_size == 0)
2851     {
2852         /* Try to steal the dircache buffer. */
2853         tempbuf = dircache_steal_buffer(&tempbuf_size);
2854         tempbuf_size &= ~0x03;
2855
2856         if (tempbuf_size > 0)
2857         {
2858             dircache_buffer_stolen = true;
2859         }
2860     }
2861 #endif
2862
2863 #ifdef HAVE_TC_RAMCACHE
2864     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2865     {
2866         tempbuf = (char *)(hdr + 1);
2867         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2868         tempbuf_size &= ~0x03;
2869     }
2870 #endif
2871
2872     /* And finally fail if there are no buffers available. */
2873     if (tempbuf_size == 0)
2874     {
2875         logf("delaying commit until next boot");
2876         tc_stat.commit_delayed = true;
2877         close(tmpfd);
2878         read_lock--;
2879         return false;
2880     }
2881
2882     logf("commit %ld entries...", tch.entry_count);
2883
2884     /* Mark DB dirty so it will stay disabled if commit fails. */
2885     current_tcmh.dirty = true;
2886     update_master_header();
2887
2888     /* Now create the index files. */
2889     tc_stat.commit_step = 0;
2890     tch.datasize = 0;
2891     tc_stat.commit_delayed = false;
2892
2893     for (i = 0; i < TAG_COUNT; i++)
2894     {
2895         int ret;
2896
2897         if (tagcache_is_numeric_tag(i))
2898             continue;
2899
2900         tc_stat.commit_step++;
2901         ret = build_index(i, &tch, tmpfd);
2902         if (ret <= 0)
2903         {
2904             close(tmpfd);
2905             logf("tagcache failed init");
2906             if (ret < 0)
2907                 remove_files();
2908             else
2909                 tc_stat.commit_delayed = true;
2910             tc_stat.commit_step = 0;
2911             read_lock--;
2912             return false;
2913         }
2914     }
2915
2916     if (!build_numeric_indices(&tch, tmpfd))
2917     {
2918         logf("Failure to commit numeric indices");
2919         close(tmpfd);
2920         remove_files();
2921         tc_stat.commit_step = 0;
2922         read_lock--;
2923         return false;
2924     }
2925
2926     close(tmpfd);
2927     tc_stat.commit_step = 0;
2928
2929     /* Update the master index headers. */
2930     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2931     {
2932         read_lock--;
2933         return false;
2934     }
2935
2936     tcmh.tch.entry_count += tch.entry_count;
2937     tcmh.tch.datasize = sizeof(struct master_header)
2938         + sizeof(struct index_entry) * tcmh.tch.entry_count
2939         + tch.datasize;
2940     tcmh.dirty = false;
2941     tcmh.commitid++;
2942
2943     lseek(masterfd, 0, SEEK_SET);
2944     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2945     close(masterfd);
2946
2947     logf("tagcache committed");
2948     remove(TAGCACHE_FILE_TEMP);
2949     tc_stat.ready = check_all_headers();
2950     tc_stat.readyvalid = true;
2951
2952     if (local_allocation)
2953     {
2954         tempbuf = NULL;
2955         tempbuf_size = 0;
2956     }
2957
2958 #ifdef HAVE_DIRCACHE
2959     /* Rebuild the dircache, if we stole the buffer. */
2960     if (dircache_buffer_stolen)
2961         dircache_build(0);
2962 #endif
2963
2964 #ifdef HAVE_TC_RAMCACHE
2965     /* Reload tagcache. */
2966     if (tc_stat.ramcache_allocated > 0)
2967         tagcache_start_scan();
2968 #endif
2969
2970     read_lock--;
2971
2972     return true;
2973 }
2974
2975 static void allocate_tempbuf(void)
2976 {
2977     /* Yeah, malloc would be really nice now :) */
2978 #ifdef __PCTOOL__
2979     tempbuf_size = 32*1024*1024;
2980     tempbuf = malloc(tempbuf_size);
2981 #else
2982     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2983     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2984     audiobuf += tempbuf_size;
2985 #endif
2986 }
2987
2988 static void free_tempbuf(void)
2989 {
2990     if (tempbuf_size == 0)
2991         return ;
2992
2993 #ifdef __PCTOOL__
2994     free(tempbuf);
2995 #else
2996     audiobuf -= tempbuf_size;
2997 #endif
2998     tempbuf = NULL;
2999     tempbuf_size = 0;
3000 }
3001
3002 #ifndef __PCTOOL__
3003
3004 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3005 {
3006     struct index_entry idx;
3007
3008     if (!tc_stat.ready)
3009         return false;
3010
3011     if (!tagcache_is_numeric_tag(tag))
3012         return false;
3013
3014     if (!get_index(masterfd, idx_id, &idx, false))
3015         return false;
3016
3017     idx.tag_seek[tag] = data;
3018     idx.flag |= FLAG_DIRTYNUM;
3019
3020     return write_index(masterfd, idx_id, &idx);
3021 }
3022
3023 #if 0
3024 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3025                                    int tag, long data)
3026 {
3027     struct master_header myhdr;
3028
3029     if (tcs->masterfd < 0)
3030     {
3031         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3032             return false;
3033     }
3034
3035     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3036 }
3037 #endif
3038
3039 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3040
3041 static bool command_queue_is_full(void)
3042 {
3043     int next;
3044
3045     next = command_queue_widx + 1;
3046     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3047         next = 0;
3048
3049     return (next == command_queue_ridx);
3050 }
3051
3052 static bool command_queue_sync_callback(void)
3053 {
3054
3055     struct master_header myhdr;
3056     int masterfd;
3057
3058     mutex_lock(&command_queue_mutex);
3059
3060     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3061         return false;
3062
3063     while (command_queue_ridx != command_queue_widx)
3064     {
3065         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3066
3067         switch (ce->command)
3068         {
3069             case CMD_UPDATE_MASTER_HEADER:
3070             {
3071                 close(masterfd);
3072                 update_master_header();
3073
3074                 /* Re-open the masterfd. */
3075                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3076                     return true;
3077
3078                 break;
3079             }
3080             case CMD_UPDATE_NUMERIC:
3081             {
3082                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3083                 break;
3084             }
3085         }
3086
3087         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3088             command_queue_ridx = 0;
3089     }
3090
3091     close(masterfd);
3092
3093     tc_stat.queue_length = 0;
3094     mutex_unlock(&command_queue_mutex);
3095     return true;
3096 }
3097
3098 static void run_command_queue(bool force)
3099 {
3100     if (COMMAND_QUEUE_IS_EMPTY)
3101         return;
3102
3103     if (force || command_queue_is_full())
3104         command_queue_sync_callback();
3105     else
3106         register_storage_idle_func(command_queue_sync_callback);
3107 }
3108
3109 static void queue_command(int cmd, long idx_id, int tag, long data)
3110 {
3111     while (1)
3112     {
3113         int next;
3114
3115         mutex_lock(&command_queue_mutex);
3116         next = command_queue_widx + 1;
3117         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3118             next = 0;
3119
3120         /* Make sure queue is not full. */
3121         if (next != command_queue_ridx)
3122         {
3123             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3124
3125             ce->command = cmd;
3126             ce->idx_id = idx_id;
3127             ce->tag = tag;
3128             ce->data = data;
3129
3130             command_queue_widx = next;
3131
3132             tc_stat.queue_length++;
3133
3134             mutex_unlock(&command_queue_mutex);
3135             break;
3136         }
3137
3138         /* Queue is full, try again later... */
3139         mutex_unlock(&command_queue_mutex);
3140         sleep(1);
3141     }
3142 }
3143
3144 long tagcache_increase_serial(void)
3145 {
3146     long old;
3147
3148     if (!tc_stat.ready)
3149         return -2;
3150
3151     while (read_lock)
3152         sleep(1);
3153
3154     old = current_tcmh.serial++;
3155     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3156
3157     return old;
3158 }
3159
3160 void tagcache_update_numeric(int idx_id, int tag, long data)
3161 {
3162     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3163 }
3164 #endif /* !__PCTOOL__ */
3165
3166 long tagcache_get_serial(void)
3167 {
3168     return current_tcmh.serial;
3169 }
3170
3171 long tagcache_get_commitid(void)
3172 {
3173     return current_tcmh.commitid;
3174 }
3175
3176 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3177 {
3178     char buf[512];
3179     int i;
3180
3181     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3182     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3183     {
3184         if (*datastr == '\0')
3185             break;
3186
3187         if (*datastr == '"' || *datastr == '\\')
3188             buf[i++] = '\\';
3189
3190         buf[i] = *(datastr++);
3191     }
3192
3193     strcpy(&buf[i], "\" ");
3194
3195     write(fd, buf, i + 2);
3196
3197     return true;
3198 }
3199
3200 #ifndef __PCTOOL__
3201
3202 static bool read_tag(char *dest, long size,
3203                      const char *src, const char *tagstr)
3204 {
3205     int pos;
3206     char current_tag[32];
3207
3208     while (*src != '\0')
3209     {
3210         /* Skip all whitespace */
3211         while (*src == ' ')
3212             src++;
3213
3214         if (*src == '\0')
3215             break;
3216
3217         pos = 0;
3218         /* Read in tag name */
3219         while (*src != '=' && *src != ' ')
3220         {
3221             current_tag[pos] = *src;
3222             src++;
3223             pos++;
3224
3225             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3226                 return false;
3227         }
3228         current_tag[pos] = '\0';
3229
3230         /* Read in tag data */
3231
3232         /* Find the start. */
3233         while (*src != '"' && *src != '\0')
3234             src++;
3235
3236         if (*src == '\0' || *(++src) == '\0')
3237             return false;
3238
3239         /* Read the data. */
3240         for (pos = 0; pos < size; pos++)
3241         {
3242             if (*src == '\0')
3243                 break;
3244
3245             if (*src == '\\')
3246             {
3247                 dest[pos] = *(src+1);
3248                 src += 2;
3249                 continue;
3250             }
3251
3252             dest[pos] = *src;
3253
3254             if (*src == '"')
3255             {
3256                 src++;
3257                 break;
3258             }
3259
3260             if (*src == '\0')
3261                 break;
3262
3263             src++;
3264         }
3265         dest[pos] = '\0';
3266
3267         if (!strcasecmp(tagstr, current_tag))
3268             return true;
3269     }
3270
3271     return false;
3272 }
3273
3274 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3275 {
3276     struct index_entry idx;
3277     char tag_data[TAG_MAXLEN+32];
3278     int idx_id;
3279     long masterfd = (long)parameters;
3280     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3281         tag_commitid };
3282     int i;
3283     (void)line_n;
3284
3285     if (*buf == '#')
3286         return 0;
3287
3288     logf("%d/%s", line_n, buf);
3289     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3290     {
3291         logf("filename missing");
3292         logf("-> %s", buf);
3293         return 0;
3294     }
3295
3296     idx_id = find_index(tag_data);
3297     if (idx_id < 0)
3298     {
3299         logf("entry not found");
3300         return 0;
3301     }
3302
3303     if (!get_index(masterfd, idx_id, &idx, false))
3304     {
3305         logf("failed to retrieve index entry");
3306         return 0;
3307     }
3308
3309     /* Stop if tag has already been modified. */
3310     if (idx.flag & FLAG_DIRTYNUM)
3311         return 0;
3312
3313     logf("import: %s", tag_data);
3314
3315     idx.flag |= FLAG_DIRTYNUM;
3316     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3317     {
3318         int data;
3319
3320         if (!read_tag(tag_data, sizeof tag_data, buf,
3321                       tagcache_tag_to_str(import_tags[i])))
3322         {
3323             continue;
3324         }
3325
3326         data = atoi(tag_data);
3327         if (data < 0)
3328             continue;
3329
3330         idx.tag_seek[import_tags[i]] = data;
3331
3332         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3333             current_tcmh.serial = data + 1;
3334         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3335             current_tcmh.commitid = data + 1;
3336     }
3337
3338     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3339 }
3340
3341 bool tagcache_import_changelog(void)
3342 {
3343     struct master_header myhdr;
3344     struct tagcache_header tch;
3345     int clfd;
3346     long masterfd;
3347     char buf[2048];
3348
3349     if (!tc_stat.ready)
3350         return false;
3351
3352     while (read_lock)
3353         sleep(1);
3354
3355     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3356     if (clfd < 0)
3357     {
3358         logf("failure to open changelog");
3359         return false;
3360     }
3361
3362     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3363     {
3364         close(clfd);
3365         return false;
3366     }
3367
3368     write_lock++;
3369
3370     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3371
3372     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3373                   parse_changelog_line);
3374
3375     close(clfd);
3376     close(masterfd);
3377
3378     if (filenametag_fd >= 0)
3379         close(filenametag_fd);
3380
3381     write_lock--;
3382
3383     update_master_header();
3384
3385     return true;
3386 }
3387
3388 #endif /* !__PCTOOL__ */
3389
3390 bool tagcache_create_changelog(struct tagcache_search *tcs)
3391 {
3392     struct master_header myhdr;
3393     struct index_entry idx;
3394     char buf[TAG_MAXLEN+32];
3395     char temp[32];
3396     int clfd;
3397     int i, j;
3398
3399     if (!tc_stat.ready)
3400         return false;
3401
3402     if (!tagcache_search(tcs, tag_filename))
3403         return false;
3404
3405     /* Initialize the changelog */
3406     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3407     if (clfd < 0)
3408     {
3409         logf("failure to open changelog");
3410         return false;
3411     }
3412
3413     if (tcs->masterfd < 0)
3414     {
3415         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3416             return false;
3417     }
3418     else
3419     {
3420         lseek(tcs->masterfd, 0, SEEK_SET);
3421         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3422     }
3423
3424     write(clfd, "## Changelog version 1\n", 23);
3425
3426     for (i = 0; i < myhdr.tch.entry_count; i++)
3427     {
3428         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3429             != sizeof(struct index_entry))
3430         {
3431             logf("read error #9");
3432             tagcache_search_finish(tcs);
3433             close(clfd);
3434             return false;
3435         }
3436
3437         /* Skip until the entry found has been modified. */
3438         if (! (idx.flag & FLAG_DIRTYNUM) )
3439             continue;
3440
3441         /* Skip deleted entries too. */
3442         if (idx.flag & FLAG_DELETED)
3443             continue;
3444
3445         /* Now retrieve all tags. */
3446         for (j = 0; j < TAG_COUNT; j++)
3447         {
3448             if (tagcache_is_numeric_tag(j))
3449             {
3450                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3451                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3452                 continue;
3453             }
3454
3455             tcs->type = j;
3456             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3457             write_tag(clfd, tagcache_tag_to_str(j), buf);
3458         }
3459
3460         write(clfd, "\n", 1);
3461         do_timed_yield();
3462     }
3463
3464     close(clfd);
3465
3466     tagcache_search_finish(tcs);
3467
3468     return true;
3469 }
3470
3471 static bool delete_entry(long idx_id)
3472 {
3473     int fd = -1;
3474     int masterfd = -1;
3475     int tag, i;
3476     struct index_entry idx, myidx;
3477     struct master_header myhdr;
3478     char buf[TAG_MAXLEN+32];
3479     int in_use[TAG_COUNT];
3480
3481     logf("delete_entry(): %ld", idx_id);
3482
3483 #ifdef HAVE_TC_RAMCACHE
3484     /* At first mark the entry removed from ram cache. */
3485     if (tc_stat.ramcache)
3486         hdr->indices[idx_id].flag |= FLAG_DELETED;
3487 #endif
3488
3489     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3490         return false;
3491
3492     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3493     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3494         != sizeof(struct index_entry))
3495     {
3496         logf("delete_entry(): read error");
3497         goto cleanup;
3498     }
3499
3500     if (myidx.flag & FLAG_DELETED)
3501     {
3502         logf("delete_entry(): already deleted!");
3503         goto cleanup;
3504     }
3505
3506     myidx.flag |= FLAG_DELETED;
3507     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3508     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3509         != sizeof(struct index_entry))
3510     {
3511         logf("delete_entry(): write_error #1");
3512         goto cleanup;
3513     }
3514
3515     /* Now check which tags are no longer in use (if any) */
3516     for (tag = 0; tag < TAG_COUNT; tag++)
3517         in_use[tag] = 0;
3518
3519     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3520     for (i = 0; i < myhdr.tch.entry_count; i++)
3521     {
3522         struct index_entry *idxp;
3523
3524 #ifdef HAVE_TC_RAMCACHE
3525         /* Use RAM DB if available for greater speed */
3526         if (tc_stat.ramcache)
3527             idxp = &hdr->indices[i];
3528         else
3529 #endif
3530         {
3531             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3532                 != sizeof(struct index_entry))
3533             {
3534                 logf("delete_entry(): read error #2");
3535                 goto cleanup;
3536             }
3537             idxp = &idx;
3538         }
3539
3540         if (idxp->flag & FLAG_DELETED)
3541             continue;
3542
3543         for (tag = 0; tag < TAG_COUNT; tag++)
3544         {
3545             if (tagcache_is_numeric_tag(tag))
3546                 continue;
3547
3548             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3549                 in_use[tag]++;
3550         }
3551     }
3552
3553     /* Now delete all tags no longer in use. */
3554     for (tag = 0; tag < TAG_COUNT; tag++)
3555     {
3556         struct tagcache_header tch;
3557         int oldseek = myidx.tag_seek[tag];
3558
3559         if (tagcache_is_numeric_tag(tag))
3560             continue;
3561
3562         /**
3563          * Replace tag seek with a hash value of the field string data.
3564          * That way runtime statistics of moved or altered files can be
3565          * resurrected.
3566          */
3567 #ifdef HAVE_TC_RAMCACHE
3568         if (tc_stat.ramcache && tag != tag_filename)
3569         {
3570             struct tagfile_entry *tfe;
3571             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3572
3573             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3574             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3575             myidx.tag_seek[tag] = *seek;
3576         }
3577         else
3578 #endif
3579         {
3580             struct tagfile_entry tfe;
3581
3582             /* Open the index file, which contains the tag names. */
3583             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3584                 goto cleanup;
3585
3586             /* Skip the header block */
3587             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3588             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3589                 != sizeof(struct tagfile_entry))
3590             {
3591                 logf("delete_entry(): read error #3");
3592                 goto cleanup;
3593             }
3594
3595             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3596             {
3597                 logf("delete_entry(): read error #4");
3598                 goto cleanup;
3599             }
3600
3601             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3602         }
3603
3604         if (in_use[tag])
3605         {
3606             logf("in use: %d/%d", tag, in_use[tag]);
3607             if (fd >= 0)
3608             {
3609                 close(fd);
3610                 fd = -1;
3611             }
3612             continue;
3613         }
3614
3615 #ifdef HAVE_TC_RAMCACHE
3616         /* Delete from ram. */
3617         if (tc_stat.ramcache && tag != tag_filename)
3618         {
3619             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3620             tagentry->tag_data[0] = '\0';
3621         }
3622 #endif
3623
3624         /* Open the index file, which contains the tag names. */
3625         if (fd < 0)
3626         {
3627             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3628                 goto cleanup;
3629         }
3630
3631         /* Skip the header block */
3632         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3633
3634         /* Debug, print 10 first characters of the tag
3635         read(fd, buf, 10);
3636         buf[10]='\0';
3637         logf("TAG:%s", buf);
3638         lseek(fd, -10, SEEK_CUR);
3639         */
3640
3641         /* Write first data byte in tag as \0 */
3642         write(fd, "", 1);
3643
3644         /* Now tag data has been removed */
3645         close(fd);
3646         fd = -1;
3647     }
3648
3649     /* Write index entry back into master index. */
3650     lseek(masterfd, sizeof(struct master_header) +
3651           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3652     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3653         != sizeof(struct index_entry))
3654     {
3655         logf("delete_entry(): write_error #2");
3656         goto cleanup;
3657     }
3658
3659     close(masterfd);
3660
3661     return true;
3662
3663     cleanup:
3664     if (fd >= 0)
3665         close(fd);
3666     if (masterfd >= 0)
3667         close(masterfd);
3668
3669     return false;
3670 }
3671
3672 #ifndef __PCTOOL__
3673 /**
3674  * Returns true if there is an event waiting in the queue
3675  * that requires the current operation to be aborted.
3676  */
3677 static bool check_event_queue(void)
3678 {
3679     struct queue_event ev;
3680
3681     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3682     switch (ev.id)
3683     {
3684         case Q_STOP_SCAN:
3685         case SYS_POWEROFF:
3686         case SYS_USB_CONNECTED:
3687             /* Put the event back into the queue. */
3688             queue_post(&tagcache_queue, ev.id, ev.data);
3689             return true;
3690     }
3691
3692     return false;
3693 }
3694 #endif
3695
3696 #ifdef HAVE_TC_RAMCACHE
3697 static bool allocate_tagcache(void)
3698 {
3699     struct master_header tcmh;
3700     int fd;
3701
3702     /* Load the header. */
3703     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3704     {
3705         hdr = NULL;
3706         return false;
3707     }
3708
3709     close(fd);
3710
3711     /**
3712      * Now calculate the required cache size plus
3713      * some extra space for alignment fixes.
3714      */
3715     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3716         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3717     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3718     memset(hdr, 0, sizeof(struct ramcache_header));
3719     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3720     hdr->indices = (struct index_entry *)(hdr + 1);
3721     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3722
3723     return true;
3724 }
3725
3726 # ifdef HAVE_EEPROM_SETTINGS
3727 static bool tagcache_dumpload(void)
3728 {
3729     struct statefile_header shdr;
3730     int fd, rc;
3731     long offpos;
3732     int i;
3733
3734     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3735     if (fd < 0)
3736     {
3737         logf("no tagcache statedump");
3738         return false;
3739     }
3740
3741     /* Check the statefile memory placement */
3742     hdr = buffer_alloc(0);
3743     rc = read(fd, &shdr, sizeof(struct statefile_header));
3744     if (rc != sizeof(struct statefile_header)
3745         /* || (long)hdr != (long)shdr.hdr */)
3746     {
3747         logf("incorrect statefile");
3748         hdr = NULL;
3749         close(fd);
3750         return false;
3751     }
3752
3753     offpos = (long)hdr - (long)shdr.hdr;
3754
3755     /* Lets allocate real memory and load it */
3756     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3757     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3758     close(fd);
3759
3760     if (rc != shdr.tc_stat.ramcache_allocated)
3761     {
3762         logf("read failure!");
3763         hdr = NULL;
3764         return false;
3765     }
3766
3767     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3768
3769     /* Now fix the pointers */
3770     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3771     for (i = 0; i < TAG_COUNT; i++)
3772         hdr->tags[i] += offpos;
3773
3774     return true;
3775 }
3776
3777 static bool tagcache_dumpsave(void)
3778 {
3779     struct statefile_header shdr;
3780     int fd;
3781
3782     if (!tc_stat.ramcache)
3783         return false;
3784
3785     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3786     if (fd < 0)
3787     {
3788         logf("failed to create a statedump");
3789         return false;
3790     }
3791
3792     /* Create the header */
3793     shdr.hdr = hdr;
3794     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3795     write(fd, &shdr, sizeof(struct statefile_header));
3796
3797     /* And dump the data too */
3798     write(fd, hdr, tc_stat.ramcache_allocated);
3799     close(fd);
3800
3801     return true;
3802 }
3803 # endif
3804
3805 static bool load_tagcache(void)
3806 {
3807     struct tagcache_header *tch;
3808     long bytesleft = tc_stat.ramcache_allocated;
3809     struct index_entry *idx;
3810     int rc, fd;
3811     char *p;
3812     int i, tag;
3813
3814 # ifdef HAVE_DIRCACHE
3815     while (dircache_is_initializing())
3816         sleep(1);
3817
3818     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3819 # endif
3820
3821     logf("loading tagcache to ram...");
3822
3823     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3824     if (fd < 0)
3825     {
3826         logf("tagcache open failed");
3827         return false;
3828     }
3829
3830     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3831         != sizeof(struct master_header)
3832         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3833     {
3834         logf("incorrect header");
3835         return false;
3836     }
3837
3838     idx = hdr->indices;
3839
3840     /* Load the master index table. */
3841     for (i = 0; i < hdr->h.tch.entry_count; i++)
3842     {
3843         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3844         if (rc != sizeof(struct index_entry))
3845         {
3846             logf("read error #10");
3847             close(fd);
3848             return false;
3849         }
3850
3851         bytesleft -= sizeof(struct index_entry);
3852         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3853         {
3854             logf("too big tagcache.");
3855             close(fd);
3856             return false;
3857         }
3858
3859         idx++;
3860     }
3861
3862     close(fd);
3863
3864     /* Load the tags. */
3865     p = (char *)idx;
3866     for (tag = 0; tag < TAG_COUNT; tag++)
3867     {
3868         struct tagfile_entry *fe;
3869         char buf[TAG_MAXLEN+32];
3870
3871         if (tagcache_is_numeric_tag(tag))
3872             continue ;
3873
3874         //p = ((void *)p+1);
3875         p = (char *)((long)p & ~0x03) + 0x04;
3876         hdr->tags[tag] = p;
3877
3878         /* Check the header. */
3879         tch = (struct tagcache_header *)p;
3880         p += sizeof(struct tagcache_header);
3881
3882         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3883             return false;
3884
3885         for (hdr->entry_count[tag] = 0;
3886              hdr->entry_count[tag] < tch->entry_count;
3887              hdr->entry_count[tag]++)
3888         {
3889             long pos;
3890
3891             if (do_timed_yield())
3892             {
3893                 /* Abort if we got a critical event in queue */
3894                 if (check_event_queue())
3895                     return false;
3896             }
3897
3898             fe = (struct tagfile_entry *)p;
3899             pos = lseek(fd, 0, SEEK_CUR);
3900             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3901             if (rc != sizeof(struct tagfile_entry))
3902             {
3903                 /* End of lookup table. */
3904                 logf("read error #11");
3905                 close(fd);
3906                 return false;
3907             }
3908
3909             /* We have a special handling for the filename tags. */
3910             if (tag == tag_filename)
3911             {
3912 # ifdef HAVE_DIRCACHE
3913                 const struct dirent *dc;
3914 # endif
3915
3916                 // FIXME: This is wrong!
3917                 // idx = &hdr->indices[hdr->entry_count[i]];
3918                 idx = &hdr->indices[fe->idx_id];
3919
3920                 if (fe->tag_length >= (long)sizeof(buf)-1)
3921                 {
3922                     read(fd, buf, 10);
3923                     buf[10] = '\0';
3924                     logf("TAG:%s", buf);
3925                     logf("too long filename");
3926                     close(fd);
3927                     return false;
3928                 }
3929
3930                 rc = read(fd, buf, fe->tag_length);
3931                 if (rc != fe->tag_length)
3932                 {
3933                     logf("read error #12");
3934                     close(fd);
3935                     return false;
3936                 }
3937
3938                 /* Check if the entry has already been removed */
3939                 if (idx->flag & FLAG_DELETED)
3940                     continue;
3941
3942                 /* This flag must not be used yet. */
3943                 if (idx->flag & FLAG_DIRCACHE)
3944                 {
3945                     logf("internal error!");
3946                     close(fd);
3947                     return false;
3948                 }
3949
3950                 if (idx->tag_seek[tag] != pos)
3951                 {
3952                     logf("corrupt data structures!");
3953                     close(fd);
3954                     return false;
3955                 }
3956
3957 # ifdef HAVE_DIRCACHE
3958                 if (dircache_is_enabled())
3959                 {
3960                     dc = dircache_get_entry_ptr(buf);
3961                     if (dc == NULL)
3962                     {
3963                         logf("Entry no longer valid.");
3964                         logf("-> %s", buf);
3965                         delete_entry(fe->idx_id);
3966                         continue ;
3967                     }
3968
3969                     idx->flag |= FLAG_DIRCACHE;
3970                     FLAG_SET_ATTR(idx->flag, fe->idx_id);
3971                     idx->tag_seek[tag_filename] = (long)dc;
3972                 }
3973                 else
3974 # endif
3975                 {
3976                     /* This will be very slow unless dircache is enabled
3977                        or target is flash based, but do it anyway for
3978                        consistency. */
3979                     /* Check if entry has been removed. */
3980                     if (global_settings.tagcache_autoupdate)
3981                     {
3982                         if (!file_exists(buf))
3983                         {
3984                             logf("Entry no longer valid.");
3985                             logf("-> %s", buf);
3986                             delete_entry(fe->idx_id);
3987                             continue;
3988                         }
3989                     }
3990                 }
3991
3992                 continue ;
3993             }
3994
3995             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3996             if (bytesleft < 0)
3997             {
3998                 logf("too big tagcache #2");
3999                 logf("tl: %d", fe->tag_length);
4000                 logf("bl: %ld", bytesleft);
4001                 close(fd);
4002                 return false;
4003             }
4004
4005             p = fe->tag_data;
4006             rc = read(fd, fe->tag_data, fe->tag_length);
4007             p += rc;
4008
4009             if (rc != fe->tag_length)
4010             {
4011                 logf("read error #13");
4012                 logf("rc=0x%04x", rc); // 0x431
4013                 logf("len=0x%04x", fe->tag_length); // 0x4000
4014                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4015                 logf("tag=0x%02x", tag); // 0x00
4016                 close(fd);
4017                 return false;
4018             }
4019         }
4020         close(fd);
4021     }
4022
4023     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4024     logf("tagcache loaded into ram!");
4025
4026     return true;
4027 }
4028 #endif /* HAVE_TC_RAMCACHE */
4029
4030 static bool check_deleted_files(void)
4031 {
4032     int fd;
4033     char buf[TAG_MAXLEN+32];
4034     struct tagfile_entry tfe;
4035
4036     logf("reverse scan...");
4037     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4038     fd = open(buf, O_RDONLY);
4039
4040     if (fd < 0)
4041     {
4042         logf("%s open fail", buf);
4043         return false;
4044     }
4045
4046     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4047     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4048            == sizeof(struct tagfile_entry)
4049 #ifndef __PCTOOL__
4050            && !check_event_queue()
4051 #endif
4052            )
4053     {
4054         if (tfe.tag_length >= (long)sizeof(buf)-1)
4055         {
4056             logf("too long tag");
4057             close(fd);
4058             return false;
4059         }
4060
4061         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4062         {
4063             logf("read error #14");
4064             close(fd);
4065             return false;
4066         }
4067
4068         /* Check if the file has already deleted from the db. */
4069         if (*buf == '\0')
4070             continue;
4071
4072         /* Now check if the file exists. */
4073         if (!file_exists(buf))
4074         {
4075             logf("Entry no longer valid.");
4076             logf("-> %s / %d", buf, tfe.tag_length);
4077             delete_entry(tfe.idx_id);
4078         }
4079     }
4080
4081     close(fd);
4082
4083     logf("done");
4084
4085     return true;
4086 }
4087
4088 static bool check_dir(const char *dirname, int add_files)
4089 {
4090     DIR *dir;
4091     int len;
4092     int success = false;
4093     int ignore, unignore;
4094     char newpath[MAX_PATH];
4095
4096     dir = opendir(dirname);
4097     if (!dir)
4098     {
4099         logf("tagcache: opendir() failed");
4100         return false;
4101     }
4102
4103     /* check for a database.ignore file */
4104     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4105     ignore = file_exists(newpath);
4106     /* check for a database.unignore file */
4107     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4108     unignore = file_exists(newpath);
4109
4110     /* don't do anything if both ignore and unignore are there */
4111     if (ignore != unignore)
4112         add_files = unignore;
4113
4114     /* Recursively scan the dir. */
4115 #ifdef __PCTOOL__
4116     while (1)
4117 #else
4118     while (!check_event_queue())
4119 #endif
4120     {
4121         struct dirent *entry;
4122
4123         entry = readdir(dir);
4124
4125         if (entry == NULL)
4126         {
4127             success = true;
4128             break ;
4129         }
4130
4131         if (!strcmp((char *)entry->d_name, ".") ||
4132             !strcmp((char *)entry->d_name, ".."))
4133             continue;
4134
4135         yield();
4136
4137         len = strlen(curpath);
4138         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4139                  entry->d_name);
4140
4141         processed_dir_count++;
4142         if (entry->attribute & ATTR_DIRECTORY)
4143             check_dir(curpath, add_files);
4144         else if (add_files)
4145         {
4146             tc_stat.curentry = curpath;
4147
4148             /* Add a new entry to the temporary db file. */
4149             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4150 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4151                          , dir->internal_entry
4152 #endif
4153                          );
4154
4155             /* Wait until current path for debug screen is read and unset. */
4156             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4157                 yield();
4158
4159             tc_stat.curentry = NULL;
4160         }
4161
4162         curpath[len] = '\0';
4163     }
4164
4165     closedir(dir);
4166
4167     return success;
4168 }
4169
4170 void tagcache_screensync_event(void)
4171 {
4172     tc_stat.curentry = NULL;
4173 }
4174
4175 void tagcache_screensync_enable(bool state)
4176 {
4177     tc_stat.syncscreen = state;
4178 }
4179
4180 void tagcache_build(const char *path)
4181 {
4182     struct tagcache_header header;
4183     bool ret;
4184
4185     curpath[0] = '\0';
4186     data_size = 0;
4187     total_entry_count = 0;
4188     processed_dir_count = 0;
4189
4190 #ifdef HAVE_DIRCACHE
4191     while (dircache_is_initializing())
4192         sleep(1);
4193 #endif
4194
4195     logf("updating tagcache");
4196
4197     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4198     if (cachefd >= 0)
4199     {
4200         logf("skipping, cache already waiting for commit");
4201         close(cachefd);
4202         return ;
4203     }
4204
4205     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4206     if (cachefd < 0)
4207     {
4208         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4209         return ;
4210     }
4211
4212     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4213
4214     cpu_boost(true);
4215
4216     logf("Scanning files...");
4217     /* Scan for new files. */
4218     memset(&header, 0, sizeof(struct tagcache_header));
4219     write(cachefd, &header, sizeof(struct tagcache_header));
4220
4221     if (strcmp("/", path) != 0)
4222         strcpy(curpath, path);
4223     ret = check_dir(path, true);
4224
4225     /* Write the header. */
4226     header.magic = TAGCACHE_MAGIC;
4227     header.datasize = data_size;
4228     header.entry_count = total_entry_count;
4229     lseek(cachefd, 0, SEEK_SET);
4230     write(cachefd, &header, sizeof(struct tagcache_header));
4231     close(cachefd);
4232
4233     if (filenametag_fd >= 0)
4234     {
4235         close(filenametag_fd);
4236         filenametag_fd = -1;
4237     }
4238
4239     if (!ret)
4240     {
4241         logf("Aborted.");
4242         cpu_boost(false);
4243         return ;
4244     }
4245
4246     /* Commit changes to the database. */
4247 #ifdef __PCTOOL__
4248     allocate_tempbuf();
4249 #endif
4250     if (commit())
4251     {
4252         remove(TAGCACHE_FILE_TEMP);
4253         logf("tagcache built!");
4254     }
4255 #ifdef __PCTOOL__
4256     free_tempbuf();
4257 #endif
4258
4259 #ifdef HAVE_TC_RAMCACHE
4260     if (hdr)
4261     {
4262         /* Import runtime statistics if we just initialized the db. */
4263         if (hdr->h.serial == 0)
4264             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4265     }
4266 #endif
4267
4268     cpu_boost(false);
4269 }
4270
4271 #ifdef HAVE_TC_RAMCACHE
4272 static void load_ramcache(void)
4273 {
4274     if (!hdr)
4275         return ;
4276
4277     cpu_boost(true);
4278
4279     /* At first we should load the cache (if exists). */
4280     tc_stat.ramcache = load_tagcache();
4281
4282     if (!tc_stat.ramcache)
4283     {
4284         /* If loading failed, it must indicate some problem with the db
4285          * so disable it entirely to prevent further issues. */
4286         tc_stat.ready = false;
4287         hdr = NULL;
4288     }
4289
4290     cpu_boost(false);
4291 }
4292
4293 void tagcache_unload_ramcache(void)
4294 {
4295     tc_stat.ramcache = false;
4296     /* Just to make sure there is no statefile present. */
4297     // remove(TAGCACHE_STATEFILE);
4298 }
4299 #endif
4300
4301 #ifndef __PCTOOL__
4302 static void tagcache_thread(void)
4303 {
4304     struct queue_event ev;
4305     bool check_done = false;
4306
4307     /* If the previous cache build/update was interrupted, commit
4308      * the changes first in foreground. */
4309     cpu_boost(true);
4310     allocate_tempbuf();
4311     commit();
4312     free_tempbuf();
4313
4314 #ifdef HAVE_TC_RAMCACHE
4315 # ifdef HAVE_EEPROM_SETTINGS
4316     if (firmware_settings.initialized && firmware_settings.disk_clean)
4317         check_done = tagcache_dumpload();
4318
4319     remove(TAGCACHE_STATEFILE);
4320 # endif
4321
4322     /* Allocate space for the tagcache if found on disk. */
4323     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4324         allocate_tagcache();
4325 #endif
4326
4327     cpu_boost(false);
4328     tc_stat.initialized = true;
4329
4330     /* Don't delay bootup with the header check but do it on background. */
4331     sleep(HZ);
4332     tc_stat.ready = check_all_headers();
4333     tc_stat.readyvalid = true;
4334
4335     while (1)
4336     {
4337         run_command_queue(false);
4338
4339         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4340
4341         switch (ev.id)
4342         {
4343             case Q_IMPORT_CHANGELOG:
4344                 tagcache_import_changelog();
4345                 break;
4346
4347             case Q_REBUILD:
4348                 remove_files();
4349                 remove(TAGCACHE_FILE_TEMP);
4350                 tagcache_build("/");
4351                 break;
4352
4353             case Q_UPDATE:
4354                 tagcache_build("/");
4355 #ifdef HAVE_TC_RAMCACHE
4356                 load_ramcache();
4357 #endif
4358                 check_deleted_files();
4359                 break ;
4360
4361             case Q_START_SCAN:
4362                 check_done = false;
4363             case SYS_TIMEOUT:
4364                 if (check_done || !tc_stat.ready)
4365                     break ;
4366
4367 #ifdef HAVE_TC_RAMCACHE
4368                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4369                 {
4370                     load_ramcache();
4371                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4372                         tagcache_build("/");
4373                 }
4374                 else
4375 #endif
4376                 if (global_settings.tagcache_autoupdate)
4377                 {
4378                     tagcache_build("/");
4379
4380                     /* This will be very slow unless dircache is enabled
4381                        or target is flash based, but do it anyway for
4382                        consistency. */
4383                     check_deleted_files();
4384                 }
4385
4386                 logf("tagcache check done");
4387
4388                 check_done = true;
4389                 break ;
4390
4391             case Q_STOP_SCAN:
4392                 break ;
4393
4394             case SYS_POWEROFF:
4395                 break ;
4396
4397 #ifndef SIMULATOR
4398             case SYS_USB_CONNECTED:
4399                 logf("USB: TagCache");
4400                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4401                 usb_wait_for_disconnect(&tagcache_queue);
4402                 break ;
4403 #endif
4404         }
4405     }
4406 }
4407
4408 bool tagcache_prepare_shutdown(void)
4409 {
4410     if (tagcache_get_commit_step() > 0)
4411         return false;
4412
4413     tagcache_stop_scan();
4414     while (read_lock || write_lock)
4415         sleep(1);
4416
4417     return true;
4418 }
4419
4420 void tagcache_shutdown(void)
4421 {
4422     /* Flush the command queue. */
4423     run_command_queue(true);
4424
4425 #ifdef HAVE_EEPROM_SETTINGS
4426     if (tc_stat.ramcache)
4427         tagcache_dumpsave();
4428 #endif
4429 }
4430
4431 static int get_progress(void)
4432 {
4433     int total_count = -1;
4434
4435 #ifdef HAVE_DIRCACHE
4436     if (dircache_is_enabled())
4437     {
4438         total_count = dircache_get_entry_count();
4439     }
4440     else
4441 #endif
4442 #ifdef HAVE_TC_RAMCACHE
4443     {
4444         if (hdr && tc_stat.ramcache)
4445             total_count = hdr->h.tch.entry_count;
4446     }
4447 #endif
4448
4449     if (total_count < 0)
4450         return -1;
4451
4452     return processed_dir_count * 100 / total_count;
4453 }
4454
4455 struct tagcache_stat* tagcache_get_stat(void)
4456 {
4457     tc_stat.progress = get_progress();
4458     tc_stat.processed_entries = processed_dir_count;
4459
4460     return &tc_stat;
4461 }
4462
4463 void tagcache_start_scan(void)
4464 {
4465     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4466 }
4467
4468 bool tagcache_update(void)
4469 {
4470     if (!tc_stat.ready)
4471         return false;
4472
4473     queue_post(&tagcache_queue, Q_UPDATE, 0);
4474     return false;
4475 }
4476
4477 bool tagcache_rebuild()
4478 {
4479     queue_post(&tagcache_queue, Q_REBUILD, 0);
4480     return false;
4481 }
4482
4483 void tagcache_stop_scan(void)
4484 {
4485     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4486 }
4487
4488 #ifdef HAVE_TC_RAMCACHE
4489 bool tagcache_is_ramcache(void)
4490 {
4491     return tc_stat.ramcache;
4492 }
4493 #endif
4494
4495 #endif /* !__PCTOOL__ */
4496
4497
4498 void tagcache_init(void)
4499 {
4500     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4501     memset(&current_tcmh, 0, sizeof(struct master_header));
4502     filenametag_fd = -1;
4503     write_lock = read_lock = 0;
4504
4505 #ifndef __PCTOOL__
4506     mutex_init(&command_queue_mutex);
4507     queue_init(&tagcache_queue, true);
4508     create_thread(tagcache_thread, tagcache_stack,
4509                   sizeof(tagcache_stack), 0, tagcache_thread_name
4510                   IF_PRIO(, PRIORITY_BACKGROUND)
4511                           IF_COP(, CPU));
4512 #else
4513     tc_stat.initialized = true;
4514     allocate_tempbuf();
4515     commit();
4516     free_tempbuf();
4517     tc_stat.ready = check_all_headers();
4518 #endif
4519 }
4520
4521 #ifdef __PCTOOL__
4522 void tagcache_reverse_scan(void)
4523 {
4524     logf("Checking for deleted files");
4525     check_deleted_files();
4526 }
4527 #endif
4528
4529 bool tagcache_is_initialized(void)
4530 {
4531     return tc_stat.initialized;
4532 }
4533 bool tagcache_is_usable(void)
4534 {
4535     return tc_stat.initialized && tc_stat.ready;
4536 }
4537 int tagcache_get_commit_step(void)
4538 {
4539     return tc_stat.commit_step;
4540 }
4541 int tagcache_get_max_commit_step(void)
4542 {
4543     return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4544 }
4545