Bump version numbers for 3.13
[maemo-rb.git] / apps / tagcache.c
blob07d8d1d7a2050bef8c8f3a5bc1b5599909b67742
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
20 ****************************************************************************/
23 * TagCache API
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
58 /*#define LOGF_ENABLE*/
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #ifdef APPLICATION
64 #include <unistd.h> /* readlink() */
65 #include <limits.h> /* PATH_MAX */
66 #endif
67 #include "config.h"
68 #include "ata_idle_notify.h"
69 #include "thread.h"
70 #include "kernel.h"
71 #include "system.h"
72 #include "logf.h"
73 #include "string-extra.h"
74 #include "usb.h"
75 #include "metadata.h"
76 #include "tagcache.h"
77 #include "core_alloc.h"
78 #include "crc32.h"
79 #include "misc.h"
80 #include "settings.h"
81 #include "dir.h"
82 #include "filefuncs.h"
83 #include "structec.h"
84 #include "debug.h"
86 #ifndef __PCTOOL__
87 #include "lang.h"
88 #include "eeprom_settings.h"
89 #endif
91 #ifdef __PCTOOL__
92 #define yield() do { } while(0)
93 #define sim_sleep(timeout) do { } while(0)
94 #define do_timed_yield() do { } while(0)
95 #endif
97 #ifndef __PCTOOL__
98 /* Tag Cache thread. */
99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
101 static const char tagcache_thread_name[] = "tagcache";
102 #endif
104 /* Previous path when scanning directory tree recursively. */
105 static char curpath[TAG_MAXLEN+32];
107 /* Used when removing duplicates. */
108 static char *tempbuf; /* Allocated when needed. */
109 static long tempbufidx; /* Current location in buffer. */
110 static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
111 static long tempbuf_left; /* Buffer space left. */
112 static long tempbuf_pos;
114 #define SORTED_TAGS_COUNT 8
115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
118 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
119 /* Tags we want to get sorted (loaded to the tempbuf). */
120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
121 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
122 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
126 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
127 (1LU << tag_albumartist) | (1LU << tag_grouping))
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat;
138 /* Queue commands. */
139 enum tagcache_queue {
140 Q_STOP_SCAN = 0,
141 Q_START_SCAN,
142 Q_IMPORT_CHANGELOG,
143 Q_UPDATE,
144 Q_REBUILD,
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER,
148 CMD_UPDATE_NUMERIC,
151 struct tagcache_command_entry {
152 int32_t command;
153 int32_t idx_id;
154 int32_t tag;
155 int32_t data;
158 #ifndef __PCTOOL__
159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
160 static volatile int command_queue_widx = 0;
161 static volatile int command_queue_ridx = 0;
162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
163 #endif
165 /* Tag database structures. */
167 /* Variable-length tag entry in tag files. */
168 struct tagfile_entry {
169 int32_t tag_length; /* Length of the data in bytes including '\0' */
170 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
171 char tag_data[0]; /* Begin of the tag data */
174 /* Fixed-size tag entry in master db index. */
175 struct index_entry {
176 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
177 int32_t flag; /* Status flags */
180 /* Header is the same in every file. */
181 struct tagcache_header {
182 int32_t magic; /* Header version number */
183 int32_t datasize; /* Data size in bytes */
184 int32_t entry_count; /* Number of entries in this file */
187 struct master_header {
188 struct tagcache_header tch;
189 int32_t serial; /* Increasing counting number */
190 int32_t commitid; /* Number of commits so far */
191 int32_t dirty;
194 /* For the endianess correction */
195 static const char * const tagfile_entry_ec = "ll";
197 Note: This should be (1 + TAG_COUNT) amount of l's.
199 static const char * const index_entry_ec = "llllllllllllllllllllll";
201 static const char * const tagcache_header_ec = "lll";
202 static const char * const master_header_ec = "llllll";
204 static struct master_header current_tcmh;
206 #ifdef HAVE_TC_RAMCACHE
207 /* Header is created when loading database to ram. */
208 struct ramcache_header {
209 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
210 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
211 struct index_entry indices[0]; /* Master index file content */
214 # ifdef HAVE_EEPROM_SETTINGS
215 struct statefile_header {
216 int32_t magic; /* Statefile version number */
217 struct master_header mh; /* Header from the master index */
218 struct ramcache_header *hdr; /* Old load address of hdr for relocation */
219 struct tagcache_stat tc_stat;
221 # endif
223 /* Pointer to allocated ramcache_header */
224 static struct ramcache_header *ramcache_hdr;
225 /* lock entity to temporarily prevent ramcache_hdr from moving */
226 static int move_lock;
227 #endif
229 /**
230 * Full tag entries stored in a temporary file waiting
231 * for commit to the cache. */
232 struct temp_file_entry {
233 long tag_offset[TAG_COUNT];
234 short tag_length[TAG_COUNT];
235 long flag;
237 long data_length;
240 struct tempbuf_id_list {
241 long id;
242 struct tempbuf_id_list *next;
245 struct tempbuf_searchidx {
246 long idx_id;
247 char *str;
248 int seek;
249 struct tempbuf_id_list idlist;
252 /* Lookup buffer for fixing messed up index while after sorting. */
253 static long commit_entry_count;
254 static long lookup_buffer_depth;
255 static struct tempbuf_searchidx **lookup;
257 /* Used when building the temporary file. */
258 static int cachefd = -1, filenametag_fd;
259 static int total_entry_count = 0;
260 static int data_size = 0;
261 static int processed_dir_count;
263 /* Thread safe locking */
264 static volatile int write_lock;
265 static volatile int read_lock;
267 static bool delete_entry(long idx_id);
269 const char* tagcache_tag_to_str(int tag)
271 return tags_str[tag];
274 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
275 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
277 return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
280 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
282 return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
285 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
287 return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
290 #ifdef HAVE_DIRCACHE
292 * Returns true if specified flag is still present, i.e., dircache
293 * has not been reloaded.
295 static bool is_dircache_intact(void)
297 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
299 #endif
301 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
303 int fd;
304 char buf[MAX_PATH];
305 int rc;
307 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
308 return -1;
310 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
312 fd = open(buf, write ? O_RDWR : O_RDONLY);
313 if (fd < 0)
315 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
316 tc_stat.ready = false;
317 return fd;
320 /* Check the header. */
321 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
322 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
324 logf("header error");
325 tc_stat.ready = false;
326 close(fd);
327 return -2;
330 return fd;
333 static int open_master_fd(struct master_header *hdr, bool write)
335 int fd;
336 int rc;
338 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
339 if (fd < 0)
341 logf("master file open failed for R/W");
342 tc_stat.ready = false;
343 return fd;
346 tc_stat.econ = false;
348 /* Check the header. */
349 rc = read(fd, hdr, sizeof(struct master_header));
350 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
352 /* Success. */
353 return fd;
356 /* Trying to read again, this time with endianess correction enabled. */
357 lseek(fd, 0, SEEK_SET);
359 rc = ecread(fd, hdr, 1, master_header_ec, true);
360 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
362 logf("header error");
363 tc_stat.ready = false;
364 close(fd);
365 return -2;
368 tc_stat.econ = true;
370 return fd;
373 #ifndef __PCTOOL__
374 static bool do_timed_yield(void)
376 /* Sorting can lock up for quite a while, so yield occasionally */
377 static long wakeup_tick = 0;
378 if (TIME_AFTER(current_tick, wakeup_tick))
380 wakeup_tick = current_tick + (HZ/4);
381 yield();
382 return true;
384 return false;
386 #endif
388 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
389 /* find the ramcache entry corresponding to the file indicated by
390 * filename and dc (it's corresponding dircache id). */
391 static long find_entry_ram(const char *filename, int dc)
393 static long last_pos = 0;
394 int i;
396 /* Check if tagcache is loaded into ram. */
397 if (!tc_stat.ramcache || !is_dircache_intact())
398 return -1;
400 if (dc < 0)
401 dc = dircache_get_entry_id(filename);
403 if (dc < 0)
405 logf("tagcache: file not found.");
406 return -1;
409 try_again:
411 if (last_pos > 0)
412 i = last_pos;
413 else
414 i = 0;
416 for (; i < current_tcmh.tch.entry_count; i++)
418 if (ramcache_hdr->indices[i].tag_seek[tag_filename] == dc)
420 last_pos = MAX(0, i - 3);
421 return i;
424 do_timed_yield();
427 if (last_pos > 0)
429 last_pos = 0;
430 goto try_again;
433 return -1;
435 #endif
437 static long find_entry_disk(const char *filename_raw, bool localfd)
439 struct tagcache_header tch;
440 static long last_pos = -1;
441 long pos_history[POS_HISTORY_COUNT];
442 long pos_history_idx = 0;
443 bool found = false;
444 struct tagfile_entry tfe;
445 int fd;
446 char buf[TAG_MAXLEN+32];
447 int i;
448 int pos = -1;
450 const char *filename = filename_raw;
451 #ifdef APPLICATION
452 char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
453 if (realpath(filename, pathbuf) == pathbuf)
454 filename = pathbuf;
455 #endif
457 if (!tc_stat.ready)
458 return -2;
460 fd = filenametag_fd;
461 if (fd < 0 || localfd)
463 last_pos = -1;
464 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
465 return -1;
468 check_again:
470 if (last_pos > 0)
471 lseek(fd, last_pos, SEEK_SET);
472 else
473 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
475 while (true)
477 pos = lseek(fd, 0, SEEK_CUR);
478 for (i = pos_history_idx-1; i >= 0; i--)
479 pos_history[i+1] = pos_history[i];
480 pos_history[0] = pos;
482 if (ecread_tagfile_entry(fd, &tfe)
483 != sizeof(struct tagfile_entry))
485 break ;
488 if (tfe.tag_length >= (long)sizeof(buf))
490 logf("too long tag #1");
491 close(fd);
492 if (!localfd)
493 filenametag_fd = -1;
494 last_pos = -1;
495 return -2;
498 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
500 logf("read error #2");
501 close(fd);
502 if (!localfd)
503 filenametag_fd = -1;
504 last_pos = -1;
505 return -3;
508 if (!strcmp(filename, buf))
510 last_pos = pos_history[pos_history_idx];
511 found = true;
512 break ;
515 if (pos_history_idx < POS_HISTORY_COUNT - 1)
516 pos_history_idx++;
519 /* Not found? */
520 if (!found)
522 if (last_pos > 0)
524 last_pos = -1;
525 logf("seek again");
526 goto check_again;
529 if (fd != filenametag_fd || localfd)
530 close(fd);
531 return -4;
534 if (fd != filenametag_fd || localfd)
535 close(fd);
537 return tfe.idx_id;
540 static int find_index(const char *filename)
542 long idx_id = -1;
544 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
545 idx_id = find_entry_ram(filename, -1);
546 #endif
548 if (idx_id < 0)
549 idx_id = find_entry_disk(filename, true);
551 return idx_id;
554 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
556 int idx_id;
558 if (!tc_stat.ready)
559 return false;
561 idx_id = find_index(filename);
562 if (idx_id < 0)
563 return false;
565 if (!tagcache_search(tcs, tag_filename))
566 return false;
568 tcs->entry_count = 0;
569 tcs->idx_id = idx_id;
571 return true;
574 static bool get_index(int masterfd, int idxid,
575 struct index_entry *idx, bool use_ram)
577 bool localfd = false;
579 if (idxid < 0)
581 logf("Incorrect idxid: %d", idxid);
582 return false;
585 #ifdef HAVE_TC_RAMCACHE
586 if (tc_stat.ramcache && use_ram)
588 if (ramcache_hdr->indices[idxid].flag & FLAG_DELETED)
589 return false;
591 # ifdef HAVE_DIRCACHE
592 if (!(ramcache_hdr->indices[idxid].flag & FLAG_DIRCACHE)
593 || is_dircache_intact())
594 #endif
596 memcpy(idx, &ramcache_hdr->indices[idxid], sizeof(struct index_entry));
597 return true;
600 #else
601 (void)use_ram;
602 #endif
604 if (masterfd < 0)
606 struct master_header tcmh;
608 localfd = true;
609 masterfd = open_master_fd(&tcmh, false);
610 if (masterfd < 0)
611 return false;
614 lseek(masterfd, idxid * sizeof(struct index_entry)
615 + sizeof(struct master_header), SEEK_SET);
616 if (ecread_index_entry(masterfd, idx)
617 != sizeof(struct index_entry))
619 logf("read error #3");
620 if (localfd)
621 close(masterfd);
623 return false;
626 if (localfd)
627 close(masterfd);
629 if (idx->flag & FLAG_DELETED)
630 return false;
632 return true;
635 #ifndef __PCTOOL__
637 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
639 /* We need to exclude all memory only flags & tags when writing to disk. */
640 if (idx->flag & FLAG_DIRCACHE)
642 logf("memory only flags!");
643 return false;
646 #ifdef HAVE_TC_RAMCACHE
647 /* Only update numeric data. Writing the whole index to RAM by memcpy
648 * destroys dircache pointers!
650 if (tc_stat.ramcache)
652 int tag;
653 struct index_entry *idx_ram = &ramcache_hdr->indices[idxid];
655 for (tag = 0; tag < TAG_COUNT; tag++)
657 if (TAGCACHE_IS_NUMERIC(tag))
659 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
663 /* Don't touch the dircache flag or attributes. */
664 idx_ram->flag = (idx->flag & 0x0000ffff)
665 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
667 #endif
669 lseek(masterfd, idxid * sizeof(struct index_entry)
670 + sizeof(struct master_header), SEEK_SET);
671 if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
673 logf("write error #3");
674 logf("idxid: %d", idxid);
675 return false;
678 return true;
681 #endif /* !__PCTOOL__ */
683 static bool open_files(struct tagcache_search *tcs, int tag)
685 if (tcs->idxfd[tag] < 0)
687 char fn[MAX_PATH];
689 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
690 tcs->idxfd[tag] = open(fn, O_RDONLY);
693 if (tcs->idxfd[tag] < 0)
695 logf("File not open!");
696 return false;
699 return true;
702 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
703 int tag, char *buf, long size)
705 struct tagfile_entry tfe;
706 long seek;
708 *buf = '\0';
710 if (TAGCACHE_IS_NUMERIC(tag))
711 return false;
713 seek = idx->tag_seek[tag];
714 if (seek < 0)
716 logf("Retrieve failed");
717 return false;
720 #ifdef HAVE_TC_RAMCACHE
721 if (tcs->ramsearch)
723 struct tagfile_entry *ep;
725 # ifdef HAVE_DIRCACHE
726 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE))
728 /* for tag_filename, seek is a dircache index */
729 if (is_dircache_intact())
731 dircache_copy_path(seek, buf, size);
732 return true;
734 else
736 /* The seek is useless now, there's nothing we can return. */
737 logf("retrieve: dircache gone, cannot read file name");
738 tagcache_unload_ramcache();
739 // XXX do this when there's a way to not trigger an
740 // update before reloading:
741 // tagcache_start_scan();
742 return false;
745 else
746 # endif
747 if (tag != tag_filename)
749 ep = (struct tagfile_entry *)&ramcache_hdr->tags[tag][seek];
750 strlcpy(buf, ep->tag_data, size);
752 return true;
755 #endif
757 if (!open_files(tcs, tag))
758 return false;
760 lseek(tcs->idxfd[tag], seek, SEEK_SET);
761 if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
762 != sizeof(struct tagfile_entry))
764 logf("read error #5");
765 return false;
768 if (tfe.tag_length >= size)
770 logf("too small buffer");
771 return false;
774 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
775 tfe.tag_length)
777 logf("read error #6");
778 return false;
781 buf[tfe.tag_length] = '\0';
783 return true;
786 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
788 static long find_tag(int tag, int idx_id, const struct index_entry *idx)
790 #ifndef __PCTOOL__
791 if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag))
793 /* Attempt to find tag data through store-to-load forwarding in
794 command queue */
795 long result = -1;
797 mutex_lock(&command_queue_mutex);
799 int ridx = command_queue_widx;
801 while (ridx != command_queue_ridx)
803 if (--ridx < 0)
804 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
806 if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
807 && command_queue[ridx].idx_id == idx_id
808 && command_queue[ridx].tag == tag)
810 result = command_queue[ridx].data;
811 break;
815 mutex_unlock(&command_queue_mutex);
817 if (result >= 0)
819 logf("find_tag: "
820 "Recovered tag %d value %lX from write queue",
821 tag, result);
822 return result;
825 #else
826 (void)idx_id;
827 #endif
829 return idx->tag_seek[tag];
833 static long check_virtual_tags(int tag, int idx_id,
834 const struct index_entry *idx)
836 long data = 0;
838 switch (tag)
840 case tag_virt_length_sec:
841 data = (find_tag(tag_length, idx_id, idx)/1000) % 60;
842 break;
844 case tag_virt_length_min:
845 data = (find_tag(tag_length, idx_id, idx)/1000) / 60;
846 break;
848 case tag_virt_playtime_sec:
849 data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60;
850 break;
852 case tag_virt_playtime_min:
853 data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60;
854 break;
856 case tag_virt_autoscore:
857 if (find_tag(tag_length, idx_id, idx) == 0
858 || find_tag(tag_playcount, idx_id, idx) == 0)
860 data = 0;
862 else
864 /* A straight calculus gives:
865 autoscore = 100 * playtime / length / playcout (1)
866 Now, consider the euclidian division of playtime by length:
867 playtime = alpha * length + beta
868 With:
869 0 <= beta < length
870 Now, (1) becomes:
871 autoscore = 100 * (alpha / playcout + beta / length / playcount)
872 Both terms should be small enough to avoid any overflow
874 data = 100 * (find_tag(tag_playtime, idx_id, idx)
875 / find_tag(tag_length, idx_id, idx))
876 + (100 * (find_tag(tag_playtime, idx_id, idx)
877 % find_tag(tag_length, idx_id, idx)))
878 / find_tag(tag_length, idx_id, idx);
879 data /= find_tag(tag_playcount, idx_id, idx);
881 break;
883 /* How many commits before the file has been added to the DB. */
884 case tag_virt_entryage:
885 data = current_tcmh.commitid
886 - find_tag(tag_commitid, idx_id, idx) - 1;
887 break;
889 case tag_virt_basename:
890 tag = tag_filename; /* return filename; caller handles basename */
891 /* FALLTHRU */
893 default:
894 data = find_tag(tag, idx_id, idx);
897 return data;
900 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
902 struct index_entry idx;
904 if (!tc_stat.ready)
905 return false;
907 if (!TAGCACHE_IS_NUMERIC(tag))
908 return -1;
910 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
911 return -2;
913 return check_virtual_tags(tag, tcs->idx_id, &idx);
916 inline static bool str_ends_with(const char *str1, const char *str2)
918 int str_len = strlen(str1);
919 int clause_len = strlen(str2);
921 if (clause_len > str_len)
922 return false;
924 return !strcasecmp(&str1[str_len - clause_len], str2);
927 inline static bool str_oneof(const char *str, const char *list)
929 const char *sep;
930 int l, len = strlen(str);
932 while (*list)
934 sep = strchr(list, '|');
935 l = sep ? (long)sep - (long)list : (int)strlen(list);
936 if ((l==len) && !strncasecmp(str, list, len))
937 return true;
938 list += sep ? l + 1 : l;
941 return false;
944 static bool check_against_clause(long numeric, const char *str,
945 const struct tagcache_search_clause *clause)
947 if (clause->numeric)
949 switch (clause->type)
951 case clause_is:
952 return numeric == clause->numeric_data;
953 case clause_is_not:
954 return numeric != clause->numeric_data;
955 case clause_gt:
956 return numeric > clause->numeric_data;
957 case clause_gteq:
958 return numeric >= clause->numeric_data;
959 case clause_lt:
960 return numeric < clause->numeric_data;
961 case clause_lteq:
962 return numeric <= clause->numeric_data;
963 default:
964 logf("Incorrect numeric tag: %d", clause->type);
967 else
969 switch (clause->type)
971 case clause_is:
972 return !strcasecmp(clause->str, str);
973 case clause_is_not:
974 return strcasecmp(clause->str, str);
975 case clause_gt:
976 return 0>strcasecmp(clause->str, str);
977 case clause_gteq:
978 return 0>=strcasecmp(clause->str, str);
979 case clause_lt:
980 return 0<strcasecmp(clause->str, str);
981 case clause_lteq:
982 return 0<=strcasecmp(clause->str, str);
983 case clause_contains:
984 return (strcasestr(str, clause->str) != NULL);
985 case clause_not_contains:
986 return (strcasestr(str, clause->str) == NULL);
987 case clause_begins_with:
988 return (strcasestr(str, clause->str) == str);
989 case clause_not_begins_with:
990 return (strcasestr(str, clause->str) != str);
991 case clause_ends_with:
992 return str_ends_with(str, clause->str);
993 case clause_not_ends_with:
994 return !str_ends_with(str, clause->str);
995 case clause_oneof:
996 return str_oneof(str, clause->str);
998 default:
999 logf("Incorrect tag: %d", clause->type);
1003 return false;
1006 static bool check_clauses(struct tagcache_search *tcs,
1007 struct index_entry *idx,
1008 struct tagcache_search_clause **clauses, int count)
1010 int i;
1012 /* Go through all conditional clauses. */
1013 for (i = 0; i < count; i++)
1015 int seek;
1016 char buf[256];
1017 char *str = buf;
1018 struct tagcache_search_clause *clause = clauses[i];
1020 if (clause->type == clause_logical_or)
1021 break; /* all conditions before logical-or satisfied --
1022 stop processing clauses */
1024 seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1026 #ifdef HAVE_TC_RAMCACHE
1027 if (tcs->ramsearch)
1029 struct tagfile_entry *tfe;
1031 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1033 if (clause->tag == tag_filename
1034 || clause->tag == tag_virt_basename)
1036 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1038 else
1040 tfe = (struct tagfile_entry *)
1041 &ramcache_hdr->tags[clause->tag][seek];
1042 /* str points to movable data, but no locking required here,
1043 * as no yield() is following */
1044 str = tfe->tag_data;
1048 else
1049 #endif
1051 struct tagfile_entry tfe;
1053 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1055 int tag = clause->tag;
1056 if (tag == tag_virt_basename)
1057 tag = tag_filename;
1059 int fd = tcs->idxfd[tag];
1060 lseek(fd, seek, SEEK_SET);
1061 ecread_tagfile_entry(fd, &tfe);
1062 if (tfe.tag_length >= (int)sizeof(buf))
1064 logf("Too long tag read!");
1065 return false;
1068 read(fd, str, tfe.tag_length);
1069 str[tfe.tag_length] = '\0';
1071 /* Check if entry has been deleted. */
1072 if (str[0] == '\0')
1073 return false;
1077 if (clause->tag == tag_virt_basename)
1079 char *basename = strrchr(str, '/');
1080 if (basename)
1081 str = basename + 1;
1084 if (!check_against_clause(seek, str, clause))
1086 /* Clause failed -- try finding a logical-or clause */
1087 while (++i < count)
1089 if (clauses[i]->type == clause_logical_or)
1090 break;
1093 if (i < count) /* Found logical-or? */
1094 continue; /* Check clauses after logical-or */
1096 return false;
1100 return true;
1103 bool tagcache_check_clauses(struct tagcache_search *tcs,
1104 struct tagcache_search_clause **clause, int count)
1106 struct index_entry idx;
1108 if (count == 0)
1109 return true;
1111 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1112 return false;
1114 return check_clauses(tcs, &idx, clause, count);
1117 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1119 int i;
1121 /* If uniq buffer is not defined we must return true for search to work. */
1122 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1123 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1125 return true;
1128 for (i = 0; i < tcs->unique_list_count; i++)
1130 /* Return false if entry is found. */
1131 if (tcs->unique_list[i] == id)
1132 return false;
1135 if (tcs->unique_list_count < tcs->unique_list_capacity)
1137 tcs->unique_list[i] = id;
1138 tcs->unique_list_count++;
1141 return true;
1144 static bool build_lookup_list(struct tagcache_search *tcs)
1146 struct index_entry entry;
1147 int i, j;
1149 tcs->seek_list_count = 0;
1151 #ifdef HAVE_TC_RAMCACHE
1152 if (tcs->ramsearch
1153 # ifdef HAVE_DIRCACHE
1154 && (tcs->type != tag_filename || is_dircache_intact())
1155 # endif
1158 move_lock++; /* lock because below makes a pointer to movable data */
1159 for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++)
1161 struct tagcache_seeklist_entry *seeklist;
1162 /* idx points to movable data, don't yield or reload */
1163 struct index_entry *idx = &ramcache_hdr->indices[i];
1164 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1165 break ;
1167 /* Skip deleted files. */
1168 if (idx->flag & FLAG_DELETED)
1169 continue;
1171 /* Go through all filters.. */
1172 for (j = 0; j < tcs->filter_count; j++)
1174 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1176 break ;
1180 if (j < tcs->filter_count)
1181 continue ;
1183 /* Check for conditions. */
1184 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1185 continue;
1186 /* Add to the seek list if not already in uniq buffer (doesn't yield)*/
1187 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1188 continue;
1190 /* Lets add it. */
1191 seeklist = &tcs->seeklist[tcs->seek_list_count];
1192 seeklist->seek = idx->tag_seek[tcs->type];
1193 seeklist->flag = idx->flag;
1194 seeklist->idx_id = i;
1195 tcs->seek_list_count++;
1197 move_lock--;
1199 tcs->seek_pos = i;
1201 return tcs->seek_list_count > 0;
1203 #endif
1205 if (tcs->masterfd < 0)
1207 struct master_header tcmh;
1208 tcs->masterfd = open_master_fd(&tcmh, false);
1211 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1212 sizeof(struct master_header), SEEK_SET);
1214 while (ecread_index_entry(tcs->masterfd, &entry)
1215 == sizeof(struct index_entry))
1217 struct tagcache_seeklist_entry *seeklist;
1219 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1220 break ;
1222 i = tcs->seek_pos;
1223 tcs->seek_pos++;
1225 /* Check if entry has been deleted. */
1226 if (entry.flag & FLAG_DELETED)
1227 continue;
1229 /* Go through all filters.. */
1230 for (j = 0; j < tcs->filter_count; j++)
1232 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1233 break ;
1236 if (j < tcs->filter_count)
1237 continue ;
1239 /* Check for conditions. */
1240 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1241 continue;
1243 /* Add to the seek list if not already in uniq buffer. */
1244 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1245 continue;
1247 /* Lets add it. */
1248 seeklist = &tcs->seeklist[tcs->seek_list_count];
1249 seeklist->seek = entry.tag_seek[tcs->type];
1250 seeklist->flag = entry.flag;
1251 seeklist->idx_id = i;
1252 tcs->seek_list_count++;
1254 yield();
1257 return tcs->seek_list_count > 0;
1261 static void remove_files(void)
1263 int i;
1264 char buf[MAX_PATH];
1266 tc_stat.ready = false;
1267 tc_stat.ramcache = false;
1268 tc_stat.econ = false;
1269 remove(TAGCACHE_FILE_MASTER);
1270 for (i = 0; i < TAG_COUNT; i++)
1272 if (TAGCACHE_IS_NUMERIC(i))
1273 continue;
1275 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1276 remove(buf);
1281 static bool check_all_headers(void)
1283 struct master_header myhdr;
1284 struct tagcache_header tch;
1285 int tag;
1286 int fd;
1288 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1289 return false;
1291 close(fd);
1292 if (myhdr.dirty)
1294 logf("tagcache is dirty!");
1295 return false;
1298 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1300 for (tag = 0; tag < TAG_COUNT; tag++)
1302 if (TAGCACHE_IS_NUMERIC(tag))
1303 continue;
1305 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1306 return false;
1308 close(fd);
1311 return true;
1314 bool tagcache_search(struct tagcache_search *tcs, int tag)
1316 struct tagcache_header tag_hdr;
1317 struct master_header master_hdr;
1318 int i;
1320 while (read_lock)
1321 sleep(1);
1323 memset(tcs, 0, sizeof(struct tagcache_search));
1324 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1325 return false;
1327 tcs->position = sizeof(struct tagcache_header);
1328 tcs->type = tag;
1329 tcs->seek_pos = 0;
1330 tcs->list_position = 0;
1331 tcs->seek_list_count = 0;
1332 tcs->filter_count = 0;
1333 tcs->masterfd = -1;
1335 for (i = 0; i < TAG_COUNT; i++)
1336 tcs->idxfd[i] = -1;
1338 #ifndef HAVE_TC_RAMCACHE
1339 tcs->ramsearch = false;
1340 #else
1341 tcs->ramsearch = tc_stat.ramcache;
1342 if (tcs->ramsearch)
1344 tcs->entry_count = ramcache_hdr->entry_count[tcs->type];
1346 else
1347 #endif
1349 /* Always open as R/W so we can pass tcs to functions that modify data also
1350 * without failing. */
1351 tcs->masterfd = open_master_fd(&master_hdr, true);
1352 if (tcs->masterfd < 0)
1353 return false;
1355 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1357 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1358 if (tcs->idxfd[tcs->type] < 0)
1359 return false;
1361 tcs->entry_count = tag_hdr.entry_count;
1363 else
1365 tcs->entry_count = master_hdr.tch.entry_count;
1369 tcs->valid = true;
1370 tcs->initialized = true;
1371 write_lock++;
1373 return true;
1376 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1377 void *buffer, long length)
1379 tcs->unique_list = (unsigned long *)buffer;
1380 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1381 tcs->unique_list_count = 0;
1384 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1385 int tag, int seek)
1387 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1388 return false;
1390 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1391 return false;
1393 tcs->filter_tag[tcs->filter_count] = tag;
1394 tcs->filter_seek[tcs->filter_count] = seek;
1395 tcs->filter_count++;
1397 return true;
1400 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1401 struct tagcache_search_clause *clause)
1403 int i;
1405 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1407 logf("Too many clauses");
1408 return false;
1411 if (clause->type != clause_logical_or)
1413 /* Check if there is already a similar filter in present (filters are
1414 * much faster than clauses).
1416 for (i = 0; i < tcs->filter_count; i++)
1418 if (tcs->filter_tag[i] == clause->tag)
1419 return true;
1422 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1424 char buf[MAX_PATH];
1426 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1427 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1431 tcs->clause[tcs->clause_count] = clause;
1432 tcs->clause_count++;
1434 return true;
1437 static bool get_next(struct tagcache_search *tcs)
1439 static char buf[TAG_MAXLEN+32];
1440 struct tagfile_entry entry;
1441 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1442 long flag = 0;
1443 #endif
1445 if (!tcs->valid || !tc_stat.ready)
1446 return false;
1448 if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1449 #ifdef HAVE_TC_RAMCACHE
1450 && !tcs->ramsearch
1451 #endif
1453 return false;
1455 /* Relative fetch. */
1456 if (tcs->filter_count > 0 || tcs->clause_count > 0
1457 || TAGCACHE_IS_NUMERIC(tcs->type)
1458 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1459 /* We need to retrieve flag status for dircache. */
1460 || (tcs->ramsearch && tcs->type == tag_filename)
1461 #endif
1464 struct tagcache_seeklist_entry *seeklist;
1466 /* Check for end of list. */
1467 if (tcs->list_position == tcs->seek_list_count)
1469 tcs->list_position = 0;
1471 /* Try to fetch more. */
1472 if (!build_lookup_list(tcs))
1474 tcs->valid = false;
1475 return false;
1479 seeklist = &tcs->seeklist[tcs->list_position];
1480 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1481 flag = seeklist->flag;
1482 #endif
1483 tcs->position = seeklist->seek;
1484 tcs->idx_id = seeklist->idx_id;
1485 tcs->list_position++;
1487 else
1489 if (tcs->entry_count == 0)
1491 tcs->valid = false;
1492 return false;
1495 tcs->entry_count--;
1498 tcs->result_seek = tcs->position;
1500 if (TAGCACHE_IS_NUMERIC(tcs->type))
1502 snprintf(buf, sizeof(buf), "%ld", tcs->position);
1503 tcs->result = buf;
1504 tcs->result_len = strlen(buf) + 1;
1505 return true;
1508 /* Direct fetch. */
1509 #ifdef HAVE_TC_RAMCACHE
1510 if (tcs->ramsearch)
1513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1514 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE))
1516 if (is_dircache_intact())
1518 size_t len = dircache_copy_path(tcs->position, buf, sizeof buf);
1519 tcs->result_len = len + 1;
1520 tcs->result = buf;
1521 tcs->ramresult = false;
1523 return true;
1525 else
1527 /* The seek is useless now, there's nothing we can return. */
1528 logf("get_next: dircache gone, cannot read file name");
1529 tagcache_unload_ramcache();
1530 // XXX do this when there's a way to not trigger an
1531 // update before reloading:
1532 // tagcache_start_scan();
1533 tcs->valid = false;
1534 return false;
1537 else
1538 #endif
1539 if (tcs->type != tag_filename)
1541 struct tagfile_entry *ep;
1543 ep = (struct tagfile_entry *)&ramcache_hdr->tags[tcs->type][tcs->position];
1544 /* don't return ep->tag_data directly as it may move */
1545 tcs->result_len = strlcpy(buf, ep->tag_data, sizeof(buf)) + 1;
1546 tcs->result = buf;
1547 tcs->idx_id = ep->idx_id;
1548 tcs->ramresult = false; /* was true before we copied to buf too */
1550 /* Increase position for the next run. This may get overwritten. */
1551 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1553 return true;
1556 #endif
1558 if (!open_files(tcs, tcs->type))
1560 tcs->valid = false;
1561 return false;
1564 /* Seek stream to the correct position and continue to direct fetch. */
1565 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1567 if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1569 logf("read error #5");
1570 tcs->valid = false;
1571 return false;
1574 if (entry.tag_length > (long)sizeof(buf))
1576 tcs->valid = false;
1577 logf("too long tag #2");
1578 logf("P:%lX/%lX", tcs->position, entry.tag_length);
1579 return false;
1582 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1584 tcs->valid = false;
1585 logf("read error #4");
1586 return false;
1590 Update the position for the next read (this may be overridden
1591 if filters or clauses are being used).
1593 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1594 tcs->result = buf;
1595 tcs->result_len = strlen(tcs->result) + 1;
1596 tcs->idx_id = entry.idx_id;
1597 tcs->ramresult = false;
1599 return true;
1602 bool tagcache_get_next(struct tagcache_search *tcs)
1604 while (get_next(tcs))
1606 if (tcs->result_len > 1)
1607 return true;
1610 return false;
1613 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1614 int tag, char *buf, long size)
1616 struct index_entry idx;
1618 *buf = '\0';
1619 if (!get_index(tcs->masterfd, idxid, &idx, true))
1620 return false;
1622 return retrieve(tcs, &idx, tag, buf, size);
1625 static bool update_master_header(void)
1627 struct master_header myhdr;
1628 int fd;
1630 if (!tc_stat.ready)
1631 return false;
1633 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1634 return false;
1636 myhdr.serial = current_tcmh.serial;
1637 myhdr.commitid = current_tcmh.commitid;
1638 myhdr.dirty = current_tcmh.dirty;
1640 /* Write it back */
1641 lseek(fd, 0, SEEK_SET);
1642 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1643 close(fd);
1645 return true;
1648 void tagcache_search_finish(struct tagcache_search *tcs)
1650 int i;
1652 if (!tcs->initialized)
1653 return;
1655 if (tcs->masterfd >= 0)
1657 close(tcs->masterfd);
1658 tcs->masterfd = -1;
1661 for (i = 0; i < TAG_COUNT; i++)
1663 if (tcs->idxfd[i] >= 0)
1665 close(tcs->idxfd[i]);
1666 tcs->idxfd[i] = -1;
1670 tcs->ramsearch = false;
1671 tcs->valid = false;
1672 tcs->initialized = 0;
1673 if (write_lock > 0)
1674 write_lock--;
1677 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1678 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1680 return (struct tagfile_entry *)&ramcache_hdr->tags[tag][entry->tag_seek[tag]];
1683 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1685 return check_virtual_tags(tag, idx_id, entry);
1688 static char* get_tag_string(const struct index_entry *entry, int tag)
1690 char* s = get_tag(entry, tag)->tag_data;
1691 return strcmp(s, UNTAGGED) ? s : NULL;
1694 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1696 struct index_entry *entry;
1697 int idx_id;
1699 if (!tc_stat.ready || !tc_stat.ramcache)
1700 return false;
1702 /* Find the corresponding entry in tagcache. */
1703 idx_id = find_entry_ram(filename, -1);
1704 if (idx_id < 0)
1705 return false;
1707 entry = &ramcache_hdr->indices[idx_id];
1709 memset(id3, 0, sizeof(struct mp3entry));
1710 char* buf = id3->id3v2buf;
1711 ssize_t remaining = sizeof(id3->id3v2buf);
1713 /* this macro sets id3 strings by copying to the id3v2buf */
1714 #define SET(x, y) do \
1716 if (remaining > 0) \
1718 x = NULL; /* initialize with null if tag doesn't exist */ \
1719 char* src = get_tag_string(entry, y); \
1720 if (src) \
1722 x = buf; \
1723 size_t len = strlcpy(buf, src, remaining) +1; \
1724 buf += len; remaining -= len; \
1727 } while(0)
1730 SET(id3->title, tag_title);
1731 SET(id3->artist, tag_artist);
1732 SET(id3->album, tag_album);
1733 SET(id3->genre_string, tag_genre);
1734 SET(id3->composer, tag_composer);
1735 SET(id3->comment, tag_comment);
1736 SET(id3->albumartist, tag_albumartist);
1737 SET(id3->grouping, tag_grouping);
1739 id3->length = get_tag_numeric(entry, tag_length, idx_id);
1740 id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id);
1741 id3->rating = get_tag_numeric(entry, tag_rating, idx_id);
1742 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1743 id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1744 id3->year = get_tag_numeric(entry, tag_year, idx_id);
1746 id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1747 id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1748 id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1749 if (id3->bitrate == 0)
1750 id3->bitrate = 1;
1752 #if CONFIG_CODEC == SWCODEC
1753 if (global_settings.autoresume_enable)
1755 id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1756 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1757 id3->title, id3->offset);
1759 #endif
1761 return true;
1763 #endif
1765 static inline void write_item(const char *item)
1767 int len = strlen(item) + 1;
1769 data_size += len;
1770 write(cachefd, item, len);
1773 static int check_if_empty(char **tag)
1775 int length;
1777 if (*tag == NULL || **tag == '\0')
1779 *tag = UNTAGGED;
1780 return sizeof(UNTAGGED); /* Tag length */
1783 length = strlen(*tag);
1784 if (length > TAG_MAXLEN)
1786 logf("over length tag: %s", *tag);
1787 length = TAG_MAXLEN;
1788 (*tag)[length] = '\0';
1791 return length + 1;
1794 #define ADD_TAG(entry,tag,data) \
1795 /* Adding tag */ \
1796 entry.tag_offset[tag] = offset; \
1797 entry.tag_length[tag] = check_if_empty(data); \
1798 offset += entry.tag_length[tag]
1799 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1800 * idea, as it uses lots of stack and is called from a recursive function
1801 * (check_dir).
1803 static void __attribute__ ((noinline)) add_tagcache(char *path,
1804 unsigned long mtime
1805 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1806 ,int dc
1807 #endif
1810 struct mp3entry id3;
1811 struct temp_file_entry entry;
1812 bool ret;
1813 int fd;
1814 int idx_id = -1;
1815 char tracknumfix[3];
1816 int offset = 0;
1817 int path_length = strlen(path);
1818 bool has_albumartist;
1819 bool has_grouping;
1821 #ifdef SIMULATOR
1822 /* Crude logging for the sim - to aid in debugging */
1823 int logfd = open(ROCKBOX_DIR "/database.log",
1824 O_WRONLY | O_APPEND | O_CREAT, 0666);
1825 if (logfd >= 0) {
1826 write(logfd, path, strlen(path));
1827 write(logfd, "\n", 1);
1828 close(logfd);
1830 #endif
1832 if (cachefd < 0)
1833 return ;
1835 /* Check for overlength file path. */
1836 if (path_length > TAG_MAXLEN)
1838 /* Path can't be shortened. */
1839 logf("Too long path: %s", path);
1840 return ;
1843 /* Check if the file is supported. */
1844 if (probe_file_format(path) == AFMT_UNKNOWN)
1845 return ;
1847 /* Check if the file is already cached. */
1848 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1849 idx_id = find_entry_ram(path, dc);
1850 #endif
1852 /* Be sure the entry doesn't exist. */
1853 if (filenametag_fd >= 0 && idx_id < 0)
1854 idx_id = find_entry_disk(path, false);
1856 /* Check if file has been modified. */
1857 if (idx_id >= 0)
1859 struct index_entry idx;
1861 /* TODO: Mark that the index exists (for fast reverse scan) */
1862 //found_idx[idx_id/8] |= idx_id%8;
1864 if (!get_index(-1, idx_id, &idx, true))
1866 logf("failed to retrieve index entry");
1867 return ;
1870 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1872 /* No changes to file. */
1873 return ;
1876 /* Metadata might have been changed. Delete the entry. */
1877 logf("Re-adding: %s", path);
1878 if (!delete_entry(idx_id))
1880 logf("delete_entry failed: %d", idx_id);
1881 return ;
1885 fd = open(path, O_RDONLY);
1886 if (fd < 0)
1888 logf("open fail: %s", path);
1889 return ;
1892 memset(&id3, 0, sizeof(struct mp3entry));
1893 memset(&entry, 0, sizeof(struct temp_file_entry));
1894 memset(&tracknumfix, 0, sizeof(tracknumfix));
1895 ret = get_metadata(&id3, fd, path);
1896 close(fd);
1898 if (!ret)
1899 return ;
1901 logf("-> %s", path);
1903 if (id3.tracknum <= 0) /* Track number missing? */
1905 id3.tracknum = -1;
1908 /* Numeric tags */
1909 entry.tag_offset[tag_year] = id3.year;
1910 entry.tag_offset[tag_discnumber] = id3.discnum;
1911 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1912 entry.tag_offset[tag_length] = id3.length;
1913 entry.tag_offset[tag_bitrate] = id3.bitrate;
1914 entry.tag_offset[tag_mtime] = mtime;
1916 /* String tags. */
1917 has_albumartist = id3.albumartist != NULL
1918 && strlen(id3.albumartist) > 0;
1919 has_grouping = id3.grouping != NULL
1920 && strlen(id3.grouping) > 0;
1922 ADD_TAG(entry, tag_filename, &path);
1923 ADD_TAG(entry, tag_title, &id3.title);
1924 ADD_TAG(entry, tag_artist, &id3.artist);
1925 ADD_TAG(entry, tag_album, &id3.album);
1926 ADD_TAG(entry, tag_genre, &id3.genre_string);
1927 ADD_TAG(entry, tag_composer, &id3.composer);
1928 ADD_TAG(entry, tag_comment, &id3.comment);
1929 if (has_albumartist)
1931 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1933 else
1935 ADD_TAG(entry, tag_albumartist, &id3.artist);
1937 if (has_grouping)
1939 ADD_TAG(entry, tag_grouping, &id3.grouping);
1941 else
1943 ADD_TAG(entry, tag_grouping, &id3.title);
1945 entry.data_length = offset;
1947 /* Write the header */
1948 write(cachefd, &entry, sizeof(struct temp_file_entry));
1950 /* And tags also... Correct order is critical */
1951 write_item(path);
1952 write_item(id3.title);
1953 write_item(id3.artist);
1954 write_item(id3.album);
1955 write_item(id3.genre_string);
1956 write_item(id3.composer);
1957 write_item(id3.comment);
1958 if (has_albumartist)
1960 write_item(id3.albumartist);
1962 else
1964 write_item(id3.artist);
1966 if (has_grouping)
1968 write_item(id3.grouping);
1970 else
1972 write_item(id3.title);
1974 total_entry_count++;
1977 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1979 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1980 int len = strlen(str)+1;
1981 int i;
1982 unsigned crc32;
1983 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1984 char buf[TAG_MAXLEN+32];
1986 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1987 buf[i] = tolower(str[i]);
1988 buf[i] = '\0';
1990 crc32 = crc_32(buf, i, 0xffffffff);
1992 if (unique)
1994 /* Check if the crc does not exist -> entry does not exist for sure. */
1995 for (i = 0; i < tempbufidx; i++)
1997 if (crcbuf[-i] != crc32)
1998 continue;
2000 if (!strcasecmp(str, index[i].str))
2002 if (id < 0 || id >= lookup_buffer_depth)
2004 logf("lookup buf overf.: %d", id);
2005 return false;
2008 lookup[id] = &index[i];
2009 return true;
2014 /* Insert to CRC buffer. */
2015 crcbuf[-tempbufidx] = crc32;
2016 tempbuf_left -= 4;
2018 /* Insert it to the buffer. */
2019 tempbuf_left -= len;
2020 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
2021 return false;
2023 if (id >= lookup_buffer_depth)
2025 logf("lookup buf overf. #2: %d", id);
2026 return false;
2029 if (id >= 0)
2031 lookup[id] = &index[tempbufidx];
2032 index[tempbufidx].idlist.id = id;
2034 else
2035 index[tempbufidx].idlist.id = -1;
2037 index[tempbufidx].idlist.next = NULL;
2038 index[tempbufidx].idx_id = idx_id;
2039 index[tempbufidx].seek = -1;
2040 index[tempbufidx].str = &tempbuf[tempbuf_pos];
2041 memcpy(index[tempbufidx].str, str, len);
2042 tempbuf_pos += len;
2043 tempbufidx++;
2045 return true;
2048 static int compare(const void *p1, const void *p2)
2050 do_timed_yield();
2052 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2053 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2055 if (strcmp(e1->str, UNTAGGED) == 0)
2057 if (strcmp(e2->str, UNTAGGED) == 0)
2058 return 0;
2059 return -1;
2061 else if (strcmp(e2->str, UNTAGGED) == 0)
2062 return 1;
2064 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2067 static int tempbuf_sort(int fd)
2069 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2070 struct tagfile_entry fe;
2071 int i;
2072 int length;
2074 /* Generate reverse lookup entries. */
2075 for (i = 0; i < lookup_buffer_depth; i++)
2077 struct tempbuf_id_list *idlist;
2079 if (!lookup[i])
2080 continue;
2082 if (lookup[i]->idlist.id == i)
2083 continue;
2085 idlist = &lookup[i]->idlist;
2086 while (idlist->next != NULL)
2087 idlist = idlist->next;
2089 tempbuf_left -= sizeof(struct tempbuf_id_list);
2090 if (tempbuf_left - 4 < 0)
2091 return -1;
2093 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2094 if (tempbuf_pos & 0x03)
2096 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2097 tempbuf_left -= 3;
2098 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2100 tempbuf_pos += sizeof(struct tempbuf_id_list);
2102 idlist = idlist->next;
2103 idlist->id = i;
2104 idlist->next = NULL;
2106 do_timed_yield();
2109 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2110 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2112 for (i = 0; i < tempbufidx; i++)
2114 struct tempbuf_id_list *idlist = &index[i].idlist;
2116 /* Fix the lookup list. */
2117 while (idlist != NULL)
2119 if (idlist->id >= 0)
2120 lookup[idlist->id] = &index[i];
2121 idlist = idlist->next;
2124 index[i].seek = lseek(fd, 0, SEEK_CUR);
2125 length = strlen(index[i].str) + 1;
2126 fe.tag_length = length;
2127 fe.idx_id = index[i].idx_id;
2129 /* Check the chunk alignment. */
2130 if ((fe.tag_length + sizeof(struct tagfile_entry))
2131 % TAGFILE_ENTRY_CHUNK_LENGTH)
2133 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2134 ((fe.tag_length + sizeof(struct tagfile_entry))
2135 % TAGFILE_ENTRY_CHUNK_LENGTH);
2138 #ifdef TAGCACHE_STRICT_ALIGN
2139 /* Make sure the entry is long aligned. */
2140 if (index[i].seek & 0x03)
2142 logf("tempbuf_sort: alignment error!");
2143 return -3;
2145 #endif
2147 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2148 sizeof(struct tagfile_entry))
2150 logf("tempbuf_sort: write error #1");
2151 return -1;
2154 if (write(fd, index[i].str, length) != length)
2156 logf("tempbuf_sort: write error #2");
2157 return -2;
2160 /* Write some padding. */
2161 if (fe.tag_length - length > 0)
2162 write(fd, "XXXXXXXX", fe.tag_length - length);
2165 return i;
2168 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2170 if (id < 0 || id >= lookup_buffer_depth)
2171 return NULL;
2173 return lookup[id];
2177 inline static int tempbuf_find_location(int id)
2179 struct tempbuf_searchidx *entry;
2181 entry = tempbuf_locate(id);
2182 if (entry == NULL)
2183 return -1;
2185 return entry->seek;
2188 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2190 struct master_header tcmh;
2191 struct index_entry idx;
2192 int masterfd;
2193 int masterfd_pos;
2194 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2195 int max_entries;
2196 int entries_processed = 0;
2197 int i, j;
2198 char buf[TAG_MAXLEN];
2200 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2202 logf("Building numeric indices...");
2203 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2205 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2206 return false;
2208 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2209 SEEK_CUR);
2210 if (masterfd_pos == filesize(masterfd))
2212 logf("we can't append!");
2213 close(masterfd);
2214 return false;
2217 while (entries_processed < h->entry_count)
2219 int count = MIN(h->entry_count - entries_processed, max_entries);
2221 /* Read in as many entries as possible. */
2222 for (i = 0; i < count; i++)
2224 struct temp_file_entry *tfe = &entrybuf[i];
2225 int datastart;
2227 /* Read in numeric data. */
2228 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2229 sizeof(struct temp_file_entry))
2231 logf("read fail #1");
2232 close(masterfd);
2233 return false;
2236 datastart = lseek(tmpfd, 0, SEEK_CUR);
2239 * Read string data from the following tags:
2240 * - tag_filename
2241 * - tag_artist
2242 * - tag_album
2243 * - tag_title
2245 * A crc32 hash is calculated from the read data
2246 * and stored back to the data offset field kept in memory.
2248 #define tmpdb_read_string_tag(tag) \
2249 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2250 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2252 logf("read fail: buffer overflow"); \
2253 close(masterfd); \
2254 return false; \
2257 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2258 tfe->tag_length[tag]) \
2260 logf("read fail #2"); \
2261 close(masterfd); \
2262 return false; \
2265 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2266 lseek(tmpfd, datastart, SEEK_SET)
2268 tmpdb_read_string_tag(tag_filename);
2269 tmpdb_read_string_tag(tag_artist);
2270 tmpdb_read_string_tag(tag_album);
2271 tmpdb_read_string_tag(tag_title);
2273 /* Seek to the end of the string data. */
2274 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2277 /* Backup the master index position. */
2278 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2279 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2281 /* Check if we can resurrect some deleted runtime statistics data. */
2282 for (i = 0; i < tcmh.tch.entry_count; i++)
2284 /* Read the index entry. */
2285 if (ecread_index_entry(masterfd, &idx)
2286 != sizeof(struct index_entry))
2288 logf("read fail #3");
2289 close(masterfd);
2290 return false;
2294 * Skip unless the entry is marked as being deleted
2295 * or the data has already been resurrected.
2297 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2298 continue;
2300 /* Now try to match the entry. */
2302 * To succesfully match a song, the following conditions
2303 * must apply:
2305 * For numeric fields: tag_length
2306 * - Full identical match is required
2308 * If tag_filename matches, no further checking necessary.
2310 * For string hashes: tag_artist, tag_album, tag_title
2311 * - All three of these must match
2313 for (j = 0; j < count; j++)
2315 struct temp_file_entry *tfe = &entrybuf[j];
2317 /* Try to match numeric fields first. */
2318 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2319 continue;
2321 /* Now it's time to do the hash matching. */
2322 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2324 int match_count = 0;
2326 /* No filename match, check if we can match two other tags. */
2327 #define tmpdb_match(tag) \
2328 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2329 match_count++
2331 tmpdb_match(tag_artist);
2332 tmpdb_match(tag_album);
2333 tmpdb_match(tag_title);
2335 if (match_count < 3)
2337 /* Still no match found, give up. */
2338 continue;
2342 /* A match found, now copy & resurrect the statistical data. */
2343 #define tmpdb_copy_tag(tag) \
2344 tfe->tag_offset[tag] = idx.tag_seek[tag]
2346 tmpdb_copy_tag(tag_playcount);
2347 tmpdb_copy_tag(tag_rating);
2348 tmpdb_copy_tag(tag_playtime);
2349 tmpdb_copy_tag(tag_lastplayed);
2350 tmpdb_copy_tag(tag_commitid);
2351 tmpdb_copy_tag(tag_lastoffset);
2353 /* Avoid processing this entry again. */
2354 idx.flag |= FLAG_RESURRECTED;
2356 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2357 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2359 logf("masterfd writeback fail #1");
2360 close(masterfd);
2361 return false;
2364 logf("Entry resurrected");
2369 /* Restore the master index position. */
2370 lseek(masterfd, masterfd_pos, SEEK_SET);
2372 /* Commit the data to the index. */
2373 for (i = 0; i < count; i++)
2375 int loc = lseek(masterfd, 0, SEEK_CUR);
2377 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2379 logf("read fail #3");
2380 close(masterfd);
2381 return false;
2384 for (j = 0; j < TAG_COUNT; j++)
2386 if (!TAGCACHE_IS_NUMERIC(j))
2387 continue;
2389 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2391 idx.flag = entrybuf[i].flag;
2393 if (idx.tag_seek[tag_commitid])
2395 /* Data has been resurrected. */
2396 idx.flag |= FLAG_DIRTYNUM;
2398 else if (tc_stat.ready && current_tcmh.commitid > 0)
2400 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2401 idx.flag |= FLAG_DIRTYNUM;
2404 /* Write back the updated index. */
2405 lseek(masterfd, loc, SEEK_SET);
2406 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2408 logf("write fail");
2409 close(masterfd);
2410 return false;
2414 entries_processed += count;
2415 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2418 close(masterfd);
2420 return true;
2424 * Return values:
2425 * > 0 success
2426 * == 0 temporary failure
2427 * < 0 fatal error
2429 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2431 int i;
2432 struct tagcache_header tch;
2433 struct master_header tcmh;
2434 struct index_entry idxbuf[IDX_BUF_DEPTH];
2435 int idxbuf_pos;
2436 char buf[TAG_MAXLEN+32];
2437 int fd = -1, masterfd;
2438 bool error = false;
2439 int init;
2440 int masterfd_pos;
2442 logf("Building index: %d", index_type);
2444 /* Check the number of entries we need to allocate ram for. */
2445 commit_entry_count = h->entry_count + 1;
2447 masterfd = open_master_fd(&tcmh, false);
2448 if (masterfd >= 0)
2450 commit_entry_count += tcmh.tch.entry_count;
2451 close(masterfd);
2453 else
2454 remove_files(); /* Just to be sure we are clean. */
2456 /* Open the index file, which contains the tag names. */
2457 fd = open_tag_fd(&tch, index_type, true);
2458 if (fd >= 0)
2460 logf("tch.datasize=%ld", tch.datasize);
2461 lookup_buffer_depth = 1 +
2462 /* First part */ commit_entry_count +
2463 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2465 else
2467 lookup_buffer_depth = 1 +
2468 /* First part */ commit_entry_count +
2469 /* Second part */ 0;
2472 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2473 logf("commit_entry_count=%ld", commit_entry_count);
2475 /* Allocate buffer for all index entries from both old and new
2476 * tag files. */
2477 tempbufidx = 0;
2478 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2480 /* Allocate lookup buffer. The first portion of commit_entry_count
2481 * contains the new tags in the temporary file and the second
2482 * part for locating entries already in the db.
2484 * New tags Old tags
2485 * +---------+---------------------------+
2486 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2487 * +---------+---------------------------+
2489 * Old tags are inserted to a temporary buffer with position:
2490 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2491 * And new tags with index:
2492 * tempbuf_insert(idx, ...);
2494 * The buffer is sorted and written into tag file:
2495 * tempbuf_sort(...);
2496 * leaving master index locations messed up.
2498 * That is fixed using the lookup buffer for old tags:
2499 * new_seek = tempbuf_find_location(old_seek, ...);
2500 * and for new tags:
2501 * new_seek = tempbuf_find_location(idx);
2503 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2504 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2505 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2507 /* And calculate the remaining data space used mainly for storing
2508 * tag data (strings). */
2509 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2510 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2512 logf("Buffer way too small!");
2513 return 0;
2516 if (fd >= 0)
2519 * If tag file contains unique tags (sorted index), we will load
2520 * it entirely into memory so we can resort it later for use with
2521 * chunked browsing.
2523 if (TAGCACHE_IS_SORTED(index_type))
2525 logf("loading tags...");
2526 for (i = 0; i < tch.entry_count; i++)
2528 struct tagfile_entry entry;
2529 int loc = lseek(fd, 0, SEEK_CUR);
2530 bool ret;
2532 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2534 logf("read error #7");
2535 close(fd);
2536 return -2;
2539 if (entry.tag_length >= (int)sizeof(buf))
2541 logf("too long tag #3");
2542 close(fd);
2543 return -2;
2546 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2548 logf("read error #8");
2549 close(fd);
2550 return -2;
2553 /* Skip deleted entries. */
2554 if (buf[0] == '\0')
2555 continue;
2558 * Save the tag and tag id in the memory buffer. Tag id
2559 * is saved so we can later reindex the master lookup
2560 * table when the index gets resorted.
2562 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2563 + commit_entry_count, entry.idx_id,
2564 TAGCACHE_IS_UNIQUE(index_type));
2565 if (!ret)
2567 close(fd);
2568 return -3;
2570 do_timed_yield();
2572 logf("done");
2574 else
2575 tempbufidx = tch.entry_count;
2577 else
2580 * Creating new index file to store the tags. No need to preload
2581 * anything whether the index type is sorted or not.
2583 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2584 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2585 if (fd < 0)
2587 logf("%s open fail", buf);
2588 return -2;
2591 tch.magic = TAGCACHE_MAGIC;
2592 tch.entry_count = 0;
2593 tch.datasize = 0;
2595 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2596 != sizeof(struct tagcache_header))
2598 logf("header write failed");
2599 close(fd);
2600 return -2;
2604 /* Loading the tag lookup file as "master file". */
2605 logf("Loading index file");
2606 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2608 if (masterfd < 0)
2610 logf("Creating new DB");
2611 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2613 if (masterfd < 0)
2615 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2616 close(fd);
2617 return -2;
2620 /* Write the header (write real values later). */
2621 memset(&tcmh, 0, sizeof(struct master_header));
2622 tcmh.tch = *h;
2623 tcmh.tch.entry_count = 0;
2624 tcmh.tch.datasize = 0;
2625 tcmh.dirty = true;
2626 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2627 init = true;
2628 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2630 else
2633 * Master file already exists so we need to process the current
2634 * file first.
2636 init = false;
2638 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2639 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2641 logf("header error");
2642 close(fd);
2643 close(masterfd);
2644 return -2;
2648 * If we reach end of the master file, we need to expand it to
2649 * hold new tags. If the current index is not sorted, we can
2650 * simply append new data to end of the file.
2651 * However, if the index is sorted, we need to update all tag
2652 * pointers in the master file for the current index.
2654 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2655 SEEK_CUR);
2656 if (masterfd_pos == filesize(masterfd))
2658 logf("appending...");
2659 init = true;
2664 * Load new unique tags in memory to be sorted later and added
2665 * to the master lookup file.
2667 if (TAGCACHE_IS_SORTED(index_type))
2669 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2670 /* h is the header of the temporary file containing new tags. */
2671 logf("inserting new tags...");
2672 for (i = 0; i < h->entry_count; i++)
2674 struct temp_file_entry entry;
2676 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2677 sizeof(struct temp_file_entry))
2679 logf("read fail #3");
2680 error = true;
2681 goto error_exit;
2684 /* Read data. */
2685 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2687 logf("too long entry!");
2688 error = true;
2689 goto error_exit;
2692 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2693 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2694 entry.tag_length[index_type])
2696 logf("read fail #4");
2697 error = true;
2698 goto error_exit;
2701 if (TAGCACHE_IS_UNIQUE(index_type))
2702 error = !tempbuf_insert(buf, i, -1, true);
2703 else
2704 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2706 if (error)
2708 logf("insert error");
2709 goto error_exit;
2712 /* Skip to next. */
2713 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2714 entry.tag_length[index_type], SEEK_CUR);
2715 do_timed_yield();
2717 logf("done");
2719 /* Sort the buffer data and write it to the index file. */
2720 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2722 * We need to truncate the index file now. There can be junk left
2723 * at the end of file (however, we _should_ always follow the
2724 * entry_count and don't crash with that).
2726 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2728 i = tempbuf_sort(fd);
2729 if (i < 0)
2730 goto error_exit;
2731 logf("sorted %d tags", i);
2734 * Now update all indexes in the master lookup file.
2736 logf("updating indices...");
2737 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2738 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2740 int j;
2741 int loc = lseek(masterfd, 0, SEEK_CUR);
2743 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2745 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2746 != (int)sizeof(struct index_entry)*idxbuf_pos)
2748 logf("read fail #5");
2749 error = true;
2750 goto error_exit ;
2752 lseek(masterfd, loc, SEEK_SET);
2754 for (j = 0; j < idxbuf_pos; j++)
2756 if (idxbuf[j].flag & FLAG_DELETED)
2758 /* We can just ignore deleted entries. */
2759 // idxbuf[j].tag_seek[index_type] = 0;
2760 continue;
2763 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2764 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2765 + commit_entry_count);
2767 if (idxbuf[j].tag_seek[index_type] < 0)
2769 logf("update error: %ld/%d/%ld",
2770 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2771 error = true;
2772 goto error_exit;
2775 do_timed_yield();
2778 /* Write back the updated index. */
2779 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2780 index_entry_ec, tc_stat.econ) !=
2781 (int)sizeof(struct index_entry)*idxbuf_pos)
2783 logf("write fail");
2784 error = true;
2785 goto error_exit;
2788 logf("done");
2792 * Walk through the temporary file containing the new tags.
2794 // build_normal_index(h, tmpfd, masterfd, idx);
2795 logf("updating new indices...");
2796 lseek(masterfd, masterfd_pos, SEEK_SET);
2797 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2798 lseek(fd, 0, SEEK_END);
2799 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2801 int j;
2803 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2804 if (init)
2806 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2808 else
2810 int loc = lseek(masterfd, 0, SEEK_CUR);
2812 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2813 != (int)sizeof(struct index_entry)*idxbuf_pos)
2815 logf("read fail #6");
2816 error = true;
2817 break ;
2819 lseek(masterfd, loc, SEEK_SET);
2822 /* Read entry headers. */
2823 for (j = 0; j < idxbuf_pos; j++)
2825 if (!TAGCACHE_IS_SORTED(index_type))
2827 struct temp_file_entry entry;
2828 struct tagfile_entry fe;
2830 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2831 sizeof(struct temp_file_entry))
2833 logf("read fail #7");
2834 error = true;
2835 break ;
2838 /* Read data. */
2839 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2841 logf("too long entry!");
2842 logf("length=%d", entry.tag_length[index_type]);
2843 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2844 error = true;
2845 break ;
2848 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2849 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2850 entry.tag_length[index_type])
2852 logf("read fail #8");
2853 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2854 logf("length=0x%02x", entry.tag_length[index_type]);
2855 error = true;
2856 break ;
2859 /* Write to index file. */
2860 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2861 fe.tag_length = entry.tag_length[index_type];
2862 fe.idx_id = tcmh.tch.entry_count + i + j;
2863 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2864 write(fd, buf, fe.tag_length);
2865 tempbufidx++;
2867 /* Skip to next. */
2868 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2869 entry.tag_length[index_type], SEEK_CUR);
2871 else
2873 /* Locate the correct entry from the sorted array. */
2874 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2875 if (idxbuf[j].tag_seek[index_type] < 0)
2877 logf("entry not found (%d)", j);
2878 error = true;
2879 break ;
2884 /* Write index. */
2885 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2886 index_entry_ec, tc_stat.econ) !=
2887 (int)sizeof(struct index_entry)*idxbuf_pos)
2889 logf("tagcache: write fail #4");
2890 error = true;
2891 break ;
2894 do_timed_yield();
2896 logf("done");
2898 /* Finally write the header. */
2899 tch.magic = TAGCACHE_MAGIC;
2900 tch.entry_count = tempbufidx;
2901 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2902 lseek(fd, 0, SEEK_SET);
2903 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2905 if (index_type != tag_filename)
2906 h->datasize += tch.datasize;
2907 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2908 error_exit:
2910 close(fd);
2911 close(masterfd);
2913 if (error)
2914 return -2;
2916 return 1;
2919 static bool commit(void)
2921 struct tagcache_header tch;
2922 struct master_header tcmh;
2923 int i, len, rc;
2924 int tmpfd;
2925 int masterfd;
2926 #ifdef HAVE_DIRCACHE
2927 bool dircache_buffer_stolen = false;
2928 #endif
2929 #ifdef HAVE_TC_RAMCACHE
2930 bool ramcache_buffer_stolen = false;
2931 #endif
2932 bool local_allocation = false;
2934 logf("committing tagcache");
2936 while (write_lock)
2937 sleep(1);
2939 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2940 if (tmpfd < 0)
2942 logf("nothing to commit");
2943 return true;
2947 /* Load the header. */
2948 len = sizeof(struct tagcache_header);
2949 rc = read(tmpfd, &tch, len);
2951 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2953 logf("incorrect tmpheader");
2954 close(tmpfd);
2955 remove(TAGCACHE_FILE_TEMP);
2956 return false;
2959 if (tch.entry_count == 0)
2961 logf("nothing to commit");
2962 close(tmpfd);
2963 remove(TAGCACHE_FILE_TEMP);
2964 return true;
2967 /* Fully initialize existing headers (if any) before going further. */
2968 tc_stat.ready = check_all_headers();
2970 #ifdef HAVE_EEPROM_SETTINGS
2971 remove(TAGCACHE_STATEFILE);
2972 #endif
2974 /* At first be sure to unload the ramcache! */
2975 #ifdef HAVE_TC_RAMCACHE
2976 tc_stat.ramcache = false;
2977 #endif
2979 read_lock++;
2981 /* Try to steal every buffer we can :) */
2982 if (tempbuf_size == 0)
2983 local_allocation = true;
2985 #ifdef HAVE_DIRCACHE
2986 if (tempbuf_size == 0)
2988 /* Try to steal the dircache buffer. */
2989 tempbuf = dircache_steal_buffer(&tempbuf_size);
2990 tempbuf_size &= ~0x03;
2992 if (tempbuf_size > 0)
2994 dircache_buffer_stolen = true;
2997 #endif
2999 #ifdef HAVE_TC_RAMCACHE
3000 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
3002 tempbuf = (char *)(ramcache_hdr + 1);
3003 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
3004 tempbuf_size &= ~0x03;
3005 move_lock++;
3006 ramcache_buffer_stolen = true;
3008 #endif
3010 /* And finally fail if there are no buffers available. */
3011 if (tempbuf_size == 0)
3013 logf("delaying commit until next boot");
3014 tc_stat.commit_delayed = true;
3015 close(tmpfd);
3016 read_lock--;
3017 return false;
3020 logf("commit %ld entries...", tch.entry_count);
3022 /* Mark DB dirty so it will stay disabled if commit fails. */
3023 current_tcmh.dirty = true;
3024 update_master_header();
3026 /* Now create the index files. */
3027 tc_stat.commit_step = 0;
3028 tch.datasize = 0;
3029 tc_stat.commit_delayed = false;
3031 for (i = 0; i < TAG_COUNT; i++)
3033 int ret;
3035 if (TAGCACHE_IS_NUMERIC(i))
3036 continue;
3038 tc_stat.commit_step++;
3039 ret = build_index(i, &tch, tmpfd);
3040 if (ret <= 0)
3042 close(tmpfd);
3043 logf("tagcache failed init");
3044 if (ret == 0)
3045 tc_stat.commit_delayed = true;
3047 tc_stat.commit_step = 0;
3048 read_lock--;
3049 return false;
3053 if (!build_numeric_indices(&tch, tmpfd))
3055 logf("Failure to commit numeric indices");
3056 close(tmpfd);
3057 tc_stat.commit_step = 0;
3058 read_lock--;
3059 return false;
3062 close(tmpfd);
3064 tc_stat.commit_step = 0;
3066 /* Update the master index headers. */
3067 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3069 read_lock--;
3070 return false;
3073 remove(TAGCACHE_FILE_TEMP);
3075 tcmh.tch.entry_count += tch.entry_count;
3076 tcmh.tch.datasize = sizeof(struct master_header)
3077 + sizeof(struct index_entry) * tcmh.tch.entry_count
3078 + tch.datasize;
3079 tcmh.dirty = false;
3080 tcmh.commitid++;
3082 lseek(masterfd, 0, SEEK_SET);
3083 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3084 close(masterfd);
3086 logf("tagcache committed");
3087 tc_stat.ready = check_all_headers();
3088 tc_stat.readyvalid = true;
3090 if (local_allocation)
3092 tempbuf = NULL;
3093 tempbuf_size = 0;
3096 #ifdef HAVE_DIRCACHE
3097 /* Rebuild the dircache, if we stole the buffer. */
3098 if (dircache_buffer_stolen)
3099 dircache_resume();
3100 #endif
3102 #ifdef HAVE_TC_RAMCACHE
3103 if (ramcache_buffer_stolen)
3104 move_lock--;
3105 /* Reload tagcache. */
3106 if (tc_stat.ramcache_allocated > 0)
3107 tagcache_start_scan();
3108 #endif
3110 read_lock--;
3112 return true;
3115 #ifndef __PCTOOL__
3116 static int tempbuf_handle;
3117 #endif
3119 static void allocate_tempbuf(void)
3121 /* Yeah, malloc would be really nice now :) */
3122 #ifdef __PCTOOL__
3123 tempbuf_size = 32*1024*1024;
3124 tempbuf = malloc(tempbuf_size);
3125 #else
3126 tempbuf_handle = core_alloc_maximum("tc tempbuf", &tempbuf_size, NULL);
3127 tempbuf = core_get_data(tempbuf_handle);
3128 #endif
3131 static void free_tempbuf(void)
3133 if (tempbuf_size == 0)
3134 return ;
3136 #ifdef __PCTOOL__
3137 free(tempbuf);
3138 #else
3139 tempbuf_handle = core_free(tempbuf_handle);
3140 #endif
3141 tempbuf = NULL;
3142 tempbuf_size = 0;
3145 #ifndef __PCTOOL__
3147 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3149 struct index_entry idx;
3151 if (!tc_stat.ready)
3152 return false;
3154 if (!TAGCACHE_IS_NUMERIC(tag))
3155 return false;
3157 if (!get_index(masterfd, idx_id, &idx, false))
3158 return false;
3160 idx.tag_seek[tag] = data;
3161 idx.flag |= FLAG_DIRTYNUM;
3163 return write_index(masterfd, idx_id, &idx);
3166 #if 0
3167 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3168 int tag, long data)
3170 struct master_header myhdr;
3172 if (tcs->masterfd < 0)
3174 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3175 return false;
3178 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3180 #endif
3182 static bool command_queue_is_full(void)
3184 int next;
3186 next = command_queue_widx + 1;
3187 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3188 next = 0;
3190 return (next == command_queue_ridx);
3193 static void command_queue_sync_callback(void *data)
3195 (void)data;
3196 struct master_header myhdr;
3197 int masterfd;
3199 mutex_lock(&command_queue_mutex);
3201 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3202 return;
3204 while (command_queue_ridx != command_queue_widx)
3206 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3208 switch (ce->command)
3210 case CMD_UPDATE_MASTER_HEADER:
3212 close(masterfd);
3213 update_master_header();
3215 /* Re-open the masterfd. */
3216 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3217 return;
3219 break;
3221 case CMD_UPDATE_NUMERIC:
3223 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3224 break;
3228 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3229 command_queue_ridx = 0;
3232 close(masterfd);
3234 tc_stat.queue_length = 0;
3235 mutex_unlock(&command_queue_mutex);
3238 static void run_command_queue(bool force)
3240 if (COMMAND_QUEUE_IS_EMPTY)
3241 return;
3243 if (force || command_queue_is_full())
3244 command_queue_sync_callback(NULL);
3245 else
3246 register_storage_idle_func(command_queue_sync_callback);
3249 static void queue_command(int cmd, long idx_id, int tag, long data)
3251 while (1)
3253 int next;
3255 mutex_lock(&command_queue_mutex);
3256 next = command_queue_widx + 1;
3257 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3258 next = 0;
3260 /* Make sure queue is not full. */
3261 if (next != command_queue_ridx)
3263 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3265 ce->command = cmd;
3266 ce->idx_id = idx_id;
3267 ce->tag = tag;
3268 ce->data = data;
3270 command_queue_widx = next;
3272 tc_stat.queue_length++;
3274 mutex_unlock(&command_queue_mutex);
3275 break;
3278 /* Queue is full, try again later... */
3279 mutex_unlock(&command_queue_mutex);
3280 sleep(1);
3284 long tagcache_increase_serial(void)
3286 long old;
3288 if (!tc_stat.ready)
3289 return -2;
3291 while (read_lock)
3292 sleep(1);
3294 old = current_tcmh.serial++;
3295 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3297 return old;
3300 void tagcache_update_numeric(int idx_id, int tag, long data)
3302 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3304 #endif /* !__PCTOOL__ */
3306 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3308 char buf[512];
3309 int i;
3311 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3312 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3314 if (*datastr == '\0')
3315 break;
3317 if (*datastr == '"' || *datastr == '\\')
3318 buf[i++] = '\\';
3320 else if (*datastr == '\n')
3322 buf[i++] = '\\';
3323 buf[i] = 'n';
3324 continue;
3327 buf[i] = *(datastr++);
3330 strcpy(&buf[i], "\" ");
3332 write(fd, buf, i + 2);
3334 return true;
3337 #ifndef __PCTOOL__
3339 static bool read_tag(char *dest, long size,
3340 const char *src, const char *tagstr)
3342 int pos;
3343 char current_tag[32];
3345 while (*src != '\0')
3347 /* Skip all whitespace */
3348 while (*src == ' ')
3349 src++;
3351 if (*src == '\0')
3352 break;
3354 pos = 0;
3355 /* Read in tag name */
3356 while (*src != '=' && *src != ' ')
3358 current_tag[pos] = *src;
3359 src++;
3360 pos++;
3362 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3363 return false;
3365 current_tag[pos] = '\0';
3367 /* Read in tag data */
3369 /* Find the start. */
3370 while (*src != '"' && *src != '\0')
3371 src++;
3373 if (*src == '\0' || *(++src) == '\0')
3374 return false;
3376 /* Read the data. */
3377 for (pos = 0; pos < size; pos++)
3379 if (*src == '\0')
3380 break;
3382 if (*src == '\\')
3384 src++;
3385 if (*src == 'n')
3386 dest[pos] = '\n';
3387 else
3388 dest[pos] = *src;
3390 src++;
3391 continue;
3394 if (*src == '\0')
3395 break;
3397 if (*src == '"')
3399 src++;
3400 break;
3403 dest[pos] = *(src++);
3406 dest[pos] = '\0';
3408 if (!strcasecmp(tagstr, current_tag))
3409 return true;
3412 return false;
3415 static int parse_changelog_line(int line_n, char *buf, void *parameters)
3417 struct index_entry idx;
3418 char tag_data[TAG_MAXLEN+32];
3419 int idx_id;
3420 long masterfd = (long)parameters;
3421 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3422 tag_lastplayed, tag_commitid, tag_lastoffset };
3423 int i;
3424 (void)line_n;
3426 if (*buf == '#')
3427 return 0;
3429 /* logf("%d/%s", line_n, buf); */
3430 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3432 logf("%d/filename missing", line_n);
3433 logf("-> %s", buf);
3434 return 0;
3437 idx_id = find_index(tag_data);
3438 if (idx_id < 0)
3440 logf("%d/entry not found", line_n);
3441 return 0;
3444 if (!get_index(masterfd, idx_id, &idx, false))
3446 logf("%d/failed to retrieve index entry", line_n);
3447 return 0;
3450 /* Stop if tag has already been modified. */
3451 if (idx.flag & FLAG_DIRTYNUM)
3452 return 0;
3454 logf("%d/import: %s", line_n, tag_data);
3456 idx.flag |= FLAG_DIRTYNUM;
3457 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3459 int data;
3461 if (!read_tag(tag_data, sizeof tag_data, buf,
3462 tagcache_tag_to_str(import_tags[i])))
3464 continue;
3467 data = atoi(tag_data);
3468 if (data < 0)
3469 continue;
3471 idx.tag_seek[import_tags[i]] = data;
3473 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3474 current_tcmh.serial = data + 1;
3475 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3476 current_tcmh.commitid = data + 1;
3479 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3482 bool tagcache_import_changelog(void)
3484 struct master_header myhdr;
3485 struct tagcache_header tch;
3486 int clfd;
3487 long masterfd;
3488 char buf[2048];
3490 if (!tc_stat.ready)
3491 return false;
3493 while (read_lock)
3494 sleep(1);
3496 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3497 if (clfd < 0)
3499 logf("failure to open changelog");
3500 return false;
3503 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3505 close(clfd);
3506 return false;
3509 write_lock++;
3511 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3513 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3514 parse_changelog_line);
3516 close(clfd);
3517 close(masterfd);
3519 if (filenametag_fd >= 0)
3521 close(filenametag_fd);
3522 filenametag_fd = -1;
3525 write_lock--;
3527 update_master_header();
3529 return true;
3532 #endif /* !__PCTOOL__ */
3534 bool tagcache_create_changelog(struct tagcache_search *tcs)
3536 struct master_header myhdr;
3537 struct index_entry idx;
3538 char buf[TAG_MAXLEN+32];
3539 char temp[32];
3540 int clfd;
3541 int i, j;
3543 if (!tc_stat.ready)
3544 return false;
3546 if (!tagcache_search(tcs, tag_filename))
3547 return false;
3549 /* Initialize the changelog */
3550 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3551 if (clfd < 0)
3553 logf("failure to open changelog");
3554 return false;
3557 if (tcs->masterfd < 0)
3559 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3561 close(clfd);
3562 return false;
3565 else
3567 lseek(tcs->masterfd, 0, SEEK_SET);
3568 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3571 write(clfd, "## Changelog version 1\n", 23);
3573 for (i = 0; i < myhdr.tch.entry_count; i++)
3575 if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3577 logf("read error #9");
3578 tagcache_search_finish(tcs);
3579 close(clfd);
3580 return false;
3583 /* Skip until the entry found has been modified. */
3584 if (! (idx.flag & FLAG_DIRTYNUM) )
3585 continue;
3587 /* Skip deleted entries too. */
3588 if (idx.flag & FLAG_DELETED)
3589 continue;
3591 /* Now retrieve all tags. */
3592 for (j = 0; j < TAG_COUNT; j++)
3594 if (TAGCACHE_IS_NUMERIC(j))
3596 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3597 write_tag(clfd, tagcache_tag_to_str(j), temp);
3598 continue;
3601 tcs->type = j;
3602 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3603 write_tag(clfd, tagcache_tag_to_str(j), buf);
3606 write(clfd, "\n", 1);
3607 do_timed_yield();
3610 close(clfd);
3612 tagcache_search_finish(tcs);
3614 return true;
3617 static bool delete_entry(long idx_id)
3619 int fd = -1;
3620 int masterfd = -1;
3621 int tag, i;
3622 struct index_entry idx, myidx;
3623 struct master_header myhdr;
3624 char buf[TAG_MAXLEN+32];
3625 int in_use[TAG_COUNT];
3627 logf("delete_entry(): %ld", idx_id);
3629 #ifdef HAVE_TC_RAMCACHE
3630 /* At first mark the entry removed from ram cache. */
3631 if (tc_stat.ramcache)
3632 ramcache_hdr->indices[idx_id].flag |= FLAG_DELETED;
3633 #endif
3635 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3636 return false;
3638 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3639 if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3641 logf("delete_entry(): read error");
3642 goto cleanup;
3645 if (myidx.flag & FLAG_DELETED)
3647 logf("delete_entry(): already deleted!");
3648 goto cleanup;
3651 myidx.flag |= FLAG_DELETED;
3652 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3653 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3655 logf("delete_entry(): write_error #1");
3656 goto cleanup;
3659 /* Now check which tags are no longer in use (if any) */
3660 for (tag = 0; tag < TAG_COUNT; tag++)
3661 in_use[tag] = 0;
3663 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3664 for (i = 0; i < myhdr.tch.entry_count; i++)
3666 struct index_entry *idxp;
3668 #ifdef HAVE_TC_RAMCACHE
3669 /* Use RAM DB if available for greater speed */
3670 if (tc_stat.ramcache)
3671 idxp = &ramcache_hdr->indices[i];
3672 else
3673 #endif
3675 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3677 logf("delete_entry(): read error #2");
3678 goto cleanup;
3680 idxp = &idx;
3683 if (idxp->flag & FLAG_DELETED)
3684 continue;
3686 for (tag = 0; tag < TAG_COUNT; tag++)
3688 if (TAGCACHE_IS_NUMERIC(tag))
3689 continue;
3691 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3692 in_use[tag]++;
3696 /* Now delete all tags no longer in use. */
3697 for (tag = 0; tag < TAG_COUNT; tag++)
3699 struct tagcache_header tch;
3700 int oldseek = myidx.tag_seek[tag];
3702 if (TAGCACHE_IS_NUMERIC(tag))
3703 continue;
3705 /**
3706 * Replace tag seek with a hash value of the field string data.
3707 * That way runtime statistics of moved or altered files can be
3708 * resurrected.
3710 #ifdef HAVE_TC_RAMCACHE
3711 if (tc_stat.ramcache && tag != tag_filename)
3713 struct tagfile_entry *tfe;
3714 int32_t *seek = &ramcache_hdr->indices[idx_id].tag_seek[tag];
3716 tfe = (struct tagfile_entry *)&ramcache_hdr->tags[tag][*seek];
3717 move_lock++; /* protect tfe and seek if crc_32() yield()s */
3718 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3719 move_lock--;
3720 myidx.tag_seek[tag] = *seek;
3722 else
3723 #endif
3725 struct tagfile_entry tfe;
3727 /* Open the index file, which contains the tag names. */
3728 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3729 goto cleanup;
3731 /* Skip the header block */
3732 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3733 if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3735 logf("delete_entry(): read error #3");
3736 goto cleanup;
3739 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3741 logf("delete_entry(): read error #4");
3742 goto cleanup;
3745 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3748 if (in_use[tag])
3750 logf("in use: %d/%d", tag, in_use[tag]);
3751 if (fd >= 0)
3753 close(fd);
3754 fd = -1;
3756 continue;
3759 #ifdef HAVE_TC_RAMCACHE
3760 /* Delete from ram. */
3761 if (tc_stat.ramcache && tag != tag_filename)
3763 struct tagfile_entry *tagentry =
3764 (struct tagfile_entry *)&ramcache_hdr->tags[tag][oldseek];
3765 tagentry->tag_data[0] = '\0';
3767 #endif
3769 /* Open the index file, which contains the tag names. */
3770 if (fd < 0)
3772 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3773 goto cleanup;
3776 /* Skip the header block */
3777 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3779 /* Debug, print 10 first characters of the tag
3780 read(fd, buf, 10);
3781 buf[10]='\0';
3782 logf("TAG:%s", buf);
3783 lseek(fd, -10, SEEK_CUR);
3786 /* Write first data byte in tag as \0 */
3787 write(fd, "", 1);
3789 /* Now tag data has been removed */
3790 close(fd);
3791 fd = -1;
3794 /* Write index entry back into master index. */
3795 lseek(masterfd, sizeof(struct master_header) +
3796 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3797 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3799 logf("delete_entry(): write_error #2");
3800 goto cleanup;
3803 close(masterfd);
3805 return true;
3807 cleanup:
3808 if (fd >= 0)
3809 close(fd);
3810 if (masterfd >= 0)
3811 close(masterfd);
3813 return false;
3816 #ifndef __PCTOOL__
3818 * Returns true if there is an event waiting in the queue
3819 * that requires the current operation to be aborted.
3821 static bool check_event_queue(void)
3823 struct queue_event ev;
3825 if(!queue_peek(&tagcache_queue, &ev))
3826 return false;
3828 switch (ev.id)
3830 case Q_STOP_SCAN:
3831 case SYS_POWEROFF:
3832 case SYS_USB_CONNECTED:
3833 return true;
3836 return false;
3838 #endif
3840 #ifdef HAVE_TC_RAMCACHE
3842 static void fix_ramcache(void* old_addr, void* new_addr)
3844 ptrdiff_t offpos = new_addr - old_addr;
3845 for (int i = 0; i < TAG_COUNT; i++)
3846 ramcache_hdr->tags[i] += offpos;
3849 static int move_cb(int handle, void* current, void* new)
3851 (void)handle;
3852 if (move_lock > 0)
3853 return BUFLIB_CB_CANNOT_MOVE;
3855 fix_ramcache(current, new);
3856 ramcache_hdr = new;
3857 return BUFLIB_CB_OK;
3860 static struct buflib_callbacks ops = {
3861 .move_callback = move_cb,
3862 .shrink_callback = NULL,
3865 static bool allocate_tagcache(void)
3867 struct master_header tcmh;
3868 int fd;
3870 /* Load the header. */
3871 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3873 ramcache_hdr = NULL;
3874 return false;
3877 close(fd);
3879 /**
3880 * Now calculate the required cache size plus
3881 * some extra space for alignment fixes.
3883 tc_stat.ramcache_allocated = tcmh.tch.datasize + 256 + TAGCACHE_RESERVE +
3884 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3885 int handle = core_alloc_ex("tc ramcache", tc_stat.ramcache_allocated, &ops);
3886 ramcache_hdr = core_get_data(handle);
3887 memset(ramcache_hdr, 0, sizeof(struct ramcache_header));
3888 memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3889 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3891 return true;
3894 # ifdef HAVE_EEPROM_SETTINGS
3895 static bool tagcache_dumpload(void)
3897 struct statefile_header shdr;
3898 int fd, rc, handle;
3900 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3901 if (fd < 0)
3903 logf("no tagcache statedump");
3904 return false;
3907 /* Check the statefile memory placement */
3908 rc = read(fd, &shdr, sizeof(struct statefile_header));
3909 if (rc != sizeof(struct statefile_header)
3910 || shdr.magic != TAGCACHE_STATEFILE_MAGIC
3911 || shdr.mh.tch.magic != TAGCACHE_MAGIC)
3913 logf("incorrect statefile");
3914 ramcache_hdr = NULL;
3915 close(fd);
3916 return false;
3920 /* Lets allocate real memory and load it */
3921 handle = core_alloc_ex("tc ramcache", shdr.tc_stat.ramcache_allocated, &ops);
3922 ramcache_hdr = core_get_data(handle);
3923 move_lock++;
3924 rc = read(fd, ramcache_hdr, shdr.tc_stat.ramcache_allocated);
3925 move_lock--;
3926 close(fd);
3928 if (rc != shdr.tc_stat.ramcache_allocated)
3930 logf("read failure!");
3931 ramcache_hdr = NULL;
3932 return false;
3935 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3937 /* Now fix the pointers */
3938 fix_ramcache(shdr.hdr, ramcache_hdr);
3940 /* Load the tagcache master header (should match the actual DB file header). */
3941 memcpy(&current_tcmh, &shdr.mh, sizeof current_tcmh);
3943 return true;
3946 static bool tagcache_dumpsave(void)
3948 struct statefile_header shdr;
3949 int fd;
3951 if (!tc_stat.ramcache)
3952 return false;
3954 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3955 if (fd < 0)
3957 logf("failed to create a statedump");
3958 return false;
3961 /* Create the header */
3962 shdr.magic = TAGCACHE_STATEFILE_MAGIC;
3963 shdr.hdr = ramcache_hdr;
3964 memcpy(&shdr.mh, &current_tcmh, sizeof current_tcmh);
3965 memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat);
3966 write(fd, &shdr, sizeof shdr);
3968 /* And dump the data too */
3969 move_lock++;
3970 write(fd, ramcache_hdr, tc_stat.ramcache_allocated);
3971 move_lock--;
3972 close(fd);
3974 return true;
3976 # endif
3978 static bool load_tagcache(void)
3980 struct tagcache_header *tch;
3981 struct master_header tcmh;
3982 long bytesleft = tc_stat.ramcache_allocated - sizeof(struct ramcache_header);
3983 struct index_entry *idx;
3984 int rc, fd;
3985 char *p;
3986 int i, tag;
3988 # ifdef HAVE_DIRCACHE
3989 while (dircache_is_initializing())
3990 sleep(1);
3992 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3993 # endif
3995 logf("loading tagcache to ram...");
3997 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3998 if (fd < 0)
4000 logf("tagcache open failed");
4001 return false;
4004 if (ecread(fd, &tcmh, 1, master_header_ec, tc_stat.econ)
4005 != sizeof(struct master_header)
4006 || tcmh.tch.magic != TAGCACHE_MAGIC)
4008 logf("incorrect header");
4009 return false;
4012 /* Master header copy should already match, this can be redundant to do. */
4013 memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
4015 move_lock++; /* lock for the reset of the scan, simpler to handle */
4016 idx = ramcache_hdr->indices;
4018 /* Load the master index table. */
4019 for (i = 0; i < tcmh.tch.entry_count; i++)
4021 bytesleft -= sizeof(struct index_entry);
4022 if (bytesleft < 0)
4024 logf("too big tagcache.");
4025 goto failure;
4028 /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
4029 * may become corrupt. */
4030 rc = ecread_index_entry(fd, idx);
4031 if (rc != sizeof(struct index_entry))
4033 logf("read error #10");
4034 goto failure;
4037 idx++;
4040 close(fd);
4042 /* Load the tags. */
4043 p = (char *)idx;
4044 for (tag = 0; tag < TAG_COUNT; tag++)
4046 struct tagfile_entry *fe;
4047 char buf[TAG_MAXLEN+32];
4049 if (TAGCACHE_IS_NUMERIC(tag))
4050 continue ;
4052 //p = ((void *)p+1);
4053 p = (char *)((long)p & ~0x03) + 0x04;
4054 ramcache_hdr->tags[tag] = p;
4056 /* Check the header. */
4057 tch = (struct tagcache_header *)p;
4058 p += sizeof(struct tagcache_header);
4060 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
4061 goto failure_nofd;
4063 for (ramcache_hdr->entry_count[tag] = 0;
4064 ramcache_hdr->entry_count[tag] < tch->entry_count;
4065 ramcache_hdr->entry_count[tag]++)
4067 long pos;
4069 if (do_timed_yield())
4071 /* Abort if we got a critical event in queue */
4072 if (check_event_queue())
4073 goto failure;
4076 fe = (struct tagfile_entry *)p;
4077 pos = lseek(fd, 0, SEEK_CUR);
4078 rc = ecread_tagfile_entry(fd, fe);
4079 if (rc != sizeof(struct tagfile_entry))
4081 /* End of lookup table. */
4082 logf("read error #11");
4083 goto failure;
4086 /* We have a special handling for the filename tags. */
4087 if (tag == tag_filename)
4089 # ifdef HAVE_DIRCACHE
4090 int dc;
4091 # endif
4093 idx = &ramcache_hdr->indices[fe->idx_id];
4095 if (fe->tag_length >= (long)sizeof(buf)-1)
4097 read(fd, buf, 10);
4098 buf[10] = '\0';
4099 logf("TAG:%s", buf);
4100 logf("too long filename");
4101 goto failure;
4104 rc = read(fd, buf, fe->tag_length);
4105 if (rc != fe->tag_length)
4107 logf("read error #12");