1 /***************************************************************************
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
20 ****************************************************************************/
25 * ----------x---------x------------------x-----
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
58 /* #define LOGF_ENABLE */
64 #include "ata_idle_notify.h"
82 #include "eeprom_settings.h"
86 #define yield() do { } while(0)
87 #define sim_sleep(timeout) do { } while(0)
88 #define do_timed_yield() do { } while(0)
92 /* Tag Cache thread. */
93 static struct event_queue tagcache_queue
;
94 static long tagcache_stack
[(DEFAULT_STACK_SIZE
+ 0x4000)/sizeof(long)];
95 static const char tagcache_thread_name
[] = "tagcache";
98 #define UNTAGGED "<Untagged>"
100 /* Previous path when scanning directory tree recursively. */
101 static char curpath
[TAG_MAXLEN
+32];
103 /* Used when removing duplicates. */
104 static char *tempbuf
; /* Allocated when needed. */
105 static long tempbufidx
; /* Current location in buffer. */
106 static long tempbuf_size
; /* Buffer size (TEMPBUF_SIZE). */
107 static long tempbuf_left
; /* Buffer space left. */
108 static long tempbuf_pos
;
110 #define SORTED_TAGS_COUNT 8
111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
114 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
115 /* Tags we want to get sorted (loaded to the tempbuf). */
116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
117 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
118 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
122 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
123 (1LU << tag_albumartist) | (1LU << tag_grouping))
125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
126 static const char *tags_str
[] = { "artist", "album", "genre", "title",
127 "filename", "composer", "comment", "albumartist", "grouping", "year",
128 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
129 "playtime", "lastplayed", "commitid", "mtime" };
131 /* Status information of the tagcache. */
132 static struct tagcache_stat tc_stat
;
134 /* Queue commands. */
135 enum tagcache_queue
{
142 /* Internal tagcache command queue. */
143 CMD_UPDATE_MASTER_HEADER
,
147 struct tagcache_command_entry
{
155 static struct tagcache_command_entry command_queue
[TAGCACHE_COMMAND_QUEUE_LENGTH
];
156 static volatile int command_queue_widx
= 0;
157 static volatile int command_queue_ridx
= 0;
158 static struct mutex command_queue_mutex
;
161 /* Tag database structures. */
163 /* Variable-length tag entry in tag files. */
164 struct tagfile_entry
{
165 int32_t tag_length
; /* Length of the data in bytes including '\0' */
166 int32_t idx_id
; /* Corresponding entry location in index file of not unique tags */
167 char tag_data
[0]; /* Begin of the tag data */
170 /* Fixed-size tag entry in master db index. */
172 int32_t tag_seek
[TAG_COUNT
]; /* Location of tag data or numeric tag data */
173 int32_t flag
; /* Status flags */
176 /* Header is the same in every file. */
177 struct tagcache_header
{
178 int32_t magic
; /* Header version number */
179 int32_t datasize
; /* Data size in bytes */
180 int32_t entry_count
; /* Number of entries in this file */
183 struct master_header
{
184 struct tagcache_header tch
;
185 int32_t serial
; /* Increasing counting number */
186 int32_t commitid
; /* Number of commits so far */
190 /* For the endianess correction */
191 static const char *tagfile_entry_ec
= "ll";
193 Note: This should be (1 + TAG_COUNT) amount of l's.
195 static const char *index_entry_ec
= "lllllllllllllllllllll";
197 static const char *tagcache_header_ec
= "lll";
198 static const char *master_header_ec
= "llllll";
200 static struct master_header current_tcmh
;
202 #ifdef HAVE_TC_RAMCACHE
203 /* Header is created when loading database to ram. */
204 struct ramcache_header
{
205 struct master_header h
; /* Header from the master index */
206 struct index_entry
*indices
; /* Master index file content */
207 char *tags
[TAG_COUNT
]; /* Tag file content (not including filename tag) */
208 int entry_count
[TAG_COUNT
]; /* Number of entries in the indices. */
211 # ifdef HAVE_EEPROM_SETTINGS
212 struct statefile_header
{
213 struct ramcache_header
*hdr
;
214 struct tagcache_stat tc_stat
;
218 /* Pointer to allocated ramcache_header */
219 static struct ramcache_header
*hdr
;
223 * Full tag entries stored in a temporary file waiting
224 * for commit to the cache. */
225 struct temp_file_entry
{
226 long tag_offset
[TAG_COUNT
];
227 short tag_length
[TAG_COUNT
];
233 struct tempbuf_id_list
{
235 struct tempbuf_id_list
*next
;
238 struct tempbuf_searchidx
{
242 struct tempbuf_id_list idlist
;
245 /* Lookup buffer for fixing messed up index while after sorting. */
246 static long commit_entry_count
;
247 static long lookup_buffer_depth
;
248 static struct tempbuf_searchidx
**lookup
;
250 /* Used when building the temporary file. */
251 static int cachefd
= -1, filenametag_fd
;
252 static int total_entry_count
= 0;
253 static int data_size
= 0;
254 static int processed_dir_count
;
256 /* Thread safe locking */
257 static volatile int write_lock
;
258 static volatile int read_lock
;
260 static bool delete_entry(long idx_id
);
262 const char* tagcache_tag_to_str(int tag
)
264 return tags_str
[tag
];
269 * Returns true if specified flag is still present, i.e., dircache
270 * has not been reloaded.
272 static bool is_dircache_intact(void)
274 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
278 static int open_tag_fd(struct tagcache_header
*hdr
, int tag
, bool write
)
284 if (TAGCACHE_IS_NUMERIC(tag
) || tag
< 0 || tag
>= TAG_COUNT
)
287 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag
);
289 fd
= open(buf
, write
? O_RDWR
: O_RDONLY
);
292 logf("tag file open failed: tag=%d write=%d file=%s", tag
, write
, buf
);
293 tc_stat
.ready
= false;
297 /* Check the header. */
298 rc
= ecread(fd
, hdr
, 1, tagcache_header_ec
, tc_stat
.econ
);
299 if (hdr
->magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct tagcache_header
))
301 logf("header error");
302 tc_stat
.ready
= false;
310 static int open_master_fd(struct master_header
*hdr
, bool write
)
315 fd
= open(TAGCACHE_FILE_MASTER
, write
? O_RDWR
: O_RDONLY
);
318 logf("master file open failed for R/W");
319 tc_stat
.ready
= false;
323 tc_stat
.econ
= false;
325 /* Check the header. */
326 rc
= read(fd
, hdr
, sizeof(struct master_header
));
327 if (hdr
->tch
.magic
== TAGCACHE_MAGIC
&& rc
== sizeof(struct master_header
))
333 /* Trying to read again, this time with endianess correction enabled. */
334 lseek(fd
, 0, SEEK_SET
);
336 rc
= ecread(fd
, hdr
, 1, master_header_ec
, true);
337 if (hdr
->tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct master_header
))
339 logf("header error");
340 tc_stat
.ready
= false;
351 static bool do_timed_yield(void)
353 /* Sorting can lock up for quite a while, so yield occasionally */
354 static long wakeup_tick
= 0;
355 if (current_tick
>= wakeup_tick
)
357 wakeup_tick
= current_tick
+ (HZ
/4);
365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
366 static long find_entry_ram(const char *filename
,
367 const struct dirent
*dc
)
369 static long last_pos
= 0;
372 /* Check if we tagcache is loaded into ram. */
373 if (!tc_stat
.ramcache
)
377 dc
= dircache_get_entry_ptr(filename
);
381 logf("tagcache: file not found.");
392 for (; i
< hdr
->h
.tch
.entry_count
; i
++)
394 if (hdr
->indices
[i
].tag_seek
[tag_filename
] == (long)dc
)
396 last_pos
= MAX(0, i
- 3);
413 static long find_entry_disk(const char *filename
, bool localfd
)
415 struct tagcache_header tch
;
416 static long last_pos
= -1;
417 long pos_history
[POS_HISTORY_COUNT
];
418 long pos_history_idx
= 0;
420 struct tagfile_entry tfe
;
422 char buf
[TAG_MAXLEN
+32];
430 if (fd
< 0 || localfd
)
433 if ( (fd
= open_tag_fd(&tch
, tag_filename
, false)) < 0)
440 lseek(fd
, last_pos
, SEEK_SET
);
442 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
446 pos
= lseek(fd
, 0, SEEK_CUR
);
447 for (i
= pos_history_idx
-1; i
>= 0; i
--)
448 pos_history
[i
+1] = pos_history
[i
];
449 pos_history
[0] = pos
;
451 if (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
452 != sizeof(struct tagfile_entry
))
457 if (tfe
.tag_length
>= (long)sizeof(buf
))
459 logf("too long tag #1");
467 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
469 logf("read error #2");
477 if (!strcasecmp(filename
, buf
))
479 last_pos
= pos_history
[pos_history_idx
];
484 if (pos_history_idx
< POS_HISTORY_COUNT
- 1)
498 if (fd
!= filenametag_fd
|| localfd
)
503 if (fd
!= filenametag_fd
|| localfd
)
509 static int find_index(const char *filename
)
513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
514 if (tc_stat
.ramcache
&& is_dircache_intact())
515 idx_id
= find_entry_ram(filename
, NULL
);
519 idx_id
= find_entry_disk(filename
, true);
524 bool tagcache_find_index(struct tagcache_search
*tcs
, const char *filename
)
531 idx_id
= find_index(filename
);
535 if (!tagcache_search(tcs
, tag_filename
))
538 tcs
->entry_count
= 0;
539 tcs
->idx_id
= idx_id
;
544 static bool get_index(int masterfd
, int idxid
,
545 struct index_entry
*idx
, bool use_ram
)
547 bool localfd
= false;
551 logf("Incorrect idxid: %d", idxid
);
555 #ifdef HAVE_TC_RAMCACHE
556 if (tc_stat
.ramcache
&& use_ram
)
558 if (hdr
->indices
[idxid
].flag
& FLAG_DELETED
)
561 # ifdef HAVE_DIRCACHE
562 if (!(hdr
->indices
[idxid
].flag
& FLAG_DIRCACHE
)
563 || is_dircache_intact())
566 memcpy(idx
, &hdr
->indices
[idxid
], sizeof(struct index_entry
));
576 struct master_header tcmh
;
579 masterfd
= open_master_fd(&tcmh
, false);
584 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
585 + sizeof(struct master_header
), SEEK_SET
);
586 if (ecread(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
587 != sizeof(struct index_entry
))
589 logf("read error #3");
599 if (idx
->flag
& FLAG_DELETED
)
607 static bool write_index(int masterfd
, int idxid
, struct index_entry
*idx
)
609 /* We need to exclude all memory only flags & tags when writing to disk. */
610 if (idx
->flag
& FLAG_DIRCACHE
)
612 logf("memory only flags!");
616 #ifdef HAVE_TC_RAMCACHE
617 /* Only update numeric data. Writing the whole index to RAM by memcpy
618 * destroys dircache pointers!
620 if (tc_stat
.ramcache
)
623 struct index_entry
*idx_ram
= &hdr
->indices
[idxid
];
625 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
627 if (TAGCACHE_IS_NUMERIC(tag
))
629 idx_ram
->tag_seek
[tag
] = idx
->tag_seek
[tag
];
633 /* Don't touch the dircache flag or attributes. */
634 idx_ram
->flag
= (idx
->flag
& 0x0000ffff)
635 | (idx_ram
->flag
& (0xffff0000 | FLAG_DIRCACHE
));
639 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
640 + sizeof(struct master_header
), SEEK_SET
);
641 if (ecwrite(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
642 != sizeof(struct index_entry
))
644 logf("write error #3");
645 logf("idxid: %d", idxid
);
652 #endif /* !__PCTOOL__ */
654 static bool open_files(struct tagcache_search
*tcs
, int tag
)
656 if (tcs
->idxfd
[tag
] < 0)
660 snprintf(fn
, sizeof fn
, TAGCACHE_FILE_INDEX
, tag
);
661 tcs
->idxfd
[tag
] = open(fn
, O_RDONLY
);
664 if (tcs
->idxfd
[tag
] < 0)
666 logf("File not open!");
673 static bool retrieve(struct tagcache_search
*tcs
, struct index_entry
*idx
,
674 int tag
, char *buf
, long size
)
676 struct tagfile_entry tfe
;
681 if (TAGCACHE_IS_NUMERIC(tag
))
684 seek
= idx
->tag_seek
[tag
];
687 logf("Retrieve failed");
691 #ifdef HAVE_TC_RAMCACHE
694 struct tagfile_entry
*ep
;
696 # ifdef HAVE_DIRCACHE
697 if (tag
== tag_filename
&& (idx
->flag
& FLAG_DIRCACHE
)
698 && is_dircache_intact())
700 dircache_copy_path((struct dirent
*)seek
,
706 if (tag
!= tag_filename
)
708 ep
= (struct tagfile_entry
*)&hdr
->tags
[tag
][seek
];
709 strncpy(buf
, ep
->tag_data
, size
-1);
716 if (!open_files(tcs
, tag
))
719 lseek(tcs
->idxfd
[tag
], seek
, SEEK_SET
);
720 if (ecread(tcs
->idxfd
[tag
], &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
721 != sizeof(struct tagfile_entry
))
723 logf("read error #5");
727 if (tfe
.tag_length
>= size
)
729 logf("too small buffer");
733 if (read(tcs
->idxfd
[tag
], buf
, tfe
.tag_length
) !=
736 logf("read error #6");
740 buf
[tfe
.tag_length
] = '\0';
745 static long check_virtual_tags(int tag
, const struct index_entry
*idx
)
751 case tag_virt_length_sec
:
752 data
= (idx
->tag_seek
[tag_length
]/1000) % 60;
755 case tag_virt_length_min
:
756 data
= (idx
->tag_seek
[tag_length
]/1000) / 60;
759 case tag_virt_playtime_sec
:
760 data
= (idx
->tag_seek
[tag_playtime
]/1000) % 60;
763 case tag_virt_playtime_min
:
764 data
= (idx
->tag_seek
[tag_playtime
]/1000) / 60;
767 case tag_virt_autoscore
:
768 if (idx
->tag_seek
[tag_length
] == 0
769 || idx
->tag_seek
[tag_playcount
] == 0)
775 data
= 100 * idx
->tag_seek
[tag_playtime
]
776 / idx
->tag_seek
[tag_length
]
777 / idx
->tag_seek
[tag_playcount
];
781 /* How many commits before the file has been added to the DB. */
782 case tag_virt_entryage
:
783 data
= current_tcmh
.commitid
- idx
->tag_seek
[tag_commitid
] - 1;
787 data
= idx
->tag_seek
[tag
];
793 long tagcache_get_numeric(const struct tagcache_search
*tcs
, int tag
)
795 struct index_entry idx
;
800 if (!TAGCACHE_IS_NUMERIC(tag
))
803 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
806 return check_virtual_tags(tag
, &idx
);
809 inline static bool str_ends_with(const char *str1
, const char *str2
)
811 int str_len
= strlen(str1
);
812 int clause_len
= strlen(str2
);
814 if (clause_len
> str_len
)
817 return !strcasecmp(&str1
[str_len
- clause_len
], str2
);
820 inline static bool str_oneof(const char *str
, const char *list
)
823 int l
, len
= strlen(str
);
827 sep
= strchr(list
, '|');
828 l
= sep
? (long)sep
- (long)list
: (int)strlen(list
);
829 if ((l
==len
) && !strncasecmp(str
, list
, len
))
831 list
+= sep
? l
+ 1 : l
;
837 static bool check_against_clause(long numeric
, const char *str
,
838 const struct tagcache_search_clause
*clause
)
842 switch (clause
->type
)
845 return numeric
== clause
->numeric_data
;
847 return numeric
!= clause
->numeric_data
;
849 return numeric
> clause
->numeric_data
;
851 return numeric
>= clause
->numeric_data
;
853 return numeric
< clause
->numeric_data
;
855 return numeric
<= clause
->numeric_data
;
857 logf("Incorrect numeric tag: %d", clause
->type
);
862 switch (clause
->type
)
865 return !strcasecmp(clause
->str
, str
);
867 return strcasecmp(clause
->str
, str
);
869 return 0>strcasecmp(clause
->str
, str
);
871 return 0>=strcasecmp(clause
->str
, str
);
873 return 0<strcasecmp(clause
->str
, str
);
875 return 0<=strcasecmp(clause
->str
, str
);
876 case clause_contains
:
877 return (strcasestr(str
, clause
->str
) != NULL
);
878 case clause_not_contains
:
879 return (strcasestr(str
, clause
->str
) == NULL
);
880 case clause_begins_with
:
881 return (strcasestr(str
, clause
->str
) == str
);
882 case clause_not_begins_with
:
883 return (strcasestr(str
, clause
->str
) != str
);
884 case clause_ends_with
:
885 return str_ends_with(str
, clause
->str
);
886 case clause_not_ends_with
:
887 return !str_ends_with(str
, clause
->str
);
889 return str_oneof(str
, clause
->str
);
892 logf("Incorrect tag: %d", clause
->type
);
899 static bool check_clauses(struct tagcache_search
*tcs
,
900 struct index_entry
*idx
,
901 struct tagcache_search_clause
**clause
, int count
)
905 #ifdef HAVE_TC_RAMCACHE
908 /* Go through all conditional clauses. */
909 for (i
= 0; i
< count
; i
++)
911 struct tagfile_entry
*tfe
;
916 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
918 if (!TAGCACHE_IS_NUMERIC(clause
[i
]->tag
))
920 if (clause
[i
]->tag
== tag_filename
)
922 retrieve(tcs
, idx
, tag_filename
, buf
, sizeof buf
);
927 tfe
= (struct tagfile_entry
*)&hdr
->tags
[clause
[i
]->tag
][seek
];
932 if (!check_against_clause(seek
, str
, clause
[i
]))
939 /* Check for conditions. */
940 for (i
= 0; i
< count
; i
++)
942 struct tagfile_entry tfe
;
946 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
948 memset(str
, 0, sizeof str
);
949 if (!TAGCACHE_IS_NUMERIC(clause
[i
]->tag
))
951 int fd
= tcs
->idxfd
[clause
[i
]->tag
];
952 lseek(fd
, seek
, SEEK_SET
);
953 ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
954 if (tfe
.tag_length
>= (int)sizeof(str
))
956 logf("Too long tag read!");
960 read(fd
, str
, tfe
.tag_length
);
962 /* Check if entry has been deleted. */
967 if (!check_against_clause(seek
, str
, clause
[i
]))
975 bool tagcache_check_clauses(struct tagcache_search
*tcs
,
976 struct tagcache_search_clause
**clause
, int count
)
978 struct index_entry idx
;
983 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
986 return check_clauses(tcs
, &idx
, clause
, count
);
989 static bool add_uniqbuf(struct tagcache_search
*tcs
, unsigned long id
)
993 /* If uniq buffer is not defined we must return true for search to work. */
994 if (tcs
->unique_list
== NULL
|| (!TAGCACHE_IS_UNIQUE(tcs
->type
)
995 && !TAGCACHE_IS_NUMERIC(tcs
->type
)))
1000 for (i
= 0; i
< tcs
->unique_list_count
; i
++)
1002 /* Return false if entry is found. */
1003 if (tcs
->unique_list
[i
] == id
)
1007 if (tcs
->unique_list_count
< tcs
->unique_list_capacity
)
1009 tcs
->unique_list
[i
] = id
;
1010 tcs
->unique_list_count
++;
1016 static bool build_lookup_list(struct tagcache_search
*tcs
)
1018 struct index_entry entry
;
1021 tcs
->seek_list_count
= 0;
1023 #ifdef HAVE_TC_RAMCACHE
1025 # ifdef HAVE_DIRCACHE
1026 && (tcs
->type
!= tag_filename
|| is_dircache_intact())
1030 for (i
= tcs
->seek_pos
; i
< hdr
->h
.tch
.entry_count
; i
++)
1032 struct tagcache_seeklist_entry
*seeklist
;
1033 struct index_entry
*idx
= &hdr
->indices
[i
];
1034 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1037 /* Skip deleted files. */
1038 if (idx
->flag
& FLAG_DELETED
)
1041 /* Go through all filters.. */
1042 for (j
= 0; j
< tcs
->filter_count
; j
++)
1044 if (idx
->tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
1050 if (j
< tcs
->filter_count
)
1053 /* Check for conditions. */
1054 if (!check_clauses(tcs
, idx
, tcs
->clause
, tcs
->clause_count
))
1057 /* Add to the seek list if not already in uniq buffer. */
1058 if (!add_uniqbuf(tcs
, idx
->tag_seek
[tcs
->type
]))
1062 seeklist
= &tcs
->seeklist
[tcs
->seek_list_count
];
1063 seeklist
->seek
= idx
->tag_seek
[tcs
->type
];
1064 seeklist
->flag
= idx
->flag
;
1065 seeklist
->idx_id
= i
;
1066 tcs
->seek_list_count
++;
1071 return tcs
->seek_list_count
> 0;
1075 if (tcs
->masterfd
< 0)
1077 struct master_header tcmh
;
1078 tcs
->masterfd
= open_master_fd(&tcmh
, false);
1081 lseek(tcs
->masterfd
, tcs
->seek_pos
* sizeof(struct index_entry
) +
1082 sizeof(struct master_header
), SEEK_SET
);
1084 while (ecread(tcs
->masterfd
, &entry
, 1, index_entry_ec
, tc_stat
.econ
)
1085 == sizeof(struct index_entry
))
1087 struct tagcache_seeklist_entry
*seeklist
;
1089 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
1095 /* Check if entry has been deleted. */
1096 if (entry
.flag
& FLAG_DELETED
)
1099 /* Go through all filters.. */
1100 for (j
= 0; j
< tcs
->filter_count
; j
++)
1102 if (entry
.tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
1106 if (j
< tcs
->filter_count
)
1109 /* Check for conditions. */
1110 if (!check_clauses(tcs
, &entry
, tcs
->clause
, tcs
->clause_count
))
1113 /* Add to the seek list if not already in uniq buffer. */
1114 if (!add_uniqbuf(tcs
, entry
.tag_seek
[tcs
->type
]))
1118 seeklist
= &tcs
->seeklist
[tcs
->seek_list_count
];
1119 seeklist
->seek
= entry
.tag_seek
[tcs
->type
];
1120 seeklist
->flag
= entry
.flag
;
1121 seeklist
->idx_id
= i
;
1122 tcs
->seek_list_count
++;
1127 return tcs
->seek_list_count
> 0;
1131 static void remove_files(void)
1136 tc_stat
.ready
= false;
1137 tc_stat
.ramcache
= false;
1138 tc_stat
.econ
= false;
1139 remove(TAGCACHE_FILE_MASTER
);
1140 for (i
= 0; i
< TAG_COUNT
; i
++)
1142 if (TAGCACHE_IS_NUMERIC(i
))
1145 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, i
);
1151 static bool check_all_headers(void)
1153 struct master_header myhdr
;
1154 struct tagcache_header tch
;
1158 if ( (fd
= open_master_fd(&myhdr
, false)) < 0)
1164 logf("tagcache is dirty!");
1168 memcpy(¤t_tcmh
, &myhdr
, sizeof(struct master_header
));
1170 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
1172 if (TAGCACHE_IS_NUMERIC(tag
))
1175 if ( (fd
= open_tag_fd(&tch
, tag
, false)) < 0)
1184 bool tagcache_is_busy(void)
1186 return read_lock
|| write_lock
;
1189 bool tagcache_search(struct tagcache_search
*tcs
, int tag
)
1191 struct tagcache_header tag_hdr
;
1192 struct master_header master_hdr
;
1198 memset(tcs
, 0, sizeof(struct tagcache_search
));
1199 if (tc_stat
.commit_step
> 0 || !tc_stat
.ready
)
1202 tcs
->position
= sizeof(struct tagcache_header
);
1205 tcs
->list_position
= 0;
1206 tcs
->seek_list_count
= 0;
1207 tcs
->filter_count
= 0;
1210 for (i
= 0; i
< TAG_COUNT
; i
++)
1213 #ifndef HAVE_TC_RAMCACHE
1214 tcs
->ramsearch
= false;
1216 tcs
->ramsearch
= tc_stat
.ramcache
;
1219 tcs
->entry_count
= hdr
->entry_count
[tcs
->type
];
1224 /* Always open as R/W so we can pass tcs to functions that modify data also
1225 * without failing. */
1226 tcs
->masterfd
= open_master_fd(&master_hdr
, true);
1227 if (tcs
->masterfd
< 0)
1230 if (!TAGCACHE_IS_NUMERIC(tcs
->type
))
1232 tcs
->idxfd
[tcs
->type
] = open_tag_fd(&tag_hdr
, tcs
->type
, false);
1233 if (tcs
->idxfd
[tcs
->type
] < 0)
1236 tcs
->entry_count
= tag_hdr
.entry_count
;
1240 tcs
->entry_count
= master_hdr
.tch
.entry_count
;
1245 tcs
->initialized
= true;
1251 void tagcache_search_set_uniqbuf(struct tagcache_search
*tcs
,
1252 void *buffer
, long length
)
1254 tcs
->unique_list
= (unsigned long *)buffer
;
1255 tcs
->unique_list_capacity
= length
/ sizeof(*tcs
->unique_list
);
1256 tcs
->unique_list_count
= 0;
1259 bool tagcache_search_add_filter(struct tagcache_search
*tcs
,
1262 if (tcs
->filter_count
== TAGCACHE_MAX_FILTERS
)
1265 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag
))
1268 tcs
->filter_tag
[tcs
->filter_count
] = tag
;
1269 tcs
->filter_seek
[tcs
->filter_count
] = seek
;
1270 tcs
->filter_count
++;
1275 bool tagcache_search_add_clause(struct tagcache_search
*tcs
,
1276 struct tagcache_search_clause
*clause
)
1280 if (tcs
->clause_count
>= TAGCACHE_MAX_CLAUSES
)
1282 logf("Too many clauses");
1286 /* Check if there is already a similar filter in present (filters are
1287 * much faster than clauses).
1289 for (i
= 0; i
< tcs
->filter_count
; i
++)
1291 if (tcs
->filter_tag
[i
] == clause
->tag
)
1295 if (!TAGCACHE_IS_NUMERIC(clause
->tag
) && tcs
->idxfd
[clause
->tag
] < 0)
1299 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, clause
->tag
);
1300 tcs
->idxfd
[clause
->tag
] = open(buf
, O_RDONLY
);
1303 tcs
->clause
[tcs
->clause_count
] = clause
;
1304 tcs
->clause_count
++;
1309 static bool get_next(struct tagcache_search
*tcs
)
1311 static char buf
[TAG_MAXLEN
+32];
1312 struct tagfile_entry entry
;
1315 if (!tcs
->valid
|| !tc_stat
.ready
)
1318 if (tcs
->idxfd
[tcs
->type
] < 0 && !TAGCACHE_IS_NUMERIC(tcs
->type
)
1319 #ifdef HAVE_TC_RAMCACHE
1325 /* Relative fetch. */
1326 if (tcs
->filter_count
> 0 || tcs
->clause_count
> 0
1327 || TAGCACHE_IS_NUMERIC(tcs
->type
)
1328 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1329 /* We need to retrieve flag status for dircache. */
1330 || (tcs
->ramsearch
&& tcs
->type
== tag_filename
)
1334 struct tagcache_seeklist_entry
*seeklist
;
1336 /* Check for end of list. */
1337 if (tcs
->list_position
== tcs
->seek_list_count
)
1339 tcs
->list_position
= 0;
1341 /* Try to fetch more. */
1342 if (!build_lookup_list(tcs
))
1349 seeklist
= &tcs
->seeklist
[tcs
->list_position
];
1350 flag
= seeklist
->flag
;
1351 tcs
->position
= seeklist
->seek
;
1352 tcs
->idx_id
= seeklist
->idx_id
;
1353 tcs
->list_position
++;
1357 if (tcs
->entry_count
== 0)
1366 tcs
->result_seek
= tcs
->position
;
1368 if (TAGCACHE_IS_NUMERIC(tcs
->type
))
1370 snprintf(buf
, sizeof(buf
), "%d", tcs
->position
);
1372 tcs
->result_len
= strlen(buf
) + 1;
1377 #ifdef HAVE_TC_RAMCACHE
1381 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1382 if (tcs
->type
== tag_filename
&& (flag
& FLAG_DIRCACHE
)
1383 && is_dircache_intact())
1385 dircache_copy_path((struct dirent
*)tcs
->position
,
1388 tcs
->result_len
= strlen(buf
) + 1;
1389 tcs
->ramresult
= false;
1395 if (tcs
->type
!= tag_filename
)
1397 struct tagfile_entry
*ep
;
1399 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->position
];
1400 tcs
->result
= ep
->tag_data
;
1401 tcs
->result_len
= strlen(tcs
->result
) + 1;
1402 tcs
->idx_id
= ep
->idx_id
;
1403 tcs
->ramresult
= true;
1405 /* Increase position for the next run. This may get overwritten. */
1406 tcs
->position
+= sizeof(struct tagfile_entry
) + ep
->tag_length
;
1413 if (!open_files(tcs
, tcs
->type
))
1419 /* Seek stream to the correct position and continue to direct fetch. */
1420 lseek(tcs
->idxfd
[tcs
->type
], tcs
->position
, SEEK_SET
);
1422 if (ecread(tcs
->idxfd
[tcs
->type
], &entry
, 1,
1423 tagfile_entry_ec
, tc_stat
.econ
) != sizeof(struct tagfile_entry
))
1425 logf("read error #5");
1430 if (entry
.tag_length
> (long)sizeof(buf
))
1433 logf("too long tag #2");
1434 logf("P:%X/%X", tcs
->position
, entry
.tag_length
);
1438 if (read(tcs
->idxfd
[tcs
->type
], buf
, entry
.tag_length
) != entry
.tag_length
)
1441 logf("read error #4");
1446 Update the position for the next read (this may be overridden
1447 if filters or clauses are being used).
1449 tcs
->position
+= sizeof(struct tagfile_entry
) + entry
.tag_length
;
1451 tcs
->result_len
= strlen(tcs
->result
) + 1;
1452 tcs
->idx_id
= entry
.idx_id
;
1453 tcs
->ramresult
= false;
1458 bool tagcache_get_next(struct tagcache_search
*tcs
)
1460 while (get_next(tcs
))
1462 if (tcs
->result_len
> 1)
1469 bool tagcache_retrieve(struct tagcache_search
*tcs
, int idxid
,
1470 int tag
, char *buf
, long size
)
1472 struct index_entry idx
;
1475 if (!get_index(tcs
->masterfd
, idxid
, &idx
, true))
1478 return retrieve(tcs
, &idx
, tag
, buf
, size
);
1481 static bool update_master_header(void)
1483 struct master_header myhdr
;
1489 if ( (fd
= open_master_fd(&myhdr
, true)) < 0)
1492 myhdr
.serial
= current_tcmh
.serial
;
1493 myhdr
.commitid
= current_tcmh
.commitid
;
1494 myhdr
.dirty
= current_tcmh
.dirty
;
1497 lseek(fd
, 0, SEEK_SET
);
1498 ecwrite(fd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
1501 #ifdef HAVE_TC_RAMCACHE
1504 hdr
->h
.serial
= current_tcmh
.serial
;
1505 hdr
->h
.commitid
= current_tcmh
.commitid
;
1506 hdr
->h
.dirty
= current_tcmh
.dirty
;
1515 void tagcache_modify(struct tagcache_search
*tcs
, int type
, const char *text
)
1517 struct tagentry
*entry
;
1519 if (tcs
->type
!= tag_title
)
1522 /* We will need reserve buffer for this. */
1525 struct tagfile_entry
*ep
;
1527 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->result_seek
];
1528 tcs
->seek_list
[tcs
->seek_list_count
];
1531 entry
= find_entry_ram();
1536 void tagcache_search_finish(struct tagcache_search
*tcs
)
1540 if (!tcs
->initialized
)
1543 if (tcs
->masterfd
>= 0)
1545 close(tcs
->masterfd
);
1549 for (i
= 0; i
< TAG_COUNT
; i
++)
1551 if (tcs
->idxfd
[i
] >= 0)
1553 close(tcs
->idxfd
[i
]);
1558 tcs
->ramsearch
= false;
1560 tcs
->initialized
= 0;
1565 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1566 static struct tagfile_entry
*get_tag(const struct index_entry
*entry
, int tag
)
1568 return (struct tagfile_entry
*)&hdr
->tags
[tag
][entry
->tag_seek
[tag
]];
1571 static long get_tag_numeric(const struct index_entry
*entry
, int tag
)
1573 return check_virtual_tags(tag
, entry
);
1576 static char* get_tag_string(const struct index_entry
*entry
, int tag
)
1578 char* s
= get_tag(entry
, tag
)->tag_data
;
1579 return strcmp(s
, UNTAGGED
) ? s
: NULL
;
1582 bool tagcache_fill_tags(struct mp3entry
*id3
, const char *filename
)
1584 struct index_entry
*entry
;
1587 if (!tc_stat
.ready
|| !tc_stat
.ramcache
)
1590 /* Find the corresponding entry in tagcache. */
1591 idx_id
= find_entry_ram(filename
, NULL
);
1595 entry
= &hdr
->indices
[idx_id
];
1597 id3
->title
= get_tag_string(entry
, tag_title
);
1598 id3
->artist
= get_tag_string(entry
, tag_artist
);
1599 id3
->album
= get_tag_string(entry
, tag_album
);
1600 id3
->genre_string
= get_tag_string(entry
, tag_genre
);
1601 id3
->composer
= get_tag_string(entry
, tag_composer
);
1602 id3
->comment
= get_tag_string(entry
, tag_comment
);
1603 id3
->albumartist
= get_tag_string(entry
, tag_albumartist
);
1604 id3
->grouping
= get_tag_string(entry
, tag_grouping
);
1606 id3
->playcount
= get_tag_numeric(entry
, tag_playcount
);
1607 id3
->rating
= get_tag_numeric(entry
, tag_rating
);
1608 id3
->lastplayed
= get_tag_numeric(entry
, tag_lastplayed
);
1609 id3
->score
= get_tag_numeric(entry
, tag_virt_autoscore
) / 10;
1610 id3
->year
= get_tag_numeric(entry
, tag_year
);
1612 id3
->discnum
= get_tag_numeric(entry
, tag_discnumber
);
1613 id3
->tracknum
= get_tag_numeric(entry
, tag_tracknumber
);
1614 id3
->bitrate
= get_tag_numeric(entry
, tag_bitrate
);
1615 if (id3
->bitrate
== 0)
1622 static inline void write_item(const char *item
)
1624 int len
= strlen(item
) + 1;
1627 write(cachefd
, item
, len
);
1630 static int check_if_empty(char **tag
)
1634 if (*tag
== NULL
|| **tag
== '\0')
1637 return sizeof(UNTAGGED
); /* Tag length */
1640 length
= strlen(*tag
);
1641 if (length
> TAG_MAXLEN
)
1643 logf("over length tag: %s", *tag
);
1644 length
= TAG_MAXLEN
;
1645 (*tag
)[length
] = '\0';
1651 #define ADD_TAG(entry,tag,data) \
1653 entry.tag_offset[tag] = offset; \
1654 entry.tag_length[tag] = check_if_empty(data); \
1655 offset += entry.tag_length[tag]
1657 static void add_tagcache(char *path
, unsigned long mtime
1658 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1659 ,const struct dirent
*dc
1663 struct mp3entry id3
;
1664 struct temp_file_entry entry
;
1668 char tracknumfix
[3];
1670 int path_length
= strlen(path
);
1671 bool has_albumartist
;
1677 /* Check for overlength file path. */
1678 if (path_length
> TAG_MAXLEN
)
1680 /* Path can't be shortened. */
1681 logf("Too long path: %s", path
);
1685 /* Check if the file is supported. */
1686 if (probe_file_format(path
) == AFMT_UNKNOWN
)
1689 /* Check if the file is already cached. */
1690 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1691 if (tc_stat
.ramcache
&& is_dircache_intact())
1693 idx_id
= find_entry_ram(path
, dc
);
1697 /* Be sure the entry doesn't exist. */
1698 if (filenametag_fd
>= 0 && idx_id
< 0)
1699 idx_id
= find_entry_disk(path
, false);
1701 /* Check if file has been modified. */
1704 struct index_entry idx
;
1706 /* TODO: Mark that the index exists (for fast reverse scan) */
1707 //found_idx[idx_id/8] |= idx_id%8;
1709 if (!get_index(-1, idx_id
, &idx
, true))
1711 logf("failed to retrieve index entry");
1715 if ((unsigned long)idx
.tag_seek
[tag_mtime
] == mtime
)
1717 /* No changes to file. */
1721 /* Metadata might have been changed. Delete the entry. */
1722 logf("Re-adding: %s", path
);
1723 if (!delete_entry(idx_id
))
1725 logf("delete_entry failed: %d", idx_id
);
1730 fd
= open(path
, O_RDONLY
);
1733 logf("open fail: %s", path
);
1737 memset(&id3
, 0, sizeof(struct mp3entry
));
1738 memset(&entry
, 0, sizeof(struct temp_file_entry
));
1739 memset(&tracknumfix
, 0, sizeof(tracknumfix
));
1740 ret
= get_metadata(&id3
, fd
, path
);
1746 logf("-> %s", path
);
1748 /* Generate track number if missing. */
1749 if (id3
.tracknum
<= 0)
1751 const char *p
= strrchr(path
, '.');
1754 p
= &path
[strlen(path
)-1];
1758 if (isdigit(*p
) && isdigit(*(p
-1)))
1760 tracknumfix
[1] = *p
--;
1761 tracknumfix
[0] = *p
;
1767 if (tracknumfix
[0] != '\0')
1769 id3
.tracknum
= atoi(tracknumfix
);
1770 /* Set a flag to indicate track number has been generated. */
1771 entry
.flag
|= FLAG_TRKNUMGEN
;
1775 /* Unable to generate track number. */
1781 entry
.tag_offset
[tag_year
] = id3
.year
;
1782 entry
.tag_offset
[tag_discnumber
] = id3
.discnum
;
1783 entry
.tag_offset
[tag_tracknumber
] = id3
.tracknum
;
1784 entry
.tag_offset
[tag_length
] = id3
.length
;
1785 entry
.tag_offset
[tag_bitrate
] = id3
.bitrate
;
1786 entry
.tag_offset
[tag_mtime
] = mtime
;
1789 has_albumartist
= id3
.albumartist
!= NULL
1790 && strlen(id3
.albumartist
) > 0;
1791 has_grouping
= id3
.grouping
!= NULL
1792 && strlen(id3
.grouping
) > 0;
1794 ADD_TAG(entry
, tag_filename
, &path
);
1795 ADD_TAG(entry
, tag_title
, &id3
.title
);
1796 ADD_TAG(entry
, tag_artist
, &id3
.artist
);
1797 ADD_TAG(entry
, tag_album
, &id3
.album
);
1798 ADD_TAG(entry
, tag_genre
, &id3
.genre_string
);
1799 ADD_TAG(entry
, tag_composer
, &id3
.composer
);
1800 ADD_TAG(entry
, tag_comment
, &id3
.comment
);
1801 if (has_albumartist
)
1803 ADD_TAG(entry
, tag_albumartist
, &id3
.albumartist
);
1807 ADD_TAG(entry
, tag_albumartist
, &id3
.artist
);
1811 ADD_TAG(entry
, tag_grouping
, &id3
.grouping
);
1815 ADD_TAG(entry
, tag_grouping
, &id3
.title
);
1817 entry
.data_length
= offset
;
1819 /* Write the header */
1820 write(cachefd
, &entry
, sizeof(struct temp_file_entry
));
1822 /* And tags also... Correct order is critical */
1824 write_item(id3
.title
);
1825 write_item(id3
.artist
);
1826 write_item(id3
.album
);
1827 write_item(id3
.genre_string
);
1828 write_item(id3
.composer
);
1829 write_item(id3
.comment
);
1830 if (has_albumartist
)
1832 write_item(id3
.albumartist
);
1836 write_item(id3
.artist
);
1840 write_item(id3
.grouping
);
1844 write_item(id3
.title
);
1846 total_entry_count
++;
1849 static bool tempbuf_insert(char *str
, int id
, int idx_id
, bool unique
)
1851 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1852 int len
= strlen(str
)+1;
1855 unsigned *crcbuf
= (unsigned *)&tempbuf
[tempbuf_size
-4];
1856 char buf
[TAG_MAXLEN
+32];
1858 for (i
= 0; str
[i
] != '\0' && i
< (int)sizeof(buf
)-1; i
++)
1859 buf
[i
] = tolower(str
[i
]);
1862 crc32
= crc_32(buf
, i
, 0xffffffff);
1866 /* Check if the crc does not exist -> entry does not exist for sure. */
1867 for (i
= 0; i
< tempbufidx
; i
++)
1869 if (crcbuf
[-i
] != crc32
)
1872 if (!strcasecmp(str
, index
[i
].str
))
1874 if (id
< 0 || id
>= lookup_buffer_depth
)
1876 logf("lookup buf overf.: %d", id
);
1880 lookup
[id
] = &index
[i
];
1886 /* Insert to CRC buffer. */
1887 crcbuf
[-tempbufidx
] = crc32
;
1890 /* Insert it to the buffer. */
1891 tempbuf_left
-= len
;
1892 if (tempbuf_left
- 4 < 0 || tempbufidx
>= commit_entry_count
-1)
1895 if (id
>= lookup_buffer_depth
)
1897 logf("lookup buf overf. #2: %d", id
);
1903 lookup
[id
] = &index
[tempbufidx
];
1904 index
[tempbufidx
].idlist
.id
= id
;
1907 index
[tempbufidx
].idlist
.id
= -1;
1909 index
[tempbufidx
].idlist
.next
= NULL
;
1910 index
[tempbufidx
].idx_id
= idx_id
;
1911 index
[tempbufidx
].seek
= -1;
1912 index
[tempbufidx
].str
= &tempbuf
[tempbuf_pos
];
1913 memcpy(index
[tempbufidx
].str
, str
, len
);
1920 static int compare(const void *p1
, const void *p2
)
1924 struct tempbuf_searchidx
*e1
= (struct tempbuf_searchidx
*)p1
;
1925 struct tempbuf_searchidx
*e2
= (struct tempbuf_searchidx
*)p2
;
1927 if (strcmp(e1
->str
, UNTAGGED
) == 0)
1929 if (strcmp(e2
->str
, UNTAGGED
) == 0)
1933 else if (strcmp(e2
->str
, UNTAGGED
) == 0)
1936 return strncasecmp(e1
->str
, e2
->str
, TAG_MAXLEN
);
1939 static int tempbuf_sort(int fd
)
1941 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1942 struct tagfile_entry fe
;
1946 /* Generate reverse lookup entries. */
1947 for (i
= 0; i
< lookup_buffer_depth
; i
++)
1949 struct tempbuf_id_list
*idlist
;
1954 if (lookup
[i
]->idlist
.id
== i
)
1957 idlist
= &lookup
[i
]->idlist
;
1958 while (idlist
->next
!= NULL
)
1959 idlist
= idlist
->next
;
1961 tempbuf_left
-= sizeof(struct tempbuf_id_list
);
1962 if (tempbuf_left
- 4 < 0)
1965 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1966 if (tempbuf_pos
& 0x03)
1968 tempbuf_pos
= (tempbuf_pos
& ~0x03) + 0x04;
1970 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1972 tempbuf_pos
+= sizeof(struct tempbuf_id_list
);
1974 idlist
= idlist
->next
;
1976 idlist
->next
= NULL
;
1981 qsort(index
, tempbufidx
, sizeof(struct tempbuf_searchidx
), compare
);
1982 memset(lookup
, 0, lookup_buffer_depth
* sizeof(struct tempbuf_searchidx
**));
1984 for (i
= 0; i
< tempbufidx
; i
++)
1986 struct tempbuf_id_list
*idlist
= &index
[i
].idlist
;
1988 /* Fix the lookup list. */
1989 while (idlist
!= NULL
)
1991 if (idlist
->id
>= 0)
1992 lookup
[idlist
->id
] = &index
[i
];
1993 idlist
= idlist
->next
;
1996 index
[i
].seek
= lseek(fd
, 0, SEEK_CUR
);
1997 length
= strlen(index
[i
].str
) + 1;
1998 fe
.tag_length
= length
;
1999 fe
.idx_id
= index
[i
].idx_id
;
2001 /* Check the chunk alignment. */
2002 if ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
2003 % TAGFILE_ENTRY_CHUNK_LENGTH
)
2005 fe
.tag_length
+= TAGFILE_ENTRY_CHUNK_LENGTH
-
2006 ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
2007 % TAGFILE_ENTRY_CHUNK_LENGTH
);
2010 #ifdef TAGCACHE_STRICT_ALIGN
2011 /* Make sure the entry is long aligned. */
2012 if (index
[i
].seek
& 0x03)
2014 logf("tempbuf_sort: alignment error!");
2019 if (ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
) !=
2020 sizeof(struct tagfile_entry
))
2022 logf("tempbuf_sort: write error #1");
2026 if (write(fd
, index
[i
].str
, length
) != length
)
2028 logf("tempbuf_sort: write error #2");
2032 /* Write some padding. */
2033 if (fe
.tag_length
- length
> 0)
2034 write(fd
, "XXXXXXXX", fe
.tag_length
- length
);
2040 inline static struct tempbuf_searchidx
* tempbuf_locate(int id
)
2042 if (id
< 0 || id
>= lookup_buffer_depth
)
2049 inline static int tempbuf_find_location(int id
)
2051 struct tempbuf_searchidx
*entry
;
2053 entry
= tempbuf_locate(id
);
2060 static bool build_numeric_indices(struct tagcache_header
*h
, int tmpfd
)
2062 struct master_header tcmh
;
2063 struct index_entry idx
;
2066 struct temp_file_entry
*entrybuf
= (struct temp_file_entry
*)tempbuf
;
2068 int entries_processed
= 0;
2070 char buf
[TAG_MAXLEN
];
2072 max_entries
= tempbuf_size
/ sizeof(struct temp_file_entry
) - 1;
2074 logf("Building numeric indices...");
2075 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2077 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2080 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2082 if (masterfd_pos
== filesize(masterfd
))
2084 logf("we can't append!");
2089 while (entries_processed
< h
->entry_count
)
2091 int count
= MIN(h
->entry_count
- entries_processed
, max_entries
);
2093 /* Read in as many entries as possible. */
2094 for (i
= 0; i
< count
; i
++)
2096 struct temp_file_entry
*tfe
= &entrybuf
[i
];
2099 /* Read in numeric data. */
2100 if (read(tmpfd
, tfe
, sizeof(struct temp_file_entry
)) !=
2101 sizeof(struct temp_file_entry
))
2103 logf("read fail #1");
2108 datastart
= lseek(tmpfd
, 0, SEEK_CUR
);
2111 * Read string data from the following tags:
2117 * A crc32 hash is calculated from the read data
2118 * and stored back to the data offset field kept in memory.
2120 #define tmpdb_read_string_tag(tag) \
2121 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2122 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2124 logf("read fail: buffer overflow"); \
2129 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2130 tfe->tag_length[tag]) \
2132 logf("read fail #2"); \
2137 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2138 lseek(tmpfd, datastart, SEEK_SET)
2140 tmpdb_read_string_tag(tag_filename
);
2141 tmpdb_read_string_tag(tag_artist
);
2142 tmpdb_read_string_tag(tag_album
);
2143 tmpdb_read_string_tag(tag_title
);
2145 /* Seek to the end of the string data. */
2146 lseek(tmpfd
, tfe
->data_length
, SEEK_CUR
);
2149 /* Backup the master index position. */
2150 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2151 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2153 /* Check if we can resurrect some deleted runtime statistics data. */
2154 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
++)
2156 /* Read the index entry. */
2157 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2158 != sizeof(struct index_entry
))
2160 logf("read fail #3");
2166 * Skip unless the entry is marked as being deleted
2167 * or the data has already been resurrected.
2169 if (!(idx
.flag
& FLAG_DELETED
) || idx
.flag
& FLAG_RESURRECTED
)
2172 /* Now try to match the entry. */
2174 * To succesfully match a song, the following conditions
2177 * For numeric fields: tag_length
2178 * - Full identical match is required
2180 * If tag_filename matches, no further checking necessary.
2182 * For string hashes: tag_artist, tag_album, tag_title
2183 * - Two of these must match
2185 for (j
= 0; j
< count
; j
++)
2187 struct temp_file_entry
*tfe
= &entrybuf
[j
];
2189 /* Try to match numeric fields first. */
2190 if (tfe
->tag_offset
[tag_length
] != idx
.tag_seek
[tag_length
])
2193 /* Now it's time to do the hash matching. */
2194 if (tfe
->tag_offset
[tag_filename
] != idx
.tag_seek
[tag_filename
])
2196 int match_count
= 0;
2198 /* No filename match, check if we can match two other tags. */
2199 #define tmpdb_match(tag) \
2200 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2203 tmpdb_match(tag_artist
);
2204 tmpdb_match(tag_album
);
2205 tmpdb_match(tag_title
);
2207 if (match_count
< 2)
2209 /* Still no match found, give up. */
2214 /* A match found, now copy & resurrect the statistical data. */
2215 #define tmpdb_copy_tag(tag) \
2216 tfe->tag_offset[tag] = idx.tag_seek[tag]
2218 tmpdb_copy_tag(tag_playcount
);
2219 tmpdb_copy_tag(tag_rating
);
2220 tmpdb_copy_tag(tag_playtime
);
2221 tmpdb_copy_tag(tag_lastplayed
);
2222 tmpdb_copy_tag(tag_commitid
);
2224 /* Avoid processing this entry again. */
2225 idx
.flag
|= FLAG_RESURRECTED
;
2227 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
2228 if (ecwrite(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2229 != sizeof(struct index_entry
))
2231 logf("masterfd writeback fail #1");
2236 logf("Entry resurrected");
2241 /* Restore the master index position. */
2242 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2244 /* Commit the data to the index. */
2245 for (i
= 0; i
< count
; i
++)
2247 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2249 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2250 != sizeof(struct index_entry
))
2252 logf("read fail #3");
2257 for (j
= 0; j
< TAG_COUNT
; j
++)
2259 if (!TAGCACHE_IS_NUMERIC(j
))
2262 idx
.tag_seek
[j
] = entrybuf
[i
].tag_offset
[j
];
2264 idx
.flag
= entrybuf
[i
].flag
;
2266 if (idx
.tag_seek
[tag_commitid
])
2268 /* Data has been resurrected. */
2269 idx
.flag
|= FLAG_DIRTYNUM
;
2271 else if (tc_stat
.ready
&& current_tcmh
.commitid
> 0)
2273 idx
.tag_seek
[tag_commitid
] = current_tcmh
.commitid
;
2274 idx
.flag
|= FLAG_DIRTYNUM
;
2277 /* Write back the updated index. */
2278 lseek(masterfd
, loc
, SEEK_SET
);
2279 if (ecwrite(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2280 != sizeof(struct index_entry
))
2288 entries_processed
+= count
;
2289 logf("%d/%ld entries processed", entries_processed
, h
->entry_count
);
2300 * == 0 temporary failure
2303 static int build_index(int index_type
, struct tagcache_header
*h
, int tmpfd
)
2306 struct tagcache_header tch
;
2307 struct master_header tcmh
;
2308 struct index_entry idxbuf
[IDX_BUF_DEPTH
];
2310 char buf
[TAG_MAXLEN
+32];
2311 int fd
= -1, masterfd
;
2316 logf("Building index: %d", index_type
);
2318 /* Check the number of entries we need to allocate ram for. */
2319 commit_entry_count
= h
->entry_count
+ 1;
2321 masterfd
= open_master_fd(&tcmh
, false);
2324 commit_entry_count
+= tcmh
.tch
.entry_count
;
2328 remove_files(); /* Just to be sure we are clean. */
2330 /* Open the index file, which contains the tag names. */
2331 fd
= open_tag_fd(&tch
, index_type
, true);
2334 logf("tch.datasize=%ld", tch
.datasize
);
2335 lookup_buffer_depth
= 1 +
2336 /* First part */ commit_entry_count
+
2337 /* Second part */ (tch
.datasize
/ TAGFILE_ENTRY_CHUNK_LENGTH
);
2341 lookup_buffer_depth
= 1 +
2342 /* First part */ commit_entry_count
+
2343 /* Second part */ 0;
2346 logf("lookup_buffer_depth=%ld", lookup_buffer_depth
);
2347 logf("commit_entry_count=%ld", commit_entry_count
);
2349 /* Allocate buffer for all index entries from both old and new
2352 tempbuf_pos
= commit_entry_count
* sizeof(struct tempbuf_searchidx
);
2354 /* Allocate lookup buffer. The first portion of commit_entry_count
2355 * contains the new tags in the temporary file and the second
2356 * part for locating entries already in the db.
2359 * +---------+---------------------------+
2360 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2361 * +---------+---------------------------+
2363 * Old tags are inserted to a temporary buffer with position:
2364 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2365 * And new tags with index:
2366 * tempbuf_insert(idx, ...);
2368 * The buffer is sorted and written into tag file:
2369 * tempbuf_sort(...);
2370 * leaving master index locations messed up.
2372 * That is fixed using the lookup buffer for old tags:
2373 * new_seek = tempbuf_find_location(old_seek, ...);
2375 * new_seek = tempbuf_find_location(idx);
2377 lookup
= (struct tempbuf_searchidx
**)&tempbuf
[tempbuf_pos
];
2378 tempbuf_pos
+= lookup_buffer_depth
* sizeof(void **);
2379 memset(lookup
, 0, lookup_buffer_depth
* sizeof(void **));
2381 /* And calculate the remaining data space used mainly for storing
2382 * tag data (strings). */
2383 tempbuf_left
= tempbuf_size
- tempbuf_pos
- 8;
2384 if (tempbuf_left
- TAGFILE_ENTRY_AVG_LENGTH
* commit_entry_count
< 0)
2386 logf("Buffer way too small!");
2393 * If tag file contains unique tags (sorted index), we will load
2394 * it entirely into memory so we can resort it later for use with
2397 if (TAGCACHE_IS_SORTED(index_type
))
2399 logf("loading tags...");
2400 for (i
= 0; i
< tch
.entry_count
; i
++)
2402 struct tagfile_entry entry
;
2403 int loc
= lseek(fd
, 0, SEEK_CUR
);
2406 if (ecread(fd
, &entry
, 1, tagfile_entry_ec
, tc_stat
.econ
)
2407 != sizeof(struct tagfile_entry
))
2409 logf("read error #7");
2414 if (entry
.tag_length
>= (int)sizeof(buf
))
2416 logf("too long tag #3");
2421 if (read(fd
, buf
, entry
.tag_length
) != entry
.tag_length
)
2423 logf("read error #8");
2428 /* Skip deleted entries. */
2433 * Save the tag and tag id in the memory buffer. Tag id
2434 * is saved so we can later reindex the master lookup
2435 * table when the index gets resorted.
2437 ret
= tempbuf_insert(buf
, loc
/TAGFILE_ENTRY_CHUNK_LENGTH
2438 + commit_entry_count
, entry
.idx_id
,
2439 TAGCACHE_IS_UNIQUE(index_type
));
2450 tempbufidx
= tch
.entry_count
;
2455 * Creating new index file to store the tags. No need to preload
2456 * anything whether the index type is sorted or not.
2458 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, index_type
);
2459 fd
= open(buf
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2462 logf("%s open fail", buf
);
2466 tch
.magic
= TAGCACHE_MAGIC
;
2467 tch
.entry_count
= 0;
2470 if (ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
)
2471 != sizeof(struct tagcache_header
))
2473 logf("header write failed");
2479 /* Loading the tag lookup file as "master file". */
2480 logf("Loading index file");
2481 masterfd
= open(TAGCACHE_FILE_MASTER
, O_RDWR
);
2485 logf("Creating new DB");
2486 masterfd
= open(TAGCACHE_FILE_MASTER
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2490 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER
);
2495 /* Write the header (write real values later). */
2496 memset(&tcmh
, 0, sizeof(struct master_header
));
2498 tcmh
.tch
.entry_count
= 0;
2499 tcmh
.tch
.datasize
= 0;
2501 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2503 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2508 * Master file already exists so we need to process the current
2513 if (ecread(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
) !=
2514 sizeof(struct master_header
) || tcmh
.tch
.magic
!= TAGCACHE_MAGIC
)
2516 logf("header error");
2523 * If we reach end of the master file, we need to expand it to
2524 * hold new tags. If the current index is not sorted, we can
2525 * simply append new data to end of the file.
2526 * However, if the index is sorted, we need to update all tag
2527 * pointers in the master file for the current index.
2529 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2531 if (masterfd_pos
== filesize(masterfd
))
2533 logf("appending...");
2539 * Load new unique tags in memory to be sorted later and added
2540 * to the master lookup file.
2542 if (TAGCACHE_IS_SORTED(index_type
))
2544 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2545 /* h is the header of the temporary file containing new tags. */
2546 logf("inserting new tags...");
2547 for (i
= 0; i
< h
->entry_count
; i
++)
2549 struct temp_file_entry entry
;
2551 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2552 sizeof(struct temp_file_entry
))
2554 logf("read fail #3");
2560 if (entry
.tag_length
[index_type
] >= (long)sizeof(buf
))
2562 logf("too long entry!");
2567 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2568 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2569 entry
.tag_length
[index_type
])
2571 logf("read fail #4");
2576 if (TAGCACHE_IS_UNIQUE(index_type
))
2577 error
= !tempbuf_insert(buf
, i
, -1, true);
2579 error
= !tempbuf_insert(buf
, i
, tcmh
.tch
.entry_count
+ i
, false);
2583 logf("insert error");
2588 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2589 entry
.tag_length
[index_type
], SEEK_CUR
);
2594 /* Sort the buffer data and write it to the index file. */
2595 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
2597 * We need to truncate the index file now. There can be junk left
2598 * at the end of file (however, we _should_ always follow the
2599 * entry_count and don't crash with that).
2601 ftruncate(fd
, lseek(fd
, 0, SEEK_CUR
));
2603 i
= tempbuf_sort(fd
);
2606 logf("sorted %d tags", i
);
2609 * Now update all indexes in the master lookup file.
2611 logf("updating indices...");
2612 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2613 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
+= idxbuf_pos
)
2616 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2618 idxbuf_pos
= MIN(tcmh
.tch
.entry_count
- i
, IDX_BUF_DEPTH
);
2620 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2621 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2623 logf("read fail #5");
2627 lseek(masterfd
, loc
, SEEK_SET
);
2629 for (j
= 0; j
< idxbuf_pos
; j
++)
2631 if (idxbuf
[j
].flag
& FLAG_DELETED
)
2633 /* We can just ignore deleted entries. */
2634 // idxbuf[j].tag_seek[index_type] = 0;
2638 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(
2639 idxbuf
[j
].tag_seek
[index_type
]/TAGFILE_ENTRY_CHUNK_LENGTH
2640 + commit_entry_count
);
2642 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2644 logf("update error: %d/%d/%d",
2645 idxbuf
[j
].flag
, i
+j
, tcmh
.tch
.entry_count
);
2653 /* Write back the updated index. */
2654 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2655 index_entry_ec
, tc_stat
.econ
) !=
2656 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2667 * Walk through the temporary file containing the new tags.
2669 // build_normal_index(h, tmpfd, masterfd, idx);
2670 logf("updating new indices...");
2671 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2672 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2673 lseek(fd
, 0, SEEK_END
);
2674 for (i
= 0; i
< h
->entry_count
; i
+= idxbuf_pos
)
2678 idxbuf_pos
= MIN(h
->entry_count
- i
, IDX_BUF_DEPTH
);
2681 memset(idxbuf
, 0, sizeof(struct index_entry
)*IDX_BUF_DEPTH
);
2685 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2687 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2688 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2690 logf("read fail #6");
2694 lseek(masterfd
, loc
, SEEK_SET
);
2697 /* Read entry headers. */
2698 for (j
= 0; j
< idxbuf_pos
; j
++)
2700 if (!TAGCACHE_IS_SORTED(index_type
))
2702 struct temp_file_entry entry
;
2703 struct tagfile_entry fe
;
2705 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2706 sizeof(struct temp_file_entry
))
2708 logf("read fail #7");
2714 if (entry
.tag_length
[index_type
] >= (int)sizeof(buf
))
2716 logf("too long entry!");
2717 logf("length=%d", entry
.tag_length
[index_type
]);
2718 logf("pos=0x%02lx", lseek(tmpfd
, 0, SEEK_CUR
));
2723 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2724 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2725 entry
.tag_length
[index_type
])
2727 logf("read fail #8");
2728 logf("offset=0x%02lx", entry
.tag_offset
[index_type
]);
2729 logf("length=0x%02x", entry
.tag_length
[index_type
]);
2734 /* Write to index file. */
2735 idxbuf
[j
].tag_seek
[index_type
] = lseek(fd
, 0, SEEK_CUR
);
2736 fe
.tag_length
= entry
.tag_length
[index_type
];
2737 fe
.idx_id
= tcmh
.tch
.entry_count
+ i
+ j
;
2738 ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
2739 write(fd
, buf
, fe
.tag_length
);
2743 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2744 entry
.tag_length
[index_type
], SEEK_CUR
);
2748 /* Locate the correct entry from the sorted array. */
2749 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(i
+ j
);
2750 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2752 logf("entry not found (%d)", j
);
2760 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2761 index_entry_ec
, tc_stat
.econ
) !=
2762 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2764 logf("tagcache: write fail #4");
2773 /* Finally write the header. */
2774 tch
.magic
= TAGCACHE_MAGIC
;
2775 tch
.entry_count
= tempbufidx
;
2776 tch
.datasize
= lseek(fd
, 0, SEEK_END
) - sizeof(struct tagcache_header
);
2777 lseek(fd
, 0, SEEK_SET
);
2778 ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
);
2780 if (index_type
!= tag_filename
)
2781 h
->datasize
+= tch
.datasize
;
2782 logf("s:%d/%ld/%ld", index_type
, tch
.datasize
, h
->datasize
);
2794 static bool commit(void)
2796 struct tagcache_header tch
;
2797 struct master_header tcmh
;
2801 #ifdef HAVE_DIRCACHE
2802 bool dircache_buffer_stolen
= false;
2804 bool local_allocation
= false;
2806 logf("committing tagcache");
2811 tmpfd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
2814 logf("nothing to commit");
2819 /* Load the header. */
2820 len
= sizeof(struct tagcache_header
);
2821 rc
= read(tmpfd
, &tch
, len
);
2823 if (tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= len
)
2825 logf("incorrect header");
2827 remove(TAGCACHE_FILE_TEMP
);
2831 if (tch
.entry_count
== 0)
2833 logf("nothing to commit");
2835 remove(TAGCACHE_FILE_TEMP
);
2839 #ifdef HAVE_EEPROM_SETTINGS
2840 remove(TAGCACHE_STATEFILE
);
2843 /* At first be sure to unload the ramcache! */
2844 #ifdef HAVE_TC_RAMCACHE
2845 tc_stat
.ramcache
= false;
2850 /* Try to steal every buffer we can :) */
2851 if (tempbuf_size
== 0)
2852 local_allocation
= true;
2854 #ifdef HAVE_DIRCACHE
2855 if (tempbuf_size
== 0)
2857 /* Try to steal the dircache buffer. */
2858 tempbuf
= dircache_steal_buffer(&tempbuf_size
);
2859 tempbuf_size
&= ~0x03;
2861 if (tempbuf_size
> 0)
2863 dircache_buffer_stolen
= true;
2868 #ifdef HAVE_TC_RAMCACHE
2869 if (tempbuf_size
== 0 && tc_stat
.ramcache_allocated
> 0)
2871 tempbuf
= (char *)(hdr
+ 1);
2872 tempbuf_size
= tc_stat
.ramcache_allocated
- sizeof(struct ramcache_header
) - 128;
2873 tempbuf_size
&= ~0x03;
2877 /* And finally fail if there are no buffers available. */
2878 if (tempbuf_size
== 0)
2880 logf("delaying commit until next boot");
2881 tc_stat
.commit_delayed
= true;
2887 logf("commit %ld entries...", tch
.entry_count
);
2889 /* Mark DB dirty so it will stay disabled if commit fails. */
2890 current_tcmh
.dirty
= true;
2891 update_master_header();
2893 /* Now create the index files. */
2894 tc_stat
.commit_step
= 0;
2896 tc_stat
.commit_delayed
= false;
2898 for (i
= 0; i
< TAG_COUNT
; i
++)
2902 if (TAGCACHE_IS_NUMERIC(i
))
2905 tc_stat
.commit_step
++;
2906 ret
= build_index(i
, &tch
, tmpfd
);
2910 logf("tagcache failed init");
2912 tc_stat
.commit_delayed
= true;
2914 tc_stat
.commit_step
= 0;
2920 if (!build_numeric_indices(&tch
, tmpfd
))
2922 logf("Failure to commit numeric indices");
2924 tc_stat
.commit_step
= 0;
2930 remove(TAGCACHE_FILE_TEMP
);
2932 tc_stat
.commit_step
= 0;
2934 /* Update the master index headers. */
2935 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2941 tcmh
.tch
.entry_count
+= tch
.entry_count
;
2942 tcmh
.tch
.datasize
= sizeof(struct master_header
)
2943 + sizeof(struct index_entry
) * tcmh
.tch
.entry_count
2948 lseek(masterfd
, 0, SEEK_SET
);
2949 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2952 logf("tagcache committed");
2953 tc_stat
.ready
= check_all_headers();
2954 tc_stat
.readyvalid
= true;
2956 if (local_allocation
)
2962 #ifdef HAVE_DIRCACHE
2963 /* Rebuild the dircache, if we stole the buffer. */
2964 if (dircache_buffer_stolen
)
2968 #ifdef HAVE_TC_RAMCACHE
2969 /* Reload tagcache. */
2970 if (tc_stat
.ramcache_allocated
> 0)
2971 tagcache_start_scan();
2979 static void allocate_tempbuf(void)
2981 /* Yeah, malloc would be really nice now :) */
2983 tempbuf_size
= 32*1024*1024;
2984 tempbuf
= malloc(tempbuf_size
);
2986 tempbuf
= (char *)(((long)audiobuf
& ~0x03) + 0x04);
2987 tempbuf_size
= (long)audiobufend
- (long)audiobuf
- 4;
2988 audiobuf
+= tempbuf_size
;
2992 static void free_tempbuf(void)
2994 if (tempbuf_size
== 0)
3000 audiobuf
-= tempbuf_size
;
3008 static bool modify_numeric_entry(int masterfd
, int idx_id
, int tag
, long data
)
3010 struct index_entry idx
;
3015 if (!TAGCACHE_IS_NUMERIC(tag
))
3018 if (!get_index(masterfd
, idx_id
, &idx
, false))
3021 idx
.tag_seek
[tag
] = data
;
3022 idx
.flag
|= FLAG_DIRTYNUM
;
3024 return write_index(masterfd
, idx_id
, &idx
);
3028 bool tagcache_modify_numeric_entry(struct tagcache_search
*tcs
,
3031 struct master_header myhdr
;
3033 if (tcs
->masterfd
< 0)
3035 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, true)) < 0)
3039 return modify_numeric_entry(tcs
->masterfd
, tcs
->idx_id
, tag
, data
);
3043 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3045 static bool command_queue_is_full(void)
3049 next
= command_queue_widx
+ 1;
3050 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3053 return (next
== command_queue_ridx
);
3056 static bool command_queue_sync_callback(void)
3059 struct master_header myhdr
;
3062 mutex_lock(&command_queue_mutex
);
3064 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3067 while (command_queue_ridx
!= command_queue_widx
)
3069 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_ridx
];
3071 switch (ce
->command
)
3073 case CMD_UPDATE_MASTER_HEADER
:
3076 update_master_header();
3078 /* Re-open the masterfd. */
3079 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3084 case CMD_UPDATE_NUMERIC
:
3086 modify_numeric_entry(masterfd
, ce
->idx_id
, ce
->tag
, ce
->data
);
3091 if (++command_queue_ridx
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3092 command_queue_ridx
= 0;
3097 tc_stat
.queue_length
= 0;
3098 mutex_unlock(&command_queue_mutex
);
3102 static void run_command_queue(bool force
)
3104 if (COMMAND_QUEUE_IS_EMPTY
)
3107 if (force
|| command_queue_is_full())
3108 command_queue_sync_callback();
3110 register_storage_idle_func(command_queue_sync_callback
);
3113 static void queue_command(int cmd
, long idx_id
, int tag
, long data
)
3119 mutex_lock(&command_queue_mutex
);
3120 next
= command_queue_widx
+ 1;
3121 if (next
>= TAGCACHE_COMMAND_QUEUE_LENGTH
)
3124 /* Make sure queue is not full. */
3125 if (next
!= command_queue_ridx
)
3127 struct tagcache_command_entry
*ce
= &command_queue
[command_queue_widx
];
3130 ce
->idx_id
= idx_id
;
3134 command_queue_widx
= next
;
3136 tc_stat
.queue_length
++;
3138 mutex_unlock(&command_queue_mutex
);
3142 /* Queue is full, try again later... */
3143 mutex_unlock(&command_queue_mutex
);
3148 long tagcache_increase_serial(void)
3158 old
= current_tcmh
.serial
++;
3159 queue_command(CMD_UPDATE_MASTER_HEADER
, 0, 0, 0);
3164 void tagcache_update_numeric(int idx_id
, int tag
, long data
)
3166 queue_command(CMD_UPDATE_NUMERIC
, idx_id
, tag
, data
);
3168 #endif /* !__PCTOOL__ */
3170 long tagcache_get_serial(void)
3172 return current_tcmh
.serial
;
3175 long tagcache_get_commitid(void)
3177 return current_tcmh
.commitid
;
3180 static bool write_tag(int fd
, const char *tagstr
, const char *datastr
)
3185 snprintf(buf
, sizeof buf
, "%s=\"", tagstr
);
3186 for (i
= strlen(buf
); i
< (long)sizeof(buf
)-4; i
++)
3188 if (*datastr
== '\0')
3191 if (*datastr
== '"' || *datastr
== '\\')
3194 buf
[i
] = *(datastr
++);
3197 strcpy(&buf
[i
], "\" ");
3199 write(fd
, buf
, i
+ 2);
3206 static bool read_tag(char *dest
, long size
,
3207 const char *src
, const char *tagstr
)
3210 char current_tag
[32];
3212 while (*src
!= '\0')
3214 /* Skip all whitespace */
3222 /* Read in tag name */
3223 while (*src
!= '=' && *src
!= ' ')
3225 current_tag
[pos
] = *src
;
3229 if (*src
== '\0' || pos
>= (long)sizeof(current_tag
))
3232 current_tag
[pos
] = '\0';
3234 /* Read in tag data */
3236 /* Find the start. */
3237 while (*src
!= '"' && *src
!= '\0')
3240 if (*src
== '\0' || *(++src
) == '\0')
3243 /* Read the data. */
3244 for (pos
= 0; pos
< size
; pos
++)
3251 dest
[pos
] = *(src
+1);
3271 if (!strcasecmp(tagstr
, current_tag
))
3278 static int parse_changelog_line(int line_n
, const char *buf
, void *parameters
)
3280 struct index_entry idx
;
3281 char tag_data
[TAG_MAXLEN
+32];
3283 long masterfd
= (long)parameters
;
3284 const int import_tags
[] = { tag_playcount
, tag_rating
, tag_playtime
, tag_lastplayed
,
3292 logf("%d/%s", line_n
, buf
);
3293 if (!read_tag(tag_data
, sizeof tag_data
, buf
, "filename"))
3295 logf("filename missing");
3300 idx_id
= find_index(tag_data
);
3303 logf("entry not found");
3307 if (!get_index(masterfd
, idx_id
, &idx
, false))
3309 logf("failed to retrieve index entry");
3313 /* Stop if tag has already been modified. */
3314 if (idx
.flag
& FLAG_DIRTYNUM
)
3317 logf("import: %s", tag_data
);
3319 idx
.flag
|= FLAG_DIRTYNUM
;
3320 for (i
= 0; i
< (long)(sizeof(import_tags
)/sizeof(import_tags
[0])); i
++)
3324 if (!read_tag(tag_data
, sizeof tag_data
, buf
,
3325 tagcache_tag_to_str(import_tags
[i
])))
3330 data
= atoi(tag_data
);
3334 idx
.tag_seek
[import_tags
[i
]] = data
;
3336 if (import_tags
[i
] == tag_lastplayed
&& data
>= current_tcmh
.serial
)
3337 current_tcmh
.serial
= data
+ 1;
3338 else if (import_tags
[i
] == tag_commitid
&& data
>= current_tcmh
.commitid
)
3339 current_tcmh
.commitid
= data
+ 1;
3342 return write_index(masterfd
, idx_id
, &idx
) ? 0 : -5;
3345 bool tagcache_import_changelog(void)
3347 struct master_header myhdr
;
3348 struct tagcache_header tch
;
3359 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_RDONLY
);
3362 logf("failure to open changelog");
3366 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
3374 filenametag_fd
= open_tag_fd(&tch
, tag_filename
, false);
3376 fast_readline(clfd
, buf
, sizeof buf
, (long *)masterfd
,
3377 parse_changelog_line
);
3382 if (filenametag_fd
>= 0)
3384 close(filenametag_fd
);
3385 filenametag_fd
= -1;
3390 update_master_header();
3395 #endif /* !__PCTOOL__ */
3397 bool tagcache_create_changelog(struct tagcache_search
*tcs
)
3399 struct master_header myhdr
;
3400 struct index_entry idx
;
3401 char buf
[TAG_MAXLEN
+32];
3409 if (!tagcache_search(tcs
, tag_filename
))
3412 /* Initialize the changelog */
3413 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_WRONLY
| O_CREAT
| O_TRUNC
);
3416 logf("failure to open changelog");
3420 if (tcs
->masterfd
< 0)
3422 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, false)) < 0)
3427 lseek(tcs
->masterfd
, 0, SEEK_SET
);
3428 ecread(tcs
->masterfd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
3431 write(clfd
, "## Changelog version 1\n", 23);
3433 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3435 if (ecread(tcs
->masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
3436 != sizeof(struct index_entry
))
3438 logf("read error #9");
3439 tagcache_search_finish(tcs
);
3444 /* Skip until the entry found has been modified. */
3445 if (! (idx
.flag
& FLAG_DIRTYNUM
) )
3448 /* Skip deleted entries too. */
3449 if (idx
.flag
& FLAG_DELETED
)
3452 /* Now retrieve all tags. */
3453 for (j
= 0; j
< TAG_COUNT
; j
++)
3455 if (TAGCACHE_IS_NUMERIC(j
))
3457 snprintf(temp
, sizeof temp
, "%d", idx
.tag_seek
[j
]);
3458 write_tag(clfd
, tagcache_tag_to_str(j
), temp
);
3463 tagcache_retrieve(tcs
, i
, tcs
->type
, buf
, sizeof buf
);
3464 write_tag(clfd
, tagcache_tag_to_str(j
), buf
);
3467 write(clfd
, "\n", 1);
3473 tagcache_search_finish(tcs
);
3478 static bool delete_entry(long idx_id
)
3483 struct index_entry idx
, myidx
;
3484 struct master_header myhdr
;
3485 char buf
[TAG_MAXLEN
+32];
3486 int in_use
[TAG_COUNT
];
3488 logf("delete_entry(): %ld", idx_id
);
3490 #ifdef HAVE_TC_RAMCACHE
3491 /* At first mark the entry removed from ram cache. */
3492 if (tc_stat
.ramcache
)
3493 hdr
->indices
[idx_id
].flag
|= FLAG_DELETED
;
3496 if ( (masterfd
= open_master_fd(&myhdr
, true) ) < 0)
3499 lseek(masterfd
, idx_id
* sizeof(struct index_entry
), SEEK_CUR
);
3500 if (ecread(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3501 != sizeof(struct index_entry
))
3503 logf("delete_entry(): read error");
3507 if (myidx
.flag
& FLAG_DELETED
)
3509 logf("delete_entry(): already deleted!");
3513 myidx
.flag
|= FLAG_DELETED
;
3514 lseek(masterfd
, -sizeof(struct index_entry
), SEEK_CUR
);
3515 if (ecwrite(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3516 != sizeof(struct index_entry
))
3518 logf("delete_entry(): write_error #1");
3522 /* Now check which tags are no longer in use (if any) */
3523 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3526 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
3527 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3529 struct index_entry
*idxp
;
3531 #ifdef HAVE_TC_RAMCACHE
3532 /* Use RAM DB if available for greater speed */
3533 if (tc_stat
.ramcache
)
3534 idxp
= &hdr
->indices
[i
];
3538 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
3539 != sizeof(struct index_entry
))
3541 logf("delete_entry(): read error #2");
3547 if (idxp
->flag
& FLAG_DELETED
)
3550 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3552 if (TAGCACHE_IS_NUMERIC(tag
))
3555 if (idxp
->tag_seek
[tag
] == myidx
.tag_seek
[tag
])
3560 /* Now delete all tags no longer in use. */
3561 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3563 struct tagcache_header tch
;
3564 int oldseek
= myidx
.tag_seek
[tag
];
3566 if (TAGCACHE_IS_NUMERIC(tag
))
3570 * Replace tag seek with a hash value of the field string data.
3571 * That way runtime statistics of moved or altered files can be
3574 #ifdef HAVE_TC_RAMCACHE
3575 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3577 struct tagfile_entry
*tfe
;
3578 int32_t *seek
= &hdr
->indices
[idx_id
].tag_seek
[tag
];
3580 tfe
= (struct tagfile_entry
*)&hdr
->tags
[tag
][*seek
];
3581 *seek
= crc_32(tfe
->tag_data
, strlen(tfe
->tag_data
), 0xffffffff);
3582 myidx
.tag_seek
[tag
] = *seek
;
3587 struct tagfile_entry tfe
;
3589 /* Open the index file, which contains the tag names. */
3590 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3593 /* Skip the header block */
3594 lseek(fd
, myidx
.tag_seek
[tag
], SEEK_SET
);
3595 if (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
3596 != sizeof(struct tagfile_entry
))
3598 logf("delete_entry(): read error #3");
3602 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
3604 logf("delete_entry(): read error #4");
3608 myidx
.tag_seek
[tag
] = crc_32(buf
, strlen(buf
), 0xffffffff);
3613 logf("in use: %d/%d", tag
, in_use
[tag
]);
3622 #ifdef HAVE_TC_RAMCACHE
3623 /* Delete from ram. */
3624 if (tc_stat
.ramcache
&& tag
!= tag_filename
)
3626 struct tagfile_entry
*tagentry
= (struct tagfile_entry
*)&hdr
->tags
[tag
][oldseek
];
3627 tagentry
->tag_data
[0] = '\0';
3631 /* Open the index file, which contains the tag names. */
3634 if ((fd
= open_tag_fd(&tch
, tag
, true)) < 0)
3638 /* Skip the header block */
3639 lseek(fd
, oldseek
+ sizeof(struct tagfile_entry
), SEEK_SET
);
3641 /* Debug, print 10 first characters of the tag
3644 logf("TAG:%s", buf);
3645 lseek(fd, -10, SEEK_CUR);
3648 /* Write first data byte in tag as \0 */
3651 /* Now tag data has been removed */
3656 /* Write index entry back into master index. */
3657 lseek(masterfd
, sizeof(struct master_header
) +
3658 (idx_id
* sizeof(struct index_entry
)), SEEK_SET
);
3659 if (ecwrite(masterfd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3660 != sizeof(struct index_entry
))
3662 logf("delete_entry(): write_error #2");
3681 * Returns true if there is an event waiting in the queue
3682 * that requires the current operation to be aborted.
3684 static bool check_event_queue(void)
3686 struct queue_event ev
;
3688 queue_wait_w_tmo(&tagcache_queue
, &ev
, 0);
3693 case SYS_USB_CONNECTED
:
3694 /* Put the event back into the queue. */
3695 queue_post(&tagcache_queue
, ev
.id
, ev
.data
);
3703 #ifdef HAVE_TC_RAMCACHE
3704 static bool allocate_tagcache(void)
3706 struct master_header tcmh
;
3709 /* Load the header. */
3710 if ( (fd
= open_master_fd(&tcmh
, false)) < 0)
3719 * Now calculate the required cache size plus
3720 * some extra space for alignment fixes.
3722 tc_stat
.ramcache_allocated
= tcmh
.tch
.datasize
+ 128 + TAGCACHE_RESERVE
+
3723 sizeof(struct ramcache_header
) + TAG_COUNT
*sizeof(void *);
3724 hdr
= buffer_alloc(tc_stat
.ramcache_allocated
+ 128);
3725 memset(hdr
, 0, sizeof(struct ramcache_header
));
3726 memcpy(&hdr
->h
, &tcmh
, sizeof(struct master_header
));
3727 hdr
->indices
= (struct index_entry
*)(hdr
+ 1);
3728 logf("tagcache: %d bytes allocated.", tc_stat
.ramcache_allocated
);
3733 # ifdef HAVE_EEPROM_SETTINGS
3734 static bool tagcache_dumpload(void)
3736 struct statefile_header shdr
;
3741 fd
= open(TAGCACHE_STATEFILE
, O_RDONLY
);
3744 logf("no tagcache statedump");
3748 /* Check the statefile memory placement */
3749 hdr
= buffer_alloc(0);
3750 rc
= read(fd
, &shdr
, sizeof(struct statefile_header
));
3751 if (rc
!= sizeof(struct statefile_header
)
3752 /* || (long)hdr != (long)shdr.hdr */)
3754 logf("incorrect statefile");
3760 offpos
= (long)hdr
- (long)shdr
.hdr
;
3762 /* Lets allocate real memory and load it */
3763 hdr
= buffer_alloc(shdr
.tc_stat
.ramcache_allocated
);
3764 rc
= read(fd
, hdr
, shdr
.tc_stat
.ramcache_allocated
);
3767 if (rc
!= shdr
.tc_stat
.ramcache_allocated
)
3769 logf("read failure!");
3774 memcpy(&tc_stat
, &shdr
.tc_stat
, sizeof(struct tagcache_stat
));
3776 /* Now fix the pointers */
3777 hdr
->indices
= (struct index_entry
*)((long)hdr
->indices
+ offpos
);
3778 for (i
= 0; i
< TAG_COUNT
; i
++)
3779 hdr
->tags
[i
] += offpos
;
3784 static bool tagcache_dumpsave(void)
3786 struct statefile_header shdr
;
3789 if (!tc_stat
.ramcache
)
3792 fd
= open(TAGCACHE_STATEFILE
, O_WRONLY
| O_CREAT
| O_TRUNC
);
3795 logf("failed to create a statedump");
3799 /* Create the header */
3801 memcpy(&shdr
.tc_stat
, &tc_stat
, sizeof(struct tagcache_stat
));
3802 write(fd
, &shdr
, sizeof(struct statefile_header
));
3804 /* And dump the data too */
3805 write(fd
, hdr
, tc_stat
.ramcache_allocated
);
3812 static bool load_tagcache(void)
3814 struct tagcache_header
*tch
;
3815 long bytesleft
= tc_stat
.ramcache_allocated
;
3816 struct index_entry
*idx
;
3821 # ifdef HAVE_DIRCACHE
3822 while (dircache_is_initializing())
3825 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE
);
3828 logf("loading tagcache to ram...");
3830 fd
= open(TAGCACHE_FILE_MASTER
, O_RDONLY
);
3833 logf("tagcache open failed");
3837 if (ecread(fd
, &hdr
->h
, 1, master_header_ec
, tc_stat
.econ
)
3838 != sizeof(struct master_header
)
3839 || hdr
->h
.tch
.magic
!= TAGCACHE_MAGIC
)
3841 logf("incorrect header");
3847 /* Load the master index table. */
3848 for (i
= 0; i
< hdr
->h
.tch
.entry_count
; i
++)
3850 rc
= ecread(fd
, idx
, 1, index_entry_ec
, tc_stat
.econ
);
3851 if (rc
!= sizeof(struct index_entry
))
3853 logf("read error #10");
3858 bytesleft
-= sizeof(struct index_entry
);
3859 if (bytesleft
< 0 || ((long)idx
- (long)hdr
->indices
) >= tc_stat
.ramcache_allocated
)
3861 logf("too big tagcache.");
3871 /* Load the tags. */
3873 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3875 struct tagfile_entry
*fe
;
3876 char buf
[TAG_MAXLEN
+32];
3878 if (TAGCACHE_IS_NUMERIC(tag
))
3881 //p = ((void *)p+1);
3882 p
= (char *)((long)p
& ~0x03) + 0x04;
3885 /* Check the header. */
3886 tch
= (struct tagcache_header
*)p
;
3887 p
+= sizeof(struct tagcache_header
);
3889 if ( (fd
= open_tag_fd(tch
, tag
, false)) < 0)
3892 for (hdr
->entry_count
[tag
] = 0;
3893 hdr
->entry_count
[tag
] < tch
->entry_count
;
3894 hdr
->entry_count
[tag
]++)
3898 if (do_timed_yield())
3900 /* Abort if we got a critical event in queue */
3901 if (check_event_queue())
3905 fe
= (struct tagfile_entry
*)p
;
3906 pos
= lseek(fd
, 0, SEEK_CUR
);
3907 rc
= ecread(fd
, fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
3908 if (rc
!= sizeof(struct tagfile_entry
))
3910 /* End of lookup table. */
3911 logf("read error #11");
3916 /* We have a special handling for the filename tags. */
3917 if (tag
== tag_filename
)
3919 # ifdef HAVE_DIRCACHE
3920 const struct dirent
*dc
;
3923 // FIXME: This is wrong!
3924 // idx = &hdr->indices[hdr->entry_count[i]];
3925 idx
= &hdr
->indices
[fe
->idx_id
];
3927 if (fe
->tag_length
>= (long)sizeof(buf
)-1)
3931 logf("TAG:%s", buf
);
3932 logf("too long filename");
3937 rc
= read(fd
, buf
, fe
->tag_length
);
3938 if (rc
!= fe
->tag_length
)
3940 logf("read error #12");
3945 /* Check if the entry has already been removed */
3946 if (idx
->flag
& FLAG_DELETED
)
3949 /* This flag must not be used yet. */
3950 if (idx
->flag
& FLAG_DIRCACHE
)
3952 logf("internal error!");
3957 if (idx
->tag_seek
[tag
] != pos
)
3959 logf("corrupt data structures!");
3964 # ifdef HAVE_DIRCACHE
3965 if (dircache_is_enabled())
3967 dc
= dircache_get_entry_ptr(buf
);
3970 logf("Entry no longer valid.");
3972 delete_entry(fe
->idx_id
);
3976 idx
->flag
|= FLAG_DIRCACHE
;
3977 idx
->tag_seek
[tag_filename
] = (long)dc
;
3982 /* This will be very slow unless dircache is enabled
3983 or target is flash based, but do it anyway for
3985 /* Check if entry has been removed. */
3986 if (global_settings
.tagcache_autoupdate
)
3988 if (!file_exists(buf
))
3990 logf("Entry no longer valid.");
3992 delete_entry(fe
->idx_id
);
4001 bytesleft
-= sizeof(struct tagfile_entry
) + fe
->tag_length
;
4004 logf("too big tagcache #2");
4005 logf("tl: %d", fe
->tag_length
);
4006 logf("bl: %ld", bytesleft
);
4012 rc
= read(fd
, fe
->tag_data
, fe
->tag_length
);
4015 if (rc
!= fe
->tag_length
)
4017 logf("read error #13");
4018 logf("rc=0x%04x", rc
); // 0x431
4019 logf("len=0x%04x", fe
->tag_length
); // 0x4000
4020 logf("pos=0x%04lx", lseek(fd
, 0, SEEK_CUR
)); // 0x433
4021 logf("tag=0x%02x", tag
); // 0x00
4029 tc_stat
.ramcache_used
= tc_stat
.ramcache_allocated
- bytesleft
;
4030 logf("tagcache loaded into ram!");
4034 #endif /* HAVE_TC_RAMCACHE */
4036 static bool check_deleted_files(void)
4039 char buf
[TAG_MAXLEN
+32];
4040 struct tagfile_entry tfe
;
4042 logf("reverse scan...");
4043 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag_filename
);
4044 fd
= open(buf
, O_RDONLY
);
4048 logf("%s open fail", buf
);
4052 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
4053 while (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
4054 == sizeof(struct tagfile_entry
)
4056 && !check_event_queue()
4060 if (tfe
.tag_length
>= (long)sizeof(buf
)-1)
4062 logf("too long tag");
4067 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
4069 logf("read error #14");
4074 /* Check if the file has already deleted from the db. */
4078 /* Now check if the file exists. */
4079 if (!file_exists(buf
))
4081 logf("Entry no longer valid.");
4082 logf("-> %s / %d", buf
, tfe
.tag_length
);
4083 delete_entry(tfe
.idx_id
);
4094 static bool check_dir(const char *dirname
, int add_files
)
4098 int success
= false;
4099 int ignore
, unignore
;
4100 char newpath
[MAX_PATH
];
4102 dir
= opendir(dirname
);
4105 logf("tagcache: opendir() failed");
4109 /* check for a database.ignore file */
4110 snprintf(newpath
, MAX_PATH
, "%s/database.ignore", dirname
);
4111 ignore
= file_exists(newpath
);
4112 /* check for a database.unignore file */
4113 snprintf(newpath
, MAX_PATH
, "%s/database.unignore", dirname
);
4114 unignore
= file_exists(newpath
);
4116 /* don't do anything if both ignore and unignore are there */
4117 if (ignore
!= unignore
)
4118 add_files
= unignore
;
4120 /* Recursively scan the dir. */
4124 while (!check_event_queue())
4127 struct dirent
*entry
;
4129 entry
= readdir(dir
);
4137 if (!strcmp((char *)entry
->d_name
, ".") ||
4138 !strcmp((char *)entry
->d_name
, ".."))
4143 len
= strlen(curpath
);
4144 snprintf(&curpath
[len
], sizeof(curpath
) - len
, "/%s",
4147 processed_dir_count
++;
4148 if (entry
->attribute
& ATTR_DIRECTORY
)
4149 check_dir(curpath
, add_files
);
4152 tc_stat
.curentry
= curpath
;
4154 /* Add a new entry to the temporary db file. */
4155 add_tagcache(curpath
, (entry
->wrtdate
<< 16) | entry
->wrttime
4156 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4157 , dir
->internal_entry
4161 /* Wait until current path for debug screen is read and unset. */
4162 while (tc_stat
.syncscreen
&& tc_stat
.curentry
!= NULL
)
4165 tc_stat
.curentry
= NULL
;
4168 curpath
[len
] = '\0';
4176 void tagcache_screensync_event(void)
4178 tc_stat
.curentry
= NULL
;
4181 void tagcache_screensync_enable(bool state
)
4183 tc_stat
.syncscreen
= state
;
4186 void tagcache_build(const char *path
)
4188 struct tagcache_header header
;
4193 total_entry_count
= 0;
4194 processed_dir_count
= 0;
4196 #ifdef HAVE_DIRCACHE
4197 while (dircache_is_initializing())
4201 logf("updating tagcache");
4203 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
4206 logf("skipping, cache already waiting for commit");
4211 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDWR
| O_CREAT
| O_TRUNC
);
4214 logf("master file open failed: %s", TAGCACHE_FILE_TEMP
);
4218 filenametag_fd
= open_tag_fd(&header
, tag_filename
, false);
4222 logf("Scanning files...");
4223 /* Scan for new files. */
4224 memset(&header
, 0, sizeof(struct tagcache_header
));
4225 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4227 if (strcmp("/", path
) != 0)
4228 strcpy(curpath
, path
);
4229 ret
= check_dir(path
, true);
4231 /* Write the header. */
4232 header
.magic
= TAGCACHE_MAGIC
;
4233 header
.datasize
= data_size
;
4234 header
.entry_count
= total_entry_count
;
4235 lseek(cachefd
, 0, SEEK_SET
);
4236 write(cachefd
, &header
, sizeof(struct tagcache_header
));
4239 if (filenametag_fd
>= 0)
4241 close(filenametag_fd
);
4242 filenametag_fd
= -1;
4252 /* Commit changes to the database. */
4258 remove(TAGCACHE_FILE_TEMP
);
4259 logf("tagcache built!");
4265 #ifdef HAVE_TC_RAMCACHE
4268 /* Import runtime statistics if we just initialized the db. */
4269 if (hdr
->h
.serial
== 0)
4270 queue_post(&tagcache_queue
, Q_IMPORT_CHANGELOG
, 0);
4277 #ifdef HAVE_TC_RAMCACHE
4278 static void load_ramcache(void)
4285 /* At first we should load the cache (if exists). */
4286 tc_stat
.ramcache
= load_tagcache();
4288 if (!tc_stat
.ramcache
)
4290 /* If loading failed, it must indicate some problem with the db
4291 * so disable it entirely to prevent further issues. */
4292 tc_stat
.ready
= false;
4299 void tagcache_unload_ramcache(void)
4301 tc_stat
.ramcache
= false;
4302 /* Just to make sure there is no statefile present. */
4303 // remove(TAGCACHE_STATEFILE);
4308 static void tagcache_thread(void)
4310 struct queue_event ev
;
4311 bool check_done
= false;
4313 /* If the previous cache build/update was interrupted, commit
4314 * the changes first in foreground. */
4320 #ifdef HAVE_TC_RAMCACHE
4321 # ifdef HAVE_EEPROM_SETTINGS
4322 if (firmware_settings
.initialized
&& firmware_settings
.disk_clean
)
4323 check_done
= tagcache_dumpload();
4325 remove(TAGCACHE_STATEFILE
);
4328 /* Allocate space for the tagcache if found on disk. */
4329 if (global_settings
.tagcache_ram
&& !tc_stat
.ramcache
)
4330 allocate_tagcache();
4334 tc_stat
.initialized
= true;
4336 /* Don't delay bootup with the header check but do it on background. */
4338 tc_stat
.ready
= check_all_headers();
4339 tc_stat
.readyvalid
= true;
4343 run_command_queue(false);
4345 queue_wait_w_tmo(&tagcache_queue
, &ev
, HZ
);
4349 case Q_IMPORT_CHANGELOG
:
4350 tagcache_import_changelog();
4355 remove(TAGCACHE_FILE_TEMP
);
4356 tagcache_build("/");
4360 tagcache_build("/");
4361 #ifdef HAVE_TC_RAMCACHE
4364 check_deleted_files();
4370 if (check_done
|| !tc_stat
.ready
)
4373 #ifdef HAVE_TC_RAMCACHE
4374 if (!tc_stat
.ramcache
&& global_settings
.tagcache_ram
)
4377 if (tc_stat
.ramcache
&& global_settings
.tagcache_autoupdate
)
4378 tagcache_build("/");
4382 if (global_settings
.tagcache_autoupdate
)
4384 tagcache_build("/");
4386 /* This will be very slow unless dircache is enabled
4387 or target is flash based, but do it anyway for
4389 check_deleted_files();
4392 logf("tagcache check done");
4404 case SYS_USB_CONNECTED
:
4405 logf("USB: TagCache");
4406 usb_acknowledge(SYS_USB_CONNECTED_ACK
);
4407 usb_wait_for_disconnect(&tagcache_queue
);
4414 bool tagcache_prepare_shutdown(void)
4416 if (tagcache_get_commit_step() > 0)
4419 tagcache_stop_scan();
4420 while (read_lock
|| write_lock
)
4426 void tagcache_shutdown(void)
4428 /* Flush the command queue. */
4429 run_command_queue(true);
4431 #ifdef HAVE_EEPROM_SETTINGS
4432 if (tc_stat
.ramcache
)
4433 tagcache_dumpsave();
4437 static int get_progress(void)
4439 int total_count
= -1;
4441 #ifdef HAVE_DIRCACHE
4442 if (dircache_is_enabled())
4444 total_count
= dircache_get_entry_count();
4448 #ifdef HAVE_TC_RAMCACHE
4450 if (hdr
&& tc_stat
.ramcache
)
4451 total_count
= hdr
->h
.tch
.entry_count
;
4455 if (total_count
< 0)
4458 return processed_dir_count
* 100 / total_count
;
4461 struct tagcache_stat
* tagcache_get_stat(void)
4463 tc_stat
.progress
= get_progress();
4464 tc_stat
.processed_entries
= processed_dir_count
;
4469 void tagcache_start_scan(void)
4471 queue_post(&tagcache_queue
, Q_START_SCAN
, 0);
4474 bool tagcache_update(void)
4479 queue_post(&tagcache_queue
, Q_UPDATE
, 0);
4483 bool tagcache_rebuild()
4485 queue_post(&tagcache_queue
, Q_REBUILD
, 0);
4489 void tagcache_stop_scan(void)
4491 queue_post(&tagcache_queue
, Q_STOP_SCAN
, 0);
4494 #ifdef HAVE_TC_RAMCACHE
4495 bool tagcache_is_ramcache(void)
4497 return tc_stat
.ramcache
;
4501 #endif /* !__PCTOOL__ */
4504 void tagcache_init(void)
4506 memset(&tc_stat
, 0, sizeof(struct tagcache_stat
));
4507 memset(¤t_tcmh
, 0, sizeof(struct master_header
));
4508 filenametag_fd
= -1;
4509 write_lock
= read_lock
= 0;
4512 mutex_init(&command_queue_mutex
);
4513 queue_init(&tagcache_queue
, true);
4514 create_thread(tagcache_thread
, tagcache_stack
,
4515 sizeof(tagcache_stack
), 0, tagcache_thread_name
4516 IF_PRIO(, PRIORITY_BACKGROUND
)
4519 tc_stat
.initialized
= true;
4523 tc_stat
.ready
= check_all_headers();
4528 void tagcache_reverse_scan(void)
4530 logf("Checking for deleted files");
4531 check_deleted_files();
4535 bool tagcache_is_initialized(void)
4537 return tc_stat
.initialized
;
4539 bool tagcache_is_usable(void)
4541 return tc_stat
.initialized
&& tc_stat
.ready
;
4543 int tagcache_get_commit_step(void)
4545 return tc_stat
.commit_step
;
4547 int tagcache_get_max_commit_step(void)
4549 return (int)(SORTED_TAGS_COUNT
)+1;