1 /***************************************************************************
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 * Copyright (C) 2005 by Miika Pekkarinen
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
18 ****************************************************************************/
23 * ----------x---------x------------------x-----
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
77 #include "eeprom_settings.h"
82 #define yield() do { } while(0)
83 #define sim_sleep(timeout) do { } while(0)
88 /* Tag Cache thread. */
89 static struct event_queue tagcache_queue
;
90 static long tagcache_stack
[(DEFAULT_STACK_SIZE
+ 0x4000)/sizeof(long)];
91 static const char tagcache_thread_name
[] = "tagcache";
94 #define UNTAGGED "<Untagged>"
96 /* Previous path when scanning directory tree recursively. */
97 static char curpath
[TAG_MAXLEN
+32];
98 static long curpath_size
= sizeof(curpath
);
100 /* Used when removing duplicates. */
101 static char *tempbuf
; /* Allocated when needed. */
102 static long tempbufidx
; /* Current location in buffer. */
103 static long tempbuf_size
; /* Buffer size (TEMPBUF_SIZE). */
104 static long tempbuf_left
; /* Buffer space left. */
105 static long tempbuf_pos
;
107 /* Tags we want to get sorted (loaded to the tempbuf). */
108 static const int sorted_tags
[] = { tag_artist
, tag_album
, tag_genre
,
109 tag_composer
, tag_comment
, tag_albumartist
, tag_title
};
111 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
112 static const int unique_tags
[] = { tag_artist
, tag_album
, tag_genre
,
113 tag_composer
, tag_comment
, tag_albumartist
};
115 /* Numeric tags (we can use these tags with conditional clauses). */
116 static const int numeric_tags
[] = { tag_year
, tag_tracknumber
, tag_length
,
117 tag_bitrate
, tag_playcount
, tag_playtime
, tag_lastplayed
, tag_commitid
,
118 tag_virt_entryage
, tag_virt_autoscore
};
120 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
121 static const char *tags_str
[] = { "artist", "album", "genre", "title",
122 "filename", "composer", "comment", "albumartist", "year", "tracknumber",
123 "bitrate", "length", "playcount", "playtime", "lastplayed", "commitid" };
125 /* Status information of the tagcache. */
126 static struct tagcache_stat tc_stat
;
128 /* Queue commands. */
129 enum tagcache_queue
{
138 /* Tag database structures. */
140 /* Variable-length tag entry in tag files. */
141 struct tagfile_entry
{
142 short tag_length
; /* Length of the data in bytes including '\0' */
143 short idx_id
; /* Corresponding entry location in index file of not unique tags */
144 char tag_data
[0]; /* Begin of the tag data */
147 /* Fixed-size tag entry in master db index. */
149 long tag_seek
[TAG_COUNT
]; /* Location of tag data or numeric tag data */
150 long flag
; /* Status flags */
153 /* Header is the same in every file. */
154 struct tagcache_header
{
155 long magic
; /* Header version number */
156 long datasize
; /* Data size in bytes */
157 long entry_count
; /* Number of entries in this file */
160 struct master_header
{
161 struct tagcache_header tch
;
162 long serial
; /* Increasing counting number */
163 long commitid
; /* Number of commits so far */
167 /* For the endianess correction */
168 static const char *tagfile_entry_ec
= "ss";
169 static const char *index_entry_ec
= "lllllllllllllllll"; /* (1 + TAG_COUNT) * l */
170 static const char *tagcache_header_ec
= "lll";
171 static const char *master_header_ec
= "llllll";
173 static struct master_header current_tcmh
;
175 #ifdef HAVE_TC_RAMCACHE
176 /* Header is created when loading database to ram. */
177 struct ramcache_header
{
178 struct master_header h
; /* Header from the master index */
179 struct index_entry
*indices
; /* Master index file content */
180 char *tags
[TAG_COUNT
]; /* Tag file content (not including filename tag) */
181 int entry_count
[TAG_COUNT
]; /* Number of entries in the indices. */
184 # ifdef HAVE_EEPROM_SETTINGS
185 struct statefile_header
{
186 struct ramcache_header
*hdr
;
187 struct tagcache_stat tc_stat
;
191 /* Pointer to allocated ramcache_header */
192 static struct ramcache_header
*hdr
;
196 * Full tag entries stored in a temporary file waiting
197 * for commit to the cache. */
198 struct temp_file_entry
{
199 long tag_offset
[TAG_COUNT
];
200 short tag_length
[TAG_COUNT
];
206 struct tempbuf_id_list
{
208 struct tempbuf_id_list
*next
;
211 struct tempbuf_searchidx
{
215 struct tempbuf_id_list idlist
;
218 /* Lookup buffer for fixing messed up index while after sorting. */
219 static long commit_entry_count
;
220 static long lookup_buffer_depth
;
221 static struct tempbuf_searchidx
**lookup
;
223 /* Used when building the temporary file. */
224 static int cachefd
= -1, filenametag_fd
;
225 static int total_entry_count
= 0;
226 static int data_size
= 0;
227 static int processed_dir_count
;
229 /* Thread safe locking */
230 static volatile int write_lock
;
231 static volatile int read_lock
;
233 const char* tagcache_tag_to_str(int tag
)
235 return tags_str
[tag
];
238 bool tagcache_is_numeric_tag(int type
)
242 for (i
= 0; i
< (int)(sizeof(numeric_tags
)/sizeof(numeric_tags
[0])); i
++)
244 if (type
== numeric_tags
[i
])
251 bool tagcache_is_unique_tag(int type
)
255 for (i
= 0; i
< (int)(sizeof(unique_tags
)/sizeof(unique_tags
[0])); i
++)
257 if (type
== unique_tags
[i
])
264 bool tagcache_is_sorted_tag(int type
)
268 for (i
= 0; i
< (int)(sizeof(sorted_tags
)/sizeof(sorted_tags
[0])); i
++)
270 if (type
== sorted_tags
[i
])
277 static int open_tag_fd(struct tagcache_header
*hdr
, int tag
, bool write
)
283 if (tagcache_is_numeric_tag(tag
) || tag
< 0 || tag
>= TAG_COUNT
)
286 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag
);
288 fd
= open(buf
, write
? O_RDWR
: O_RDONLY
);
291 logf("tag file open failed: tag=%d write=%d file=%s", tag
, write
, buf
);
292 tc_stat
.ready
= false;
296 /* Check the header. */
297 rc
= ecread(fd
, hdr
, 1, tagcache_header_ec
, tc_stat
.econ
);
298 if (hdr
->magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct tagcache_header
))
300 logf("header error");
301 tc_stat
.ready
= false;
309 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
310 static long find_entry_ram(const char *filename
,
311 const struct dircache_entry
*dc
)
313 static long last_pos
= 0;
317 /* Check if we tagcache is loaded into ram. */
318 if (!tc_stat
.ramcache
)
322 dc
= dircache_get_entry_ptr(filename
);
326 logf("tagcache: file not found.");
337 for (; i
< hdr
->h
.tch
.entry_count
; i
++)
339 if (hdr
->indices
[i
].tag_seek
[tag_filename
] == (long)dc
)
341 last_pos
= MAX(0, i
- 3);
345 if (++counter
== 100)
362 static long find_entry_disk(const char *filename
)
364 struct tagcache_header tch
;
365 static long last_pos
= -1;
366 long pos_history
[POS_HISTORY_COUNT
];
367 long pos_history_idx
= 0;
369 struct tagfile_entry tfe
;
371 char buf
[TAG_MAXLEN
+32];
382 if ( (fd
= open_tag_fd(&tch
, tag_filename
, false)) < 0)
389 lseek(fd
, last_pos
, SEEK_SET
);
391 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
395 pos
= lseek(fd
, 0, SEEK_CUR
);
396 for (i
= pos_history_idx
-1; i
>= 0; i
--)
397 pos_history
[i
+1] = pos_history
[i
];
398 pos_history
[0] = pos
;
400 if (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
401 != sizeof(struct tagfile_entry
))
406 if (tfe
.tag_length
>= (long)sizeof(buf
))
408 logf("too long tag #1");
414 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
416 logf("read error #2");
422 if (!strcasecmp(filename
, buf
))
424 last_pos
= pos_history
[pos_history_idx
];
429 if (pos_history_idx
< POS_HISTORY_COUNT
- 1)
443 if (fd
!= filenametag_fd
)
448 if (fd
!= filenametag_fd
)
454 static int find_index(const char *filename
)
458 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
459 if (tc_stat
.ramcache
&& dircache_is_enabled())
460 idx_id
= find_entry_ram(filename
, NULL
);
464 idx_id
= find_entry_disk(filename
);
469 bool tagcache_find_index(struct tagcache_search
*tcs
, const char *filename
)
476 idx_id
= find_index(filename
);
480 if (!tagcache_search(tcs
, tag_filename
))
483 tcs
->entry_count
= 0;
484 tcs
->idx_id
= idx_id
;
489 static bool get_index(int masterfd
, int idxid
,
490 struct index_entry
*idx
, bool use_ram
)
494 logf("Incorrect idxid: %d", idxid
);
498 #ifdef HAVE_TC_RAMCACHE
499 if (tc_stat
.ramcache
&& use_ram
)
501 if (hdr
->indices
[idxid
].flag
& FLAG_DELETED
)
504 memcpy(idx
, &hdr
->indices
[idxid
], sizeof(struct index_entry
));
511 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
512 + sizeof(struct master_header
), SEEK_SET
);
513 if (ecread(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
514 != sizeof(struct index_entry
))
516 logf("read error #3");
520 if (idx
->flag
& FLAG_DELETED
)
526 static bool write_index(int masterfd
, int idxid
, struct index_entry
*idx
)
528 /* We need to exclude all memory only flags & tags when writing to disk. */
529 if (idx
->flag
& FLAG_DIRCACHE
)
531 logf("memory only flags!");
535 #ifdef HAVE_TC_RAMCACHE
536 if (tc_stat
.ramcache
)
538 memcpy(&hdr
->indices
[idxid
], idx
, sizeof(struct index_entry
));
542 lseek(masterfd
, idxid
* sizeof(struct index_entry
)
543 + sizeof(struct master_header
), SEEK_SET
);
544 if (ecwrite(masterfd
, idx
, 1, index_entry_ec
, tc_stat
.econ
)
545 != sizeof(struct index_entry
))
547 logf("write error #3");
548 logf("idxid: %d", idxid
);
555 static bool open_files(struct tagcache_search
*tcs
, int tag
)
557 if (tcs
->idxfd
[tag
] < 0)
561 snprintf(fn
, sizeof fn
, TAGCACHE_FILE_INDEX
, tag
);
562 tcs
->idxfd
[tag
] = open(fn
, O_RDONLY
);
565 if (tcs
->idxfd
[tag
] < 0)
567 logf("File not open!");
574 static bool retrieve(struct tagcache_search
*tcs
, struct index_entry
*idx
,
575 int tag
, char *buf
, long size
)
577 struct tagfile_entry tfe
;
582 if (tagcache_is_numeric_tag(tag
))
585 seek
= idx
->tag_seek
[tag
];
588 logf("Retrieve failed");
592 #ifdef HAVE_TC_RAMCACHE
595 struct tagfile_entry
*ep
;
597 # ifdef HAVE_DIRCACHE
598 if (tag
== tag_filename
&& idx
->flag
& FLAG_DIRCACHE
)
600 dircache_copy_path((struct dircache_entry
*)seek
,
606 if (tag
!= tag_filename
)
608 ep
= (struct tagfile_entry
*)&hdr
->tags
[tag
][seek
];
609 strncpy(buf
, ep
->tag_data
, size
-1);
616 if (!open_files(tcs
, tag
))
619 lseek(tcs
->idxfd
[tag
], seek
, SEEK_SET
);
620 if (ecread(tcs
->idxfd
[tag
], &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
621 != sizeof(struct tagfile_entry
))
623 logf("read error #5");
627 if (tfe
.tag_length
>= size
)
629 logf("too small buffer");
633 if (read(tcs
->idxfd
[tag
], buf
, tfe
.tag_length
) !=
636 logf("read error #6");
640 buf
[tfe
.tag_length
] = '\0';
645 static long check_virtual_tags(int tag
, const struct index_entry
*idx
)
651 case tag_virt_autoscore
:
652 if (idx
->tag_seek
[tag_length
] == 0
653 || idx
->tag_seek
[tag_playcount
] == 0)
659 data
= 100 * idx
->tag_seek
[tag_playtime
]
660 / idx
->tag_seek
[tag_length
]
661 / idx
->tag_seek
[tag_playcount
];
665 /* How many commits before the file has been added to the DB. */
666 case tag_virt_entryage
:
667 data
= current_tcmh
.commitid
- idx
->tag_seek
[tag_commitid
] - 1;
671 data
= idx
->tag_seek
[tag
];
677 long tagcache_get_numeric(const struct tagcache_search
*tcs
, int tag
)
679 struct index_entry idx
;
684 if (!tagcache_is_numeric_tag(tag
))
687 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
690 return check_virtual_tags(tag
, &idx
);
693 inline static bool str_ends_with(const char *str1
, const char *str2
)
695 int str_len
= strlen(str1
);
696 int clause_len
= strlen(str2
);
698 if (clause_len
> str_len
)
701 return !strcasecmp(&str1
[str_len
- clause_len
], str2
);
704 inline static bool str_oneof(const char *str
, const char *list
)
707 int l
, len
= strlen(str
);
711 sep
= strchr(list
, '|');
712 l
= sep
? (long)sep
- (long)list
: (int)strlen(list
);
713 if ((l
==len
) && !strncasecmp(str
, list
, len
))
715 list
+= sep
? l
+ 1 : l
;
721 static bool check_against_clause(long numeric
, const char *str
,
722 const struct tagcache_search_clause
*clause
)
726 switch (clause
->type
)
729 return numeric
== clause
->numeric_data
;
731 return numeric
!= clause
->numeric_data
;
733 return numeric
> clause
->numeric_data
;
735 return numeric
>= clause
->numeric_data
;
737 return numeric
< clause
->numeric_data
;
739 return numeric
<= clause
->numeric_data
;
741 logf("Incorrect numeric tag: %d", clause
->type
);
746 switch (clause
->type
)
749 return !strcasecmp(clause
->str
, str
);
751 return strcasecmp(clause
->str
, str
);
753 return 0>strcasecmp(clause
->str
, str
);
755 return 0>=strcasecmp(clause
->str
, str
);
757 return 0<strcasecmp(clause
->str
, str
);
759 return 0<=strcasecmp(clause
->str
, str
);
760 case clause_contains
:
761 return (strcasestr(str
, clause
->str
) != NULL
);
762 case clause_not_contains
:
763 return (strcasestr(str
, clause
->str
) == NULL
);
764 case clause_begins_with
:
765 return (strcasestr(str
, clause
->str
) == str
);
766 case clause_not_begins_with
:
767 return (strcasestr(str
, clause
->str
) != str
);
768 case clause_ends_with
:
769 return str_ends_with(str
, clause
->str
);
770 case clause_not_ends_with
:
771 return !str_ends_with(str
, clause
->str
);
773 return str_oneof(str
, clause
->str
);
776 logf("Incorrect tag: %d", clause
->type
);
783 static bool check_clauses(struct tagcache_search
*tcs
,
784 struct index_entry
*idx
,
785 struct tagcache_search_clause
**clause
, int count
)
789 #ifdef HAVE_TC_RAMCACHE
792 /* Go through all conditional clauses. */
793 for (i
= 0; i
< count
; i
++)
795 struct tagfile_entry
*tfe
;
800 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
802 if (!tagcache_is_numeric_tag(clause
[i
]->tag
))
804 if (clause
[i
]->tag
== tag_filename
)
806 retrieve(tcs
, idx
, tag_filename
, buf
, sizeof buf
);
811 tfe
= (struct tagfile_entry
*)&hdr
->tags
[clause
[i
]->tag
][seek
];
816 if (!check_against_clause(seek
, str
, clause
[i
]))
823 /* Check for conditions. */
824 for (i
= 0; i
< count
; i
++)
826 struct tagfile_entry tfe
;
830 seek
= check_virtual_tags(clause
[i
]->tag
, idx
);
832 memset(str
, 0, sizeof str
);
833 if (!tagcache_is_numeric_tag(clause
[i
]->tag
))
835 int fd
= tcs
->idxfd
[clause
[i
]->tag
];
836 lseek(fd
, seek
, SEEK_SET
);
837 ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
838 if (tfe
.tag_length
>= (int)sizeof(str
))
840 logf("Too long tag read!");
844 read(fd
, str
, tfe
.tag_length
);
846 /* Check if entry has been deleted. */
851 if (!check_against_clause(seek
, str
, clause
[i
]))
859 bool tagcache_check_clauses(struct tagcache_search
*tcs
,
860 struct tagcache_search_clause
**clause
, int count
)
862 struct index_entry idx
;
867 if (!get_index(tcs
->masterfd
, tcs
->idx_id
, &idx
, true))
870 return check_clauses(tcs
, &idx
, clause
, count
);
873 static bool add_uniqbuf(struct tagcache_search
*tcs
, unsigned long id
)
877 /* If uniq buffer is not defined we must return true for search to work. */
878 if (tcs
->unique_list
== NULL
879 || (!tagcache_is_unique_tag(tcs
->type
)
880 && !tagcache_is_numeric_tag(tcs
->type
)))
885 for (i
= 0; i
< tcs
->unique_list_count
; i
++)
887 /* Return false if entry is found. */
888 if (tcs
->unique_list
[i
] == id
)
892 if (tcs
->unique_list_count
< tcs
->unique_list_capacity
)
894 tcs
->unique_list
[i
] = id
;
895 tcs
->unique_list_count
++;
901 static bool build_lookup_list(struct tagcache_search
*tcs
)
903 struct index_entry entry
;
906 tcs
->seek_list_count
= 0;
908 #ifdef HAVE_TC_RAMCACHE
913 for (i
= tcs
->seek_pos
; i
< hdr
->h
.tch
.entry_count
; i
++)
915 struct index_entry
*idx
= &hdr
->indices
[i
];
916 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
919 /* Skip deleted files. */
920 if (idx
->flag
& FLAG_DELETED
)
923 /* Go through all filters.. */
924 for (j
= 0; j
< tcs
->filter_count
; j
++)
926 if (idx
->tag_seek
[tcs
->filter_tag
[j
]] != tcs
->filter_seek
[j
])
932 if (j
< tcs
->filter_count
)
935 /* Check for conditions. */
936 if (!check_clauses(tcs
, idx
, tcs
->clause
, tcs
->clause_count
))
939 /* Add to the seek list if not already in uniq buffer. */
940 if (!add_uniqbuf(tcs
, idx
->tag_seek
[tcs
->type
]))
944 tcs
->seek_list
[tcs
->seek_list_count
] = idx
->tag_seek
[tcs
->type
];
945 tcs
->seek_flags
[tcs
->seek_list_count
] = idx
->flag
;
946 tcs
->seek_list_count
++;
951 return tcs
->seek_list_count
> 0;
955 lseek(tcs
->masterfd
, tcs
->seek_pos
* sizeof(struct index_entry
) +
956 sizeof(struct master_header
), SEEK_SET
);
958 while (ecread(tcs
->masterfd
, &entry
, 1, index_entry_ec
, tc_stat
.econ
)
959 == sizeof(struct index_entry
))
961 /* Check if entry has been deleted. */
962 if (entry
.flag
& FLAG_DELETED
)
965 if (tcs
->seek_list_count
== SEEK_LIST_SIZE
)
968 /* Go through all filters.. */
969 for (i
= 0; i
< tcs
->filter_count
; i
++)
971 if (entry
.tag_seek
[tcs
->filter_tag
[i
]] != tcs
->filter_seek
[i
])
977 if (i
< tcs
->filter_count
)
980 /* Check for conditions. */
981 if (!check_clauses(tcs
, &entry
, tcs
->clause
, tcs
->clause_count
))
984 /* Add to the seek list if not already in uniq buffer. */
985 if (!add_uniqbuf(tcs
, entry
.tag_seek
[tcs
->type
]))
989 tcs
->seek_list
[tcs
->seek_list_count
] = entry
.tag_seek
[tcs
->type
];
990 tcs
->seek_flags
[tcs
->seek_list_count
] = entry
.flag
;
991 tcs
->seek_list_count
++;
996 return tcs
->seek_list_count
> 0;
1000 static void remove_files(void)
1005 tc_stat
.ready
= false;
1006 tc_stat
.ramcache
= false;
1007 tc_stat
.econ
= false;
1008 remove(TAGCACHE_FILE_MASTER
);
1009 for (i
= 0; i
< TAG_COUNT
; i
++)
1011 if (tagcache_is_numeric_tag(i
))
1014 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, i
);
1020 static int open_master_fd(struct master_header
*hdr
, bool write
)
1025 fd
= open(TAGCACHE_FILE_MASTER
, write
? O_RDWR
: O_RDONLY
);
1028 logf("master file open failed for R/W");
1029 tc_stat
.ready
= false;
1033 tc_stat
.econ
= false;
1035 /* Check the header. */
1036 rc
= read(fd
, hdr
, sizeof(struct master_header
));
1037 if (hdr
->tch
.magic
== TAGCACHE_MAGIC
&& rc
== sizeof(struct master_header
))
1043 /* Trying to read again, this time with endianess correction enabled. */
1044 lseek(fd
, 0, SEEK_SET
);
1046 rc
= ecread(fd
, hdr
, 1, master_header_ec
, true);
1047 if (hdr
->tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= sizeof(struct master_header
))
1049 logf("header error");
1050 tc_stat
.ready
= false;
1055 tc_stat
.econ
= true;
1060 static bool check_all_headers(void)
1062 struct master_header myhdr
;
1063 struct tagcache_header tch
;
1067 if ( (fd
= open_master_fd(&myhdr
, false)) < 0)
1073 logf("tagcache is dirty!");
1077 memcpy(¤t_tcmh
, &myhdr
, sizeof(struct master_header
));
1079 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
1081 if (tagcache_is_numeric_tag(tag
))
1084 if ( (fd
= open_tag_fd(&tch
, tag
, false)) < 0)
1093 bool tagcache_search(struct tagcache_search
*tcs
, int tag
)
1095 struct tagcache_header tag_hdr
;
1096 struct master_header master_hdr
;
1099 if (tcs
->initialized
)
1100 tagcache_search_finish(tcs
);
1105 memset(tcs
, 0, sizeof(struct tagcache_search
));
1106 if (tc_stat
.commit_step
> 0 || !tc_stat
.ready
)
1109 tcs
->position
= sizeof(struct tagcache_header
);
1112 tcs
->seek_list_count
= 0;
1113 tcs
->filter_count
= 0;
1116 for (i
= 0; i
< TAG_COUNT
; i
++)
1119 #ifndef HAVE_TC_RAMCACHE
1120 tcs
->ramsearch
= false;
1122 tcs
->ramsearch
= tc_stat
.ramcache
;
1125 tcs
->entry_count
= hdr
->entry_count
[tcs
->type
];
1130 if (!tagcache_is_numeric_tag(tcs
->type
))
1132 tcs
->idxfd
[tcs
->type
] = open_tag_fd(&tag_hdr
, tcs
->type
, false);
1133 if (tcs
->idxfd
[tcs
->type
] < 0)
1137 /* Always open as R/W so we can pass tcs to functions that modify data also
1138 * without failing. */
1139 tcs
->masterfd
= open_master_fd(&master_hdr
, true);
1141 if (tcs
->masterfd
< 0)
1146 tcs
->initialized
= true;
1152 void tagcache_search_set_uniqbuf(struct tagcache_search
*tcs
,
1153 void *buffer
, long length
)
1155 tcs
->unique_list
= (unsigned long *)buffer
;
1156 tcs
->unique_list_capacity
= length
/ sizeof(*tcs
->unique_list
);
1157 tcs
->unique_list_count
= 0;
1160 bool tagcache_search_add_filter(struct tagcache_search
*tcs
,
1163 if (tcs
->filter_count
== TAGCACHE_MAX_FILTERS
)
1166 if (!tagcache_is_unique_tag(tag
) || tagcache_is_numeric_tag(tag
))
1169 tcs
->filter_tag
[tcs
->filter_count
] = tag
;
1170 tcs
->filter_seek
[tcs
->filter_count
] = seek
;
1171 tcs
->filter_count
++;
1176 bool tagcache_search_add_clause(struct tagcache_search
*tcs
,
1177 struct tagcache_search_clause
*clause
)
1181 if (tcs
->clause_count
>= TAGCACHE_MAX_CLAUSES
)
1183 logf("Too many clauses");
1187 /* Check if there is already a similar filter in present (filters are
1188 * much faster than clauses).
1190 for (i
= 0; i
< tcs
->filter_count
; i
++)
1192 if (tcs
->filter_tag
[i
] == clause
->tag
)
1196 if (!tagcache_is_numeric_tag(clause
->tag
) && tcs
->idxfd
[clause
->tag
] < 0)
1200 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, clause
->tag
);
1201 tcs
->idxfd
[clause
->tag
] = open(buf
, O_RDONLY
);
1204 tcs
->clause
[tcs
->clause_count
] = clause
;
1205 tcs
->clause_count
++;
1210 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1211 ? (flag & FLAG_DIRCACHE) : 1)
1213 static bool get_next(struct tagcache_search
*tcs
)
1215 static char buf
[TAG_MAXLEN
+32];
1216 struct tagfile_entry entry
;
1219 if (!tcs
->valid
|| !tc_stat
.ready
)
1222 if (tcs
->idxfd
[tcs
->type
] < 0 && !tagcache_is_numeric_tag(tcs
->type
)
1223 #ifdef HAVE_TC_RAMCACHE
1229 /* Relative fetch. */
1230 if (tcs
->filter_count
> 0 || tcs
->clause_count
> 0
1231 || tagcache_is_numeric_tag(tcs
->type
))
1233 /* Check for end of list. */
1234 if (tcs
->seek_list_count
== 0)
1236 /* Try to fetch more. */
1237 if (!build_lookup_list(tcs
))
1244 tcs
->seek_list_count
--;
1245 flag
= tcs
->seek_flags
[tcs
->seek_list_count
];
1247 /* Seek stream to the correct position and continue to direct fetch. */
1248 if ((!tcs
->ramsearch
|| !TAG_FILENAME_RAM(tcs
))
1249 && !tagcache_is_numeric_tag(tcs
->type
))
1251 if (!open_files(tcs
, tcs
->type
))
1254 lseek(tcs
->idxfd
[tcs
->type
], tcs
->seek_list
[tcs
->seek_list_count
], SEEK_SET
);
1257 tcs
->position
= tcs
->seek_list
[tcs
->seek_list_count
];
1260 if (tagcache_is_numeric_tag(tcs
->type
))
1262 snprintf(buf
, sizeof(buf
), "%d", tcs
->position
);
1263 tcs
->result_seek
= tcs
->position
;
1265 tcs
->result_len
= strlen(buf
) + 1;
1270 #ifdef HAVE_TC_RAMCACHE
1271 if (tcs
->ramsearch
&& TAG_FILENAME_RAM(tcs
))
1273 struct tagfile_entry
*ep
;
1275 if (tcs
->entry_count
== 0)
1282 tcs
->result_seek
= tcs
->position
;
1284 # ifdef HAVE_DIRCACHE
1285 if (tcs
->type
== tag_filename
)
1287 dircache_copy_path((struct dircache_entry
*)tcs
->position
,
1290 tcs
->result_len
= strlen(buf
) + 1;
1291 tcs
->idx_id
= FLAG_GET_ATTR(flag
);
1292 tcs
->ramresult
= false;
1298 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->position
];
1299 tcs
->position
+= sizeof(struct tagfile_entry
) + ep
->tag_length
;
1300 tcs
->result
= ep
->tag_data
;
1301 tcs
->result_len
= strlen(tcs
->result
) + 1;
1302 tcs
->idx_id
= ep
->idx_id
;
1303 tcs
->ramresult
= true;
1310 if (!open_files(tcs
, tcs
->type
))
1313 tcs
->result_seek
= lseek(tcs
->idxfd
[tcs
->type
], 0, SEEK_CUR
);
1314 if (ecread(tcs
->idxfd
[tcs
->type
], &entry
, 1,
1315 tagfile_entry_ec
, tc_stat
.econ
) != sizeof(struct tagfile_entry
))
1323 if (entry
.tag_length
> (long)sizeof(buf
))
1326 logf("too long tag #2");
1330 if (read(tcs
->idxfd
[tcs
->type
], buf
, entry
.tag_length
) != entry
.tag_length
)
1333 logf("read error #4");
1338 tcs
->result_len
= strlen(tcs
->result
) + 1;
1339 tcs
->idx_id
= entry
.idx_id
;
1340 tcs
->ramresult
= false;
1345 bool tagcache_get_next(struct tagcache_search
*tcs
)
1347 while (get_next(tcs
))
1349 if (tcs
->result_len
> 1)
1356 bool tagcache_retrieve(struct tagcache_search
*tcs
, int idxid
,
1357 int tag
, char *buf
, long size
)
1359 struct index_entry idx
;
1362 if (!get_index(tcs
->masterfd
, idxid
, &idx
, true))
1365 return retrieve(tcs
, &idx
, tag
, buf
, size
);
1368 static bool update_master_header(void)
1370 struct master_header myhdr
;
1376 if ( (fd
= open_master_fd(&myhdr
, true)) < 0)
1379 myhdr
.serial
= current_tcmh
.serial
;
1380 myhdr
.commitid
= current_tcmh
.commitid
;
1383 lseek(fd
, 0, SEEK_SET
);
1384 ecwrite(fd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
1387 #ifdef HAVE_TC_RAMCACHE
1390 hdr
->h
.serial
= current_tcmh
.serial
;
1391 hdr
->h
.commitid
= current_tcmh
.commitid
;
1400 void tagcache_modify(struct tagcache_search
*tcs
, int type
, const char *text
)
1402 struct tagentry
*entry
;
1404 if (tcs
->type
!= tag_title
)
1407 /* We will need reserve buffer for this. */
1410 struct tagfile_entry
*ep
;
1412 ep
= (struct tagfile_entry
*)&hdr
->tags
[tcs
->type
][tcs
->result_seek
];
1413 tcs
->seek_list
[tcs
->seek_list_count
];
1416 entry
= find_entry_ram();
1421 void tagcache_search_finish(struct tagcache_search
*tcs
)
1425 if (!tcs
->initialized
)
1428 if (tcs
->masterfd
>= 0)
1430 close(tcs
->masterfd
);
1434 for (i
= 0; i
< TAG_COUNT
; i
++)
1436 if (tcs
->idxfd
[i
] >= 0)
1438 close(tcs
->idxfd
[i
]);
1443 tcs
->ramsearch
= false;
1445 tcs
->initialized
= 0;
1450 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1451 static struct tagfile_entry
*get_tag(const struct index_entry
*entry
, int tag
)
1453 return (struct tagfile_entry
*)&hdr
->tags
[tag
][entry
->tag_seek
[tag
]];
1456 static long get_tag_numeric(const struct index_entry
*entry
, int tag
)
1458 return entry
->tag_seek
[tag
];
1461 static char* get_tag_string(const struct index_entry
*entry
, int tag
)
1463 char* s
= get_tag(entry
, tag
)->tag_data
;
1464 return strcmp(s
, UNTAGGED
) ? s
: NULL
;
1467 bool tagcache_fill_tags(struct mp3entry
*id3
, const char *filename
)
1469 struct index_entry
*entry
;
1475 /* Find the corresponding entry in tagcache. */
1476 idx_id
= find_entry_ram(filename
, NULL
);
1477 if (idx_id
< 0 || !tc_stat
.ramcache
)
1480 entry
= &hdr
->indices
[idx_id
];
1482 id3
->title
= get_tag_string(entry
, tag_title
);
1483 id3
->artist
= get_tag_string(entry
, tag_artist
);
1484 id3
->album
= get_tag_string(entry
, tag_album
);
1485 id3
->genre_string
= get_tag_string(entry
, tag_genre
);
1486 id3
->composer
= get_tag_string(entry
, tag_composer
);
1487 id3
->comment
= get_tag_string(entry
, tag_comment
);
1488 id3
->albumartist
= get_tag_string(entry
, tag_albumartist
);
1490 id3
->playcount
= get_tag_numeric(entry
, tag_playcount
);
1491 id3
->lastplayed
= get_tag_numeric(entry
, tag_lastplayed
);
1492 id3
->rating
= get_tag_numeric(entry
, tag_virt_autoscore
) / 10;
1493 id3
->year
= get_tag_numeric(entry
, tag_year
);
1495 id3
->tracknum
= get_tag_numeric(entry
, tag_tracknumber
);
1496 id3
->bitrate
= get_tag_numeric(entry
, tag_bitrate
);
1497 if (id3
->bitrate
== 0)
1504 static inline void write_item(const char *item
)
1506 int len
= strlen(item
) + 1;
1509 write(cachefd
, item
, len
);
1512 static int check_if_empty(char **tag
)
1516 if (*tag
== NULL
|| *tag
[0] == '\0')
1519 return sizeof(UNTAGGED
); /* Tag length */
1522 length
= strlen(*tag
);
1523 if (length
> TAG_MAXLEN
)
1525 logf("over length tag: %s", *tag
);
1526 length
= TAG_MAXLEN
;
1527 *tag
[length
] = '\0';
1533 #define ADD_TAG(entry,tag,data) \
1535 entry.tag_offset[tag] = offset; \
1536 entry.tag_length[tag] = check_if_empty(data); \
1537 offset += entry.tag_length[tag]
1539 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1540 static void add_tagcache(char *path
, const struct dircache_entry
*dc
)
1542 static void add_tagcache(char *path
)
1545 struct track_info track
;
1546 struct temp_file_entry entry
;
1549 char tracknumfix
[3];
1551 int path_length
= strlen(path
);
1556 /* Check for overlength file path. */
1557 if (path_length
> TAG_MAXLEN
)
1559 /* Path can't be shortened. */
1560 logf("Too long path: %s");
1564 /* Check if the file is supported. */
1565 if (probe_file_format(path
) == AFMT_UNKNOWN
)
1568 /* Check if the file is already cached. */
1569 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1570 if (tc_stat
.ramcache
&& dircache_is_enabled())
1572 if (find_entry_ram(path
, dc
) >= 0)
1578 if (filenametag_fd
>= 0)
1580 if (find_entry_disk(path
) >= 0)
1585 fd
= open(path
, O_RDONLY
);
1588 logf("open fail: %s", path
);
1592 memset(&track
, 0, sizeof(struct track_info
));
1593 memset(&entry
, 0, sizeof(struct temp_file_entry
));
1594 memset(&tracknumfix
, 0, sizeof(tracknumfix
));
1595 ret
= get_metadata(&track
, fd
, path
, false);
1601 logf("-> %s", path
);
1603 /* Generate track number if missing. */
1604 if (track
.id3
.tracknum
<= 0)
1606 const char *p
= strrchr(path
, '.');
1609 p
= &path
[strlen(path
)-1];
1613 if (isdigit(*p
) && isdigit(*(p
-1)))
1615 tracknumfix
[1] = *p
--;
1616 tracknumfix
[0] = *p
;
1622 if (tracknumfix
[0] != '\0')
1624 track
.id3
.tracknum
= atoi(tracknumfix
);
1625 /* Set a flag to indicate track number has been generated. */
1626 entry
.flag
|= FLAG_TRKNUMGEN
;
1630 /* Unable to generate track number. */
1631 track
.id3
.tracknum
= -1;
1636 entry
.tag_offset
[tag_year
] = track
.id3
.year
;
1637 entry
.tag_offset
[tag_tracknumber
] = track
.id3
.tracknum
;
1638 entry
.tag_offset
[tag_length
] = track
.id3
.length
;
1639 entry
.tag_offset
[tag_bitrate
] = track
.id3
.bitrate
;
1642 ADD_TAG(entry
, tag_filename
, &path
);
1643 ADD_TAG(entry
, tag_title
, &track
.id3
.title
);
1644 ADD_TAG(entry
, tag_artist
, &track
.id3
.artist
);
1645 ADD_TAG(entry
, tag_album
, &track
.id3
.album
);
1646 ADD_TAG(entry
, tag_genre
, &track
.id3
.genre_string
);
1647 ADD_TAG(entry
, tag_composer
, &track
.id3
.composer
);
1648 ADD_TAG(entry
, tag_comment
, &track
.id3
.comment
);
1649 ADD_TAG(entry
, tag_albumartist
, &track
.id3
.albumartist
);
1650 entry
.data_length
= offset
;
1652 /* Write the header */
1653 write(cachefd
, &entry
, sizeof(struct temp_file_entry
));
1655 /* And tags also... Correct order is critical */
1657 write_item(track
.id3
.title
);
1658 write_item(track
.id3
.artist
);
1659 write_item(track
.id3
.album
);
1660 write_item(track
.id3
.genre_string
);
1661 write_item(track
.id3
.composer
);
1662 write_item(track
.id3
.comment
);
1663 write_item(track
.id3
.albumartist
);
1664 total_entry_count
++;
1667 static bool tempbuf_insert(char *str
, int id
, int idx_id
, bool unique
)
1669 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1670 int len
= strlen(str
)+1;
1673 unsigned *crcbuf
= (unsigned *)&tempbuf
[tempbuf_size
-4];
1674 char buf
[TAG_MAXLEN
+32];
1676 for (i
= 0; str
[i
] != '\0' && i
< (int)sizeof(buf
)-1; i
++)
1677 buf
[i
] = tolower(str
[i
]);
1680 crc32
= crc_32(buf
, i
, 0xffffffff);
1684 /* Check if the crc does not exist -> entry does not exist for sure. */
1685 for (i
= 0; i
< tempbufidx
; i
++)
1687 if (crcbuf
[-i
] != crc32
)
1690 if (!strcasecmp(str
, index
[i
].str
))
1692 if (id
< 0 || id
>= lookup_buffer_depth
)
1694 logf("lookup buf overf.: %d", id
);
1698 lookup
[id
] = &index
[i
];
1704 /* Insert to CRC buffer. */
1705 crcbuf
[-tempbufidx
] = crc32
;
1708 /* Insert it to the buffer. */
1709 tempbuf_left
-= len
;
1710 if (tempbuf_left
- 4 < 0 || tempbufidx
>= commit_entry_count
-1)
1713 if (id
>= lookup_buffer_depth
)
1715 logf("lookup buf overf. #2: %d", id
);
1721 lookup
[id
] = &index
[tempbufidx
];
1722 index
[tempbufidx
].idlist
.id
= id
;
1725 index
[tempbufidx
].idlist
.id
= -1;
1727 index
[tempbufidx
].idlist
.next
= NULL
;
1728 index
[tempbufidx
].idx_id
= idx_id
;
1729 index
[tempbufidx
].seek
= -1;
1730 index
[tempbufidx
].str
= &tempbuf
[tempbuf_pos
];
1731 memcpy(index
[tempbufidx
].str
, str
, len
);
1738 static int compare(const void *p1
, const void *p2
)
1740 struct tempbuf_searchidx
*e1
= (struct tempbuf_searchidx
*)p1
;
1741 struct tempbuf_searchidx
*e2
= (struct tempbuf_searchidx
*)p2
;
1744 if (!strncasecmp("the ", e1, 4))
1746 if (!strncasecmp("the ", e2, 4))
1750 return strncasecmp(e1
->str
, e2
->str
, TAG_MAXLEN
);
1753 static int tempbuf_sort(int fd
)
1755 struct tempbuf_searchidx
*index
= (struct tempbuf_searchidx
*)tempbuf
;
1756 struct tagfile_entry fe
;
1760 /* Generate reverse lookup entries. */
1761 for (i
= 0; i
< lookup_buffer_depth
; i
++)
1763 struct tempbuf_id_list
*idlist
;
1768 if (lookup
[i
]->idlist
.id
== i
)
1771 idlist
= &lookup
[i
]->idlist
;
1772 while (idlist
->next
!= NULL
)
1773 idlist
= idlist
->next
;
1775 tempbuf_left
-= sizeof(struct tempbuf_id_list
);
1776 if (tempbuf_left
- 4 < 0)
1779 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1780 if (tempbuf_pos
& 0x03)
1782 tempbuf_pos
= (tempbuf_pos
& ~0x03) + 0x04;
1784 idlist
->next
= (struct tempbuf_id_list
*)&tempbuf
[tempbuf_pos
];
1786 tempbuf_pos
+= sizeof(struct tempbuf_id_list
);
1788 idlist
= idlist
->next
;
1790 idlist
->next
= NULL
;
1793 qsort(index
, tempbufidx
, sizeof(struct tempbuf_searchidx
), compare
);
1794 memset(lookup
, 0, lookup_buffer_depth
* sizeof(struct tempbuf_searchidx
**));
1796 for (i
= 0; i
< tempbufidx
; i
++)
1798 struct tempbuf_id_list
*idlist
= &index
[i
].idlist
;
1800 /* Fix the lookup list. */
1801 while (idlist
!= NULL
)
1803 if (idlist
->id
>= 0)
1804 lookup
[idlist
->id
] = &index
[i
];
1805 idlist
= idlist
->next
;
1808 index
[i
].seek
= lseek(fd
, 0, SEEK_CUR
);
1809 length
= strlen(index
[i
].str
) + 1;
1810 fe
.tag_length
= length
;
1811 fe
.idx_id
= index
[i
].idx_id
;
1813 /* Check the chunk alignment. */
1814 if ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
1815 % TAGFILE_ENTRY_CHUNK_LENGTH
)
1817 fe
.tag_length
+= TAGFILE_ENTRY_CHUNK_LENGTH
-
1818 ((fe
.tag_length
+ sizeof(struct tagfile_entry
))
1819 % TAGFILE_ENTRY_CHUNK_LENGTH
);
1822 #ifdef TAGCACHE_STRICT_ALIGN
1823 /* Make sure the entry is long aligned. */
1824 if (index
[i
].seek
& 0x03)
1826 logf("tempbuf_sort: alignment error!");
1831 if (ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
) !=
1832 sizeof(struct tagfile_entry
))
1834 logf("tempbuf_sort: write error #1");
1838 if (write(fd
, index
[i
].str
, length
) != length
)
1840 logf("tempbuf_sort: write error #2");
1844 /* Write some padding. */
1845 if (fe
.tag_length
- length
> 0)
1846 write(fd
, "XXXXXXXX", fe
.tag_length
- length
);
1852 inline static struct tempbuf_searchidx
* tempbuf_locate(int id
)
1854 if (id
< 0 || id
>= lookup_buffer_depth
)
1861 inline static int tempbuf_find_location(int id
)
1863 struct tempbuf_searchidx
*entry
;
1865 entry
= tempbuf_locate(id
);
1872 static bool build_numeric_indices(struct tagcache_header
*h
, int tmpfd
)
1874 struct master_header tcmh
;
1875 struct index_entry idx
;
1878 struct temp_file_entry
*entrybuf
= (struct temp_file_entry
*)tempbuf
;
1880 int entries_processed
= 0;
1883 max_entries
= tempbuf_size
/ sizeof(struct temp_file_entry
) - 1;
1885 logf("Building numeric indices...");
1886 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
1888 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
1891 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
1893 if (masterfd_pos
== filesize(masterfd
))
1895 logf("we can't append!");
1900 while (entries_processed
< h
->entry_count
)
1902 int count
= MIN(h
->entry_count
- entries_processed
, max_entries
);
1904 /* Read in as many entries as possible. */
1905 for (i
= 0; i
< count
; i
++)
1907 /* Read in numeric data. */
1908 if (read(tmpfd
, &entrybuf
[i
], sizeof(struct temp_file_entry
)) !=
1909 sizeof(struct temp_file_entry
))
1911 logf("read fail #1");
1916 /* Skip string data. */
1917 lseek(tmpfd
, entrybuf
[i
].data_length
, SEEK_CUR
);
1920 /* Commit the data to the index. */
1921 for (i
= 0; i
< count
; i
++)
1923 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
1925 if (ecread(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
1926 != sizeof(struct index_entry
))
1928 logf("read fail #2");
1933 for (j
= 0; j
< TAG_COUNT
; j
++)
1935 if (!tagcache_is_numeric_tag(j
))
1938 idx
.tag_seek
[j
] = entrybuf
[i
].tag_offset
[j
];
1940 idx
.flag
= entrybuf
[i
].flag
;
1942 if (tc_stat
.ready
&& current_tcmh
.commitid
> 0)
1944 idx
.tag_seek
[tag_commitid
] = current_tcmh
.commitid
;
1945 idx
.flag
|= FLAG_DIRTYNUM
;
1948 /* Write back the updated index. */
1949 lseek(masterfd
, loc
, SEEK_SET
);
1950 if (ecwrite(masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
1951 != sizeof(struct index_entry
))
1959 entries_processed
+= count
;
1960 logf("%d/%d entries processed", entries_processed
, h
->entry_count
);
1971 * == 0 temporary failure
1974 static int build_index(int index_type
, struct tagcache_header
*h
, int tmpfd
)
1977 struct tagcache_header tch
;
1978 struct master_header tcmh
;
1979 struct index_entry idxbuf
[IDX_BUF_DEPTH
];
1981 char buf
[TAG_MAXLEN
+32];
1982 int fd
= -1, masterfd
;
1987 logf("Building index: %d", index_type
);
1989 /* Check the number of entries we need to allocate ram for. */
1990 commit_entry_count
= h
->entry_count
+ 1;
1992 masterfd
= open_master_fd(&tcmh
, false);
1995 commit_entry_count
+= tcmh
.tch
.entry_count
;
1999 remove_files(); /* Just to be sure we are clean. */
2001 /* Open the index file, which contains the tag names. */
2002 fd
= open_tag_fd(&tch
, index_type
, true);
2005 logf("tch.datasize=%d", tch
.datasize
);
2006 lookup_buffer_depth
= 1 +
2007 /* First part */ commit_entry_count
+
2008 /* Second part */ (tch
.datasize
/ TAGFILE_ENTRY_CHUNK_LENGTH
);
2012 lookup_buffer_depth
= 1 +
2013 /* First part */ commit_entry_count
+
2014 /* Second part */ 0;
2017 logf("lookup_buffer_depth=%d", lookup_buffer_depth
);
2018 logf("commit_entry_count=%d", commit_entry_count
);
2020 /* Allocate buffer for all index entries from both old and new
2023 tempbuf_pos
= commit_entry_count
* sizeof(struct tempbuf_searchidx
);
2025 /* Allocate lookup buffer. The first portion of commit_entry_count
2026 * contains the new tags in the temporary file and the second
2027 * part for locating entries already in the db.
2030 * +---------+---------------------------+
2031 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2032 * +---------+---------------------------+
2034 * Old tags are inserted to a temporary buffer with position:
2035 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2036 * And new tags with index:
2037 * tempbuf_insert(idx, ...);
2039 * The buffer is sorted and written into tag file:
2040 * tempbuf_sort(...);
2041 * leaving master index locations messed up.
2043 * That is fixed using the lookup buffer for old tags:
2044 * new_seek = tempbuf_find_location(old_seek, ...);
2046 * new_seek = tempbuf_find_location(idx);
2048 lookup
= (struct tempbuf_searchidx
**)&tempbuf
[tempbuf_pos
];
2049 tempbuf_pos
+= lookup_buffer_depth
* sizeof(void **);
2050 memset(lookup
, 0, lookup_buffer_depth
* sizeof(void **));
2052 /* And calculate the remaining data space used mainly for storing
2053 * tag data (strings). */
2054 tempbuf_left
= tempbuf_size
- tempbuf_pos
- 8;
2055 if (tempbuf_left
- TAGFILE_ENTRY_AVG_LENGTH
* commit_entry_count
< 0)
2057 logf("Buffer way too small!");
2064 * If tag file contains unique tags (sorted index), we will load
2065 * it entirely into memory so we can resort it later for use with
2068 if (tagcache_is_sorted_tag(index_type
))
2070 logf("loading tags...");
2071 for (i
= 0; i
< tch
.entry_count
; i
++)
2073 struct tagfile_entry entry
;
2074 int loc
= lseek(fd
, 0, SEEK_CUR
);
2077 if (ecread(fd
, &entry
, 1, tagfile_entry_ec
, tc_stat
.econ
)
2078 != sizeof(struct tagfile_entry
))
2080 logf("read error #7");
2085 if (entry
.tag_length
>= (int)sizeof(buf
))
2087 logf("too long tag #3");
2092 if (read(fd
, buf
, entry
.tag_length
) != entry
.tag_length
)
2094 logf("read error #8");
2099 /* Skip deleted entries. */
2104 * Save the tag and tag id in the memory buffer. Tag id
2105 * is saved so we can later reindex the master lookup
2106 * table when the index gets resorted.
2108 ret
= tempbuf_insert(buf
, loc
/TAGFILE_ENTRY_CHUNK_LENGTH
2109 + commit_entry_count
, entry
.idx_id
,
2110 tagcache_is_unique_tag(index_type
));
2121 tempbufidx
= tch
.entry_count
;
2126 * Creating new index file to store the tags. No need to preload
2127 * anything whether the index type is sorted or not.
2129 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, index_type
);
2130 fd
= open(buf
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2133 logf("%s open fail", buf
);
2137 tch
.magic
= TAGCACHE_MAGIC
;
2138 tch
.entry_count
= 0;
2141 if (ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
)
2142 != sizeof(struct tagcache_header
))
2144 logf("header write failed");
2150 /* Loading the tag lookup file as "master file". */
2151 logf("Loading index file");
2152 masterfd
= open(TAGCACHE_FILE_MASTER
, O_RDWR
);
2156 logf("Creating new DB");
2157 masterfd
= open(TAGCACHE_FILE_MASTER
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2161 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER
);
2166 /* Write the header (write real values later). */
2167 memset(&tcmh
, 0, sizeof(struct master_header
));
2169 tcmh
.tch
.entry_count
= 0;
2170 tcmh
.tch
.datasize
= 0;
2172 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2174 masterfd_pos
= lseek(masterfd
, 0, SEEK_CUR
);
2179 * Master file already exists so we need to process the current
2184 if (ecread(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
) !=
2185 sizeof(struct master_header
) || tcmh
.tch
.magic
!= TAGCACHE_MAGIC
)
2187 logf("header error");
2194 * If we reach end of the master file, we need to expand it to
2195 * hold new tags. If the current index is not sorted, we can
2196 * simply append new data to end of the file.
2197 * However, if the index is sorted, we need to update all tag
2198 * pointers in the master file for the current index.
2200 masterfd_pos
= lseek(masterfd
, tcmh
.tch
.entry_count
* sizeof(struct index_entry
),
2202 if (masterfd_pos
== filesize(masterfd
))
2204 logf("appending...");
2210 * Load new unique tags in memory to be sorted later and added
2211 * to the master lookup file.
2213 if (tagcache_is_sorted_tag(index_type
))
2215 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2216 /* h is the header of the temporary file containing new tags. */
2217 logf("inserting new tags...");
2218 for (i
= 0; i
< h
->entry_count
; i
++)
2220 struct temp_file_entry entry
;
2222 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2223 sizeof(struct temp_file_entry
))
2225 logf("read fail #3");
2231 if (entry
.tag_length
[index_type
] >= (long)sizeof(buf
))
2233 logf("too long entry!");
2238 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2239 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2240 entry
.tag_length
[index_type
])
2242 logf("read fail #4");
2247 if (tagcache_is_unique_tag(index_type
))
2248 error
= !tempbuf_insert(buf
, i
, -1, true);
2250 error
= !tempbuf_insert(buf
, i
, tcmh
.tch
.entry_count
+ i
, false);
2254 logf("insert error");
2259 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2260 entry
.tag_length
[index_type
], SEEK_CUR
);
2265 /* Sort the buffer data and write it to the index file. */
2266 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
2267 i
= tempbuf_sort(fd
);
2270 logf("sorted %d tags", i
);
2273 * Now update all indexes in the master lookup file.
2275 logf("updating indices...");
2276 lseek(masterfd
, sizeof(struct master_header
), SEEK_SET
);
2277 for (i
= 0; i
< tcmh
.tch
.entry_count
; i
+= idxbuf_pos
)
2280 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2282 idxbuf_pos
= MIN(tcmh
.tch
.entry_count
- i
, IDX_BUF_DEPTH
);
2284 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2285 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2287 logf("read fail #5");
2291 lseek(masterfd
, loc
, SEEK_SET
);
2293 for (j
= 0; j
< idxbuf_pos
; j
++)
2295 if (idxbuf
[j
].flag
& FLAG_DELETED
)
2297 /* We can just ignore deleted entries. */
2298 idxbuf
[j
].tag_seek
[index_type
] = 0;
2302 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(
2303 idxbuf
[j
].tag_seek
[index_type
]/TAGFILE_ENTRY_CHUNK_LENGTH
2304 + commit_entry_count
);
2306 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2308 logf("update error: %d/%d", i
+j
, tcmh
.tch
.entry_count
);
2316 /* Write back the updated index. */
2317 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2318 index_entry_ec
, tc_stat
.econ
) !=
2319 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2330 * Walk through the temporary file containing the new tags.
2332 // build_normal_index(h, tmpfd, masterfd, idx);
2333 logf("updating new indices...");
2334 lseek(masterfd
, masterfd_pos
, SEEK_SET
);
2335 lseek(tmpfd
, sizeof(struct tagcache_header
), SEEK_SET
);
2336 lseek(fd
, 0, SEEK_END
);
2337 for (i
= 0; i
< h
->entry_count
; i
+= idxbuf_pos
)
2341 idxbuf_pos
= MIN(h
->entry_count
- i
, IDX_BUF_DEPTH
);
2344 memset(idxbuf
, 0, sizeof(struct index_entry
)*IDX_BUF_DEPTH
);
2348 int loc
= lseek(masterfd
, 0, SEEK_CUR
);
2350 if (ecread(masterfd
, idxbuf
, idxbuf_pos
, index_entry_ec
, tc_stat
.econ
)
2351 != (int)sizeof(struct index_entry
)*idxbuf_pos
)
2353 logf("read fail #6");
2357 lseek(masterfd
, loc
, SEEK_SET
);
2360 /* Read entry headers. */
2361 for (j
= 0; j
< idxbuf_pos
; j
++)
2363 if (!tagcache_is_sorted_tag(index_type
))
2365 struct temp_file_entry entry
;
2366 struct tagfile_entry fe
;
2368 if (read(tmpfd
, &entry
, sizeof(struct temp_file_entry
)) !=
2369 sizeof(struct temp_file_entry
))
2371 logf("read fail #7");
2377 if (entry
.tag_length
[index_type
] >= (int)sizeof(buf
))
2379 logf("too long entry!");
2380 logf("length=%d", entry
.tag_length
[index_type
]);
2381 logf("pos=0x%02x", lseek(tmpfd
, 0, SEEK_CUR
));
2386 lseek(tmpfd
, entry
.tag_offset
[index_type
], SEEK_CUR
);
2387 if (read(tmpfd
, buf
, entry
.tag_length
[index_type
]) !=
2388 entry
.tag_length
[index_type
])
2390 logf("read fail #8");
2391 logf("offset=0x%02x", entry
.tag_offset
[index_type
]);
2392 logf("length=0x%02x", entry
.tag_length
[index_type
]);
2397 /* Write to index file. */
2398 idxbuf
[j
].tag_seek
[index_type
] = lseek(fd
, 0, SEEK_CUR
);
2399 fe
.tag_length
= entry
.tag_length
[index_type
];
2400 fe
.idx_id
= tcmh
.tch
.entry_count
+ i
+ j
;
2401 ecwrite(fd
, &fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
2402 write(fd
, buf
, fe
.tag_length
);
2406 lseek(tmpfd
, entry
.data_length
- entry
.tag_offset
[index_type
] -
2407 entry
.tag_length
[index_type
], SEEK_CUR
);
2411 /* Locate the correct entry from the sorted array. */
2412 idxbuf
[j
].tag_seek
[index_type
] = tempbuf_find_location(i
+ j
);
2413 if (idxbuf
[j
].tag_seek
[index_type
] < 0)
2415 logf("entry not found (%d)");
2423 if (ecwrite(masterfd
, idxbuf
, idxbuf_pos
,
2424 index_entry_ec
, tc_stat
.econ
) !=
2425 (int)sizeof(struct index_entry
)*idxbuf_pos
)
2427 logf("tagcache: write fail #4");
2436 /* Finally write the header. */
2437 tch
.magic
= TAGCACHE_MAGIC
;
2438 tch
.entry_count
= tempbufidx
;
2439 tch
.datasize
= lseek(fd
, 0, SEEK_END
) - sizeof(struct tagcache_header
);
2440 lseek(fd
, 0, SEEK_SET
);
2441 ecwrite(fd
, &tch
, 1, tagcache_header_ec
, tc_stat
.econ
);
2443 if (index_type
!= tag_filename
)
2444 h
->datasize
+= tch
.datasize
;
2445 logf("s:%d/%d/%d", index_type
, tch
.datasize
, h
->datasize
);
2457 static bool commit(void)
2459 struct tagcache_header tch
;
2460 struct master_header tcmh
;
2464 #ifdef HAVE_DIRCACHE
2465 bool dircache_buffer_stolen
= false;
2467 bool local_allocation
= false;
2469 logf("committing tagcache");
2474 tmpfd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
2477 logf("nothing to commit");
2482 /* Load the header. */
2483 len
= sizeof(struct tagcache_header
);
2484 rc
= read(tmpfd
, &tch
, len
);
2486 if (tch
.magic
!= TAGCACHE_MAGIC
|| rc
!= len
)
2488 logf("incorrect header");
2490 remove(TAGCACHE_FILE_TEMP
);
2494 if (tch
.entry_count
== 0)
2496 logf("nothing to commit");
2498 remove(TAGCACHE_FILE_TEMP
);
2502 #ifdef HAVE_EEPROM_SETTINGS
2503 remove(TAGCACHE_STATEFILE
);
2506 /* At first be sure to unload the ramcache! */
2507 #ifdef HAVE_TC_RAMCACHE
2508 tc_stat
.ramcache
= false;
2513 /* Try to steal every buffer we can :) */
2514 if (tempbuf_size
== 0)
2515 local_allocation
= true;
2517 #ifdef HAVE_DIRCACHE
2518 if (tempbuf_size
== 0)
2520 /* Try to steal the dircache buffer. */
2521 tempbuf
= dircache_steal_buffer(&tempbuf_size
);
2522 tempbuf_size
&= ~0x03;
2524 if (tempbuf_size
> 0)
2526 dircache_buffer_stolen
= true;
2531 #ifdef HAVE_TC_RAMCACHE
2532 if (tempbuf_size
== 0 && tc_stat
.ramcache_allocated
> 0)
2534 tempbuf
= (char *)(hdr
+ 1);
2535 tempbuf_size
= tc_stat
.ramcache_allocated
- sizeof(struct ramcache_header
) - 128;
2536 tempbuf_size
&= ~0x03;
2540 /* And finally fail if there are no buffers available. */
2541 if (tempbuf_size
== 0)
2543 logf("delaying commit until next boot");
2544 tc_stat
.commit_delayed
= true;
2550 logf("commit %d entries...", tch
.entry_count
);
2552 /* Mark DB dirty so it will stay disabled if commit fails. */
2553 current_tcmh
.dirty
= true;
2554 update_master_header();
2556 /* Now create the index files. */
2557 tc_stat
.commit_step
= 0;
2559 tc_stat
.commit_delayed
= false;
2561 for (i
= 0; i
< TAG_COUNT
; i
++)
2565 if (tagcache_is_numeric_tag(i
))
2568 tc_stat
.commit_step
++;
2569 ret
= build_index(i
, &tch
, tmpfd
);
2573 logf("tagcache failed init");
2577 tc_stat
.commit_delayed
= true;
2578 tc_stat
.commit_step
= 0;
2584 if (!build_numeric_indices(&tch
, tmpfd
))
2586 logf("Failure to commit numeric indices");
2589 tc_stat
.commit_step
= 0;
2595 tc_stat
.commit_step
= 0;
2597 /* Update the master index headers. */
2598 if ( (masterfd
= open_master_fd(&tcmh
, true)) < 0)
2604 tcmh
.tch
.entry_count
+= tch
.entry_count
;
2605 tcmh
.tch
.datasize
= sizeof(struct master_header
)
2606 + sizeof(struct index_entry
) * tcmh
.tch
.entry_count
2611 lseek(masterfd
, 0, SEEK_SET
);
2612 ecwrite(masterfd
, &tcmh
, 1, master_header_ec
, tc_stat
.econ
);
2615 logf("tagcache committed");
2616 remove(TAGCACHE_FILE_TEMP
);
2617 tc_stat
.ready
= check_all_headers();
2619 if (local_allocation
)
2625 #ifdef HAVE_DIRCACHE
2626 /* Rebuild the dircache, if we stole the buffer. */
2627 if (dircache_buffer_stolen
)
2631 #ifdef HAVE_TC_RAMCACHE
2632 /* Reload tagcache. */
2633 if (tc_stat
.ramcache_allocated
> 0)
2634 tagcache_start_scan();
2642 static void allocate_tempbuf(void)
2644 /* Yeah, malloc would be really nice now :) */
2646 tempbuf_size
= 32*1024*1024;
2647 tempbuf
= malloc(tempbuf_size
);
2649 tempbuf
= (char *)(((long)audiobuf
& ~0x03) + 0x04);
2650 tempbuf_size
= (long)audiobufend
- (long)audiobuf
- 4;
2651 audiobuf
+= tempbuf_size
;
2655 static void free_tempbuf(void)
2657 if (tempbuf_size
== 0)
2663 audiobuf
-= tempbuf_size
;
2669 long tagcache_increase_serial(void)
2679 old
= current_tcmh
.serial
++;
2680 if (!update_master_header())
2686 long tagcache_get_serial(void)
2688 return current_tcmh
.serial
;
2691 long tagcache_get_commitid(void)
2693 return current_tcmh
.commitid
;
2696 static bool modify_numeric_entry(int masterfd
, int idx_id
, int tag
, long data
)
2698 struct index_entry idx
;
2703 if (!tagcache_is_numeric_tag(tag
))
2706 if (!get_index(masterfd
, idx_id
, &idx
, false))
2709 idx
.tag_seek
[tag
] = data
;
2710 idx
.flag
|= FLAG_DIRTYNUM
;
2712 return write_index(masterfd
, idx_id
, &idx
);
2715 bool tagcache_modify_numeric_entry(struct tagcache_search
*tcs
,
2718 struct master_header myhdr
;
2720 if (tcs
->masterfd
< 0)
2722 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, true)) < 0)
2726 return modify_numeric_entry(tcs
->masterfd
, tcs
->idx_id
, tag
, data
);
2729 static bool write_tag(int fd
, const char *tagstr
, const char *datastr
)
2734 snprintf(buf
, sizeof buf
, "%s=\"", tagstr
);
2735 for (i
= strlen(buf
); i
< (long)sizeof(buf
)-3; i
++)
2737 if (*datastr
== '\0')
2740 if (*datastr
== '"')
2747 buf
[i
] = *(datastr
++);
2750 strcpy(&buf
[i
], "\" ");
2752 write(fd
, buf
, i
+ 2);
2757 static bool read_tag(char *dest
, long size
,
2758 const char *src
, const char *tagstr
)
2761 char current_tag
[32];
2763 while (*src
!= '\0')
2765 /* Skip all whitespace */
2773 /* Read in tag name */
2774 while (*src
!= '=' && *src
!= ' ')
2776 current_tag
[pos
] = *src
;
2780 if (*src
== '\0' || pos
>= (long)sizeof(current_tag
))
2783 current_tag
[pos
] = '\0';
2785 /* Read in tag data */
2787 /* Find the start. */
2788 while (*src
!= '"' && *src
!= '\0')
2791 if (*src
== '\0' || *(++src
) == '\0')
2794 /* Read the data. */
2795 for (pos
= 0; pos
< size
; pos
++)
2800 if (*src
== '\\' && *(src
+1) == '"')
2822 if (!strcasecmp(tagstr
, current_tag
))
2829 static int parse_changelog_line(int line_n
, const char *buf
, void *parameters
)
2831 struct index_entry idx
;
2832 char tag_data
[TAG_MAXLEN
+32];
2834 long masterfd
= (long)parameters
;
2835 const int import_tags
[] = { tag_playcount
, tag_playtime
, tag_lastplayed
,
2843 logf("%d/%s", line_n
, buf
);
2844 if (!read_tag(tag_data
, sizeof tag_data
, buf
, "filename"))
2846 logf("filename missing");
2851 idx_id
= find_index(tag_data
);
2854 logf("entry not found");
2858 if (!get_index(masterfd
, idx_id
, &idx
, false))
2860 logf("failed to retrieve index entry");
2864 /* Stop if tag has already been modified. */
2865 if (idx
.flag
& FLAG_DIRTYNUM
)
2868 logf("import: %s", tag_data
);
2870 idx
.flag
|= FLAG_DIRTYNUM
;
2871 for (i
= 0; i
< (long)(sizeof(import_tags
)/sizeof(import_tags
[0])); i
++)
2875 if (!read_tag(tag_data
, sizeof tag_data
, buf
,
2876 tagcache_tag_to_str(import_tags
[i
])))
2881 data
= atoi(tag_data
);
2885 idx
.tag_seek
[import_tags
[i
]] = data
;
2887 if (import_tags
[i
] == tag_lastplayed
&& data
> current_tcmh
.serial
)
2888 current_tcmh
.serial
= data
;
2889 else if (import_tags
[i
] == tag_commitid
&& data
>= current_tcmh
.commitid
)
2890 current_tcmh
.commitid
= data
+ 1;
2893 return write_index(masterfd
, idx_id
, &idx
) ? 0 : -5;
2897 bool tagcache_import_changelog(void)
2899 struct master_header myhdr
;
2900 struct tagcache_header tch
;
2911 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_RDONLY
);
2914 logf("failure to open changelog");
2918 if ( (masterfd
= open_master_fd(&myhdr
, true)) < 0)
2926 filenametag_fd
= open_tag_fd(&tch
, tag_filename
, false);
2928 fast_readline(clfd
, buf
, sizeof buf
, (long *)masterfd
,
2929 parse_changelog_line
);
2934 if (filenametag_fd
>= 0)
2935 close(filenametag_fd
);
2939 update_master_header();
2945 bool tagcache_create_changelog(struct tagcache_search
*tcs
)
2947 struct master_header myhdr
;
2948 struct index_entry idx
;
2949 char buf
[TAG_MAXLEN
+32];
2957 if (!tagcache_search(tcs
, tag_filename
))
2960 /* Initialize the changelog */
2961 clfd
= open(TAGCACHE_FILE_CHANGELOG
, O_WRONLY
| O_CREAT
| O_TRUNC
);
2964 logf("failure to open changelog");
2968 if (tcs
->masterfd
< 0)
2970 if ( (tcs
->masterfd
= open_master_fd(&myhdr
, false)) < 0)
2975 lseek(tcs
->masterfd
, 0, SEEK_SET
);
2976 ecread(tcs
->masterfd
, &myhdr
, 1, master_header_ec
, tc_stat
.econ
);
2979 write(clfd
, "## Changelog version 1\n", 23);
2981 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
2983 if (ecread(tcs
->masterfd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
2984 != sizeof(struct index_entry
))
2986 logf("read error #9");
2987 tagcache_search_finish(tcs
);
2992 /* Skip until the entry found has been modified. */
2993 if (! (idx
.flag
& FLAG_DIRTYNUM
) )
2996 /* Skip deleted entries too. */
2997 if (idx
.flag
& FLAG_DELETED
)
3000 /* Now retrieve all tags. */
3001 for (j
= 0; j
< TAG_COUNT
; j
++)
3003 if (tagcache_is_numeric_tag(j
))
3005 snprintf(temp
, sizeof temp
, "%d", idx
.tag_seek
[j
]);
3006 write_tag(clfd
, tagcache_tag_to_str(j
), temp
);
3011 tagcache_retrieve(tcs
, i
, tcs
->type
, buf
, sizeof buf
);
3012 write_tag(clfd
, tagcache_tag_to_str(j
), buf
);
3015 write(clfd
, "\n", 1);
3021 tagcache_search_finish(tcs
);
3026 static bool delete_entry(long idx_id
)
3030 struct index_entry idx
, myidx
;
3031 struct master_header myhdr
;
3032 char buf
[TAG_MAXLEN
+32];
3033 int in_use
[TAG_COUNT
];
3035 logf("delete_entry(): %d", idx_id
);
3037 #ifdef HAVE_TC_RAMCACHE
3038 /* At first mark the entry removed from ram cache. */
3040 hdr
->indices
[idx_id
].flag
|= FLAG_DELETED
;
3043 if ( (fd
= open_master_fd(&myhdr
, true) ) < 0)
3046 lseek(fd
, idx_id
* sizeof(struct index_entry
), SEEK_CUR
);
3047 if (ecread(fd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3048 != sizeof(struct index_entry
))
3050 logf("delete_entry(): read error");
3055 myidx
.flag
|= FLAG_DELETED
;
3056 lseek(fd
, -sizeof(struct index_entry
), SEEK_CUR
);
3057 if (ecwrite(fd
, &myidx
, 1, index_entry_ec
, tc_stat
.econ
)
3058 != sizeof(struct index_entry
))
3060 logf("delete_entry(): write_error");
3065 /* Now check which tags are no longer in use (if any) */
3066 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3069 lseek(fd
, sizeof(struct master_header
), SEEK_SET
);
3070 for (i
= 0; i
< myhdr
.tch
.entry_count
; i
++)
3072 if (ecread(fd
, &idx
, 1, index_entry_ec
, tc_stat
.econ
)
3073 != sizeof(struct index_entry
))
3075 logf("delete_entry(): read error #2");
3080 if (idx
.flag
& FLAG_DELETED
)
3083 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3085 if (tagcache_is_numeric_tag(tag
))
3088 if (idx
.tag_seek
[tag
] == myidx
.tag_seek
[tag
])
3095 /* Now delete all tags no longer in use. */
3096 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3098 if (tagcache_is_numeric_tag(tag
))
3103 logf("in use: %d/%d", tag
, in_use
[tag
]);
3107 /* Open the index file, which contains the tag names. */
3108 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag
);
3109 fd
= open(buf
, O_RDWR
);
3113 logf("open failed");
3117 /* Skip the header block */
3118 lseek(fd
, myidx
.tag_seek
[tag
] + sizeof(struct tagfile_entry
), SEEK_SET
);
3120 /* Debug, print 10 first characters of the tag
3123 logf("TAG:%s", buf);
3124 lseek(fd, -10, SEEK_CUR);
3127 /* Write first data byte in tag as \0 */
3130 /* Now tag data has been removed */
3139 * Returns true if there is an event waiting in the queue
3140 * that requires the current operation to be aborted.
3142 static bool check_event_queue(void)
3146 queue_wait_w_tmo(&tagcache_queue
, &ev
, 0);
3151 case SYS_USB_CONNECTED
:
3152 /* Put the event back into the queue. */
3153 queue_post(&tagcache_queue
, ev
.id
, ev
.data
);
3161 #ifdef HAVE_TC_RAMCACHE
3162 static bool allocate_tagcache(void)
3164 struct master_header tcmh
;
3167 /* Load the header. */
3168 if ( (fd
= open_master_fd(&tcmh
, false)) < 0)
3177 * Now calculate the required cache size plus
3178 * some extra space for alignment fixes.
3180 tc_stat
.ramcache_allocated
= tcmh
.tch
.datasize
+ 128 + TAGCACHE_RESERVE
+
3181 sizeof(struct ramcache_header
) + TAG_COUNT
*sizeof(void *);
3182 hdr
= buffer_alloc(tc_stat
.ramcache_allocated
+ 128);
3183 memset(hdr
, 0, sizeof(struct ramcache_header
));
3184 memcpy(&hdr
->h
, &tcmh
, sizeof(struct master_header
));
3185 hdr
->indices
= (struct index_entry
*)(hdr
+ 1);
3186 logf("tagcache: %d bytes allocated.", tc_stat
.ramcache_allocated
);
3191 # ifdef HAVE_EEPROM_SETTINGS
3192 static bool tagcache_dumpload(void)
3194 struct statefile_header shdr
;
3199 fd
= open(TAGCACHE_STATEFILE
, O_RDONLY
);
3202 logf("no tagcache statedump");
3206 /* Check the statefile memory placement */
3207 hdr
= buffer_alloc(0);
3208 rc
= read(fd
, &shdr
, sizeof(struct statefile_header
));
3209 if (rc
!= sizeof(struct statefile_header
)
3210 /* || (long)hdr != (long)shdr.hdr */)
3212 logf("incorrect statefile");
3218 offpos
= (long)hdr
- (long)shdr
.hdr
;
3220 /* Lets allocate real memory and load it */
3221 hdr
= buffer_alloc(shdr
.tc_stat
.ramcache_allocated
);
3222 rc
= read(fd
, hdr
, shdr
.tc_stat
.ramcache_allocated
);
3225 if (rc
!= shdr
.tc_stat
.ramcache_allocated
)
3227 logf("read failure!");
3232 memcpy(&tc_stat
, &shdr
.tc_stat
, sizeof(struct tagcache_stat
));
3234 /* Now fix the pointers */
3235 hdr
->indices
= (struct index_entry
*)((long)hdr
->indices
+ offpos
);
3236 for (i
= 0; i
< TAG_COUNT
; i
++)
3237 hdr
->tags
[i
] += offpos
;
3242 static bool tagcache_dumpsave(void)
3244 struct statefile_header shdr
;
3247 if (!tc_stat
.ramcache
)
3250 fd
= open(TAGCACHE_STATEFILE
, O_WRONLY
| O_CREAT
| O_TRUNC
);
3253 logf("failed to create a statedump");
3257 /* Create the header */
3259 memcpy(&shdr
.tc_stat
, &tc_stat
, sizeof(struct tagcache_stat
));
3260 write(fd
, &shdr
, sizeof(struct statefile_header
));
3262 /* And dump the data too */
3263 write(fd
, hdr
, tc_stat
.ramcache_allocated
);
3270 static bool load_tagcache(void)
3272 struct tagcache_header
*tch
;
3273 long bytesleft
= tc_stat
.ramcache_allocated
;
3274 struct index_entry
*idx
;
3278 int yield_count
= 0;
3280 # ifdef HAVE_DIRCACHE
3281 while (dircache_is_initializing())
3285 logf("loading tagcache to ram...");
3287 fd
= open(TAGCACHE_FILE_MASTER
, O_RDONLY
);
3290 logf("tagcache open failed");
3294 if (ecread(fd
, &hdr
->h
, 1, master_header_ec
, tc_stat
.econ
)
3295 != sizeof(struct master_header
)
3296 || hdr
->h
.tch
.magic
!= TAGCACHE_MAGIC
)
3298 logf("incorrect header");
3304 /* Load the master index table. */
3305 for (i
= 0; i
< hdr
->h
.tch
.entry_count
; i
++)
3307 rc
= ecread(fd
, idx
, 1, index_entry_ec
, tc_stat
.econ
);
3308 if (rc
!= sizeof(struct index_entry
))
3310 logf("read error #10");
3315 bytesleft
-= sizeof(struct index_entry
);
3316 if (bytesleft
< 0 || ((long)idx
- (long)hdr
->indices
) >= tc_stat
.ramcache_allocated
)
3318 logf("too big tagcache.");
3328 /* Load the tags. */
3330 for (tag
= 0; tag
< TAG_COUNT
; tag
++)
3332 struct tagfile_entry
*fe
;
3333 char buf
[TAG_MAXLEN
+32];
3335 if (tagcache_is_numeric_tag(tag
))
3338 //p = ((void *)p+1);
3339 p
= (char *)((long)p
& ~0x03) + 0x04;
3342 /* Check the header. */
3343 tch
= (struct tagcache_header
*)p
;
3344 p
+= sizeof(struct tagcache_header
);
3346 if ( (fd
= open_tag_fd(tch
, tag
, false)) < 0)
3349 for (hdr
->entry_count
[tag
] = 0;
3350 hdr
->entry_count
[tag
] < tch
->entry_count
;
3351 hdr
->entry_count
[tag
]++)
3355 if (yield_count
++ == 100)
3358 /* Abort if we got a critical event in queue */
3359 if (check_event_queue())
3364 fe
= (struct tagfile_entry
*)p
;
3365 pos
= lseek(fd
, 0, SEEK_CUR
);
3366 rc
= ecread(fd
, fe
, 1, tagfile_entry_ec
, tc_stat
.econ
);
3367 if (rc
!= sizeof(struct tagfile_entry
))
3369 /* End of lookup table. */
3370 logf("read error #11");
3375 /* We have a special handling for the filename tags. */
3376 if (tag
== tag_filename
)
3378 # ifdef HAVE_DIRCACHE
3379 const struct dircache_entry
*dc
;
3382 // FIXME: This is wrong!
3383 // idx = &hdr->indices[hdr->entry_count[i]];
3384 idx
= &hdr
->indices
[fe
->idx_id
];
3386 if (fe
->tag_length
>= (long)sizeof(buf
)-1)
3390 logf("TAG:%s", buf
);
3391 logf("too long filename");
3396 rc
= read(fd
, buf
, fe
->tag_length
);
3397 if (rc
!= fe
->tag_length
)
3399 logf("read error #12");
3404 /* Check if the entry has already been removed */
3405 if (idx
->flag
& FLAG_DELETED
)
3408 /* This flag must not be used yet. */
3409 if (idx
->flag
& FLAG_DIRCACHE
)
3411 logf("internal error!");
3416 if (idx
->tag_seek
[tag
] != pos
)
3418 logf("corrupt data structures!");
3423 # ifdef HAVE_DIRCACHE
3424 if (dircache_is_enabled())
3426 dc
= dircache_get_entry_ptr(buf
);
3429 logf("Entry no longer valid.");
3431 delete_entry(fe
->idx_id
);
3435 idx
->flag
|= FLAG_DIRCACHE
;
3436 FLAG_SET_ATTR(idx
->flag
, fe
->idx_id
);
3437 idx
->tag_seek
[tag_filename
] = (long)dc
;
3443 # if 0 /* Maybe we could enable this for flash players. Too slow otherwise. */
3444 /* Check if entry has been removed. */
3445 if (global_settings
.tagcache_autoupdate
)
3449 testfd
= open(buf
, O_RDONLY
);
3452 logf("Entry no longer valid.");
3454 delete_entry(fe
->idx_id
);
3465 bytesleft
-= sizeof(struct tagfile_entry
) + fe
->tag_length
;
3468 logf("too big tagcache #2");
3469 logf("tl: %d", fe
->tag_length
);
3470 logf("bl: %d", bytesleft
);
3476 rc
= read(fd
, fe
->tag_data
, fe
->tag_length
);
3479 if (rc
!= fe
->tag_length
)
3481 logf("read error #13");
3482 logf("rc=0x%04x", rc
); // 0x431
3483 logf("len=0x%04x", fe
->tag_length
); // 0x4000
3484 logf("pos=0x%04x", lseek(fd
, 0, SEEK_CUR
)); // 0x433
3485 logf("tag=0x%02x", tag
); // 0x00
3493 tc_stat
.ramcache_used
= tc_stat
.ramcache_allocated
- bytesleft
;
3494 logf("tagcache loaded into ram!");
3498 #endif /* HAVE_TC_RAMCACHE */
3500 static bool check_deleted_files(void)
3503 char buf
[TAG_MAXLEN
+32];
3504 struct tagfile_entry tfe
;
3506 logf("reverse scan...");
3507 snprintf(buf
, sizeof buf
, TAGCACHE_FILE_INDEX
, tag_filename
);
3508 fd
= open(buf
, O_RDONLY
);
3512 logf("%s open fail", buf
);
3516 lseek(fd
, sizeof(struct tagcache_header
), SEEK_SET
);
3517 while (ecread(fd
, &tfe
, 1, tagfile_entry_ec
, tc_stat
.econ
)
3518 == sizeof(struct tagfile_entry
)
3520 && !check_event_queue()
3524 if (tfe
.tag_length
>= (long)sizeof(buf
)-1)
3526 logf("too long tag");
3531 if (read(fd
, buf
, tfe
.tag_length
) != tfe
.tag_length
)
3533 logf("read error #14");
3538 /* Check if the file has already deleted from the db. */
3542 /* Now check if the file exists. */
3543 testfd
= open(buf
, O_RDONLY
);
3546 logf("Entry no longer valid.");
3547 logf("-> %s / %d", buf
, tfe
.tag_length
);
3548 delete_entry(tfe
.idx_id
);
3560 static bool check_dir(const char *dirname
)
3564 int success
= false;
3566 dir
= opendir_cached(dirname
);
3569 logf("tagcache: opendir_cached() failed");
3573 /* Recursively scan the dir. */
3577 while (!check_event_queue())
3580 struct dircache_entry
*entry
;
3582 entry
= readdir_cached(dir
);
3590 if (!strcmp((char *)entry
->d_name
, ".") ||
3591 !strcmp((char *)entry
->d_name
, ".."))
3596 len
= strlen(curpath
);
3597 snprintf(&curpath
[len
], curpath_size
- len
, "/%s",
3600 processed_dir_count
++;
3601 if (entry
->attribute
& ATTR_DIRECTORY
)
3604 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
3605 add_tagcache(curpath
, dir
->internal_entry
);
3607 add_tagcache(curpath
);
3610 curpath
[len
] = '\0';
3613 closedir_cached(dir
);
3618 void build_tagcache(const char *path
)
3620 struct tagcache_header header
;
3625 total_entry_count
= 0;
3626 processed_dir_count
= 0;
3628 #ifdef HAVE_DIRCACHE
3629 while (dircache_is_initializing())
3633 logf("updating tagcache");
3635 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDONLY
);
3638 logf("skipping, cache already waiting for commit");
3643 cachefd
= open(TAGCACHE_FILE_TEMP
, O_RDWR
| O_CREAT
| O_TRUNC
);
3646 logf("master file open failed: %s", TAGCACHE_FILE_TEMP
);
3650 filenametag_fd
= open_tag_fd(&header
, tag_filename
, false);
3654 logf("Scanning files...");
3655 /* Scan for new files. */
3656 memset(&header
, 0, sizeof(struct tagcache_header
));
3657 write(cachefd
, &header
, sizeof(struct tagcache_header
));
3659 if (strcmp("/", path
) != 0)
3660 strcpy(curpath
, path
);
3661 ret
= check_dir(path
);
3663 /* Write the header. */
3664 header
.magic
= TAGCACHE_MAGIC
;
3665 header
.datasize
= data_size
;
3666 header
.entry_count
= total_entry_count
;
3667 lseek(cachefd
, 0, SEEK_SET
);
3668 write(cachefd
, &header
, sizeof(struct tagcache_header
));
3671 if (filenametag_fd
>= 0)
3673 close(filenametag_fd
);
3674 filenametag_fd
= -1;
3684 /* Commit changes to the database. */
3690 remove(TAGCACHE_FILE_TEMP
);
3691 logf("tagcache built!");
3697 #ifdef HAVE_TC_RAMCACHE
3700 /* Import runtime statistics if we just initialized the db. */
3701 if (hdr
->h
.serial
== 0)
3702 queue_post(&tagcache_queue
, Q_IMPORT_CHANGELOG
, 0);
3709 #ifdef HAVE_TC_RAMCACHE
3710 static void load_ramcache(void)
3717 /* At first we should load the cache (if exists). */
3718 tc_stat
.ramcache
= load_tagcache();
3720 if (!tc_stat
.ramcache
)
3722 /* If loading failed, it must indicate some problem with the db
3723 * so disable it entirely to prevent further issues. */
3724 tc_stat
.ready
= false;
3731 void tagcache_unload_ramcache(void)
3733 tc_stat
.ramcache
= false;
3734 /* Just to make sure there is no statefile present. */
3735 // remove(TAGCACHE_STATEFILE);
3740 static void tagcache_thread(void)
3743 bool check_done
= false;
3745 /* If the previous cache build/update was interrupted, commit
3746 * the changes first in foreground. */
3752 #ifdef HAVE_TC_RAMCACHE
3753 # ifdef HAVE_EEPROM_SETTINGS
3754 if (firmware_settings
.initialized
&& firmware_settings
.disk_clean
)
3755 check_done
= tagcache_dumpload();
3757 remove(TAGCACHE_STATEFILE
);
3760 /* Allocate space for the tagcache if found on disk. */
3761 if (global_settings
.tagcache_ram
&& !tc_stat
.ramcache
)
3762 allocate_tagcache();
3766 tc_stat
.initialized
= true;
3768 /* Don't delay bootup with the header check but do it on background. */
3770 tc_stat
.ready
= check_all_headers();
3774 queue_wait_w_tmo(&tagcache_queue
, &ev
, HZ
);
3778 case Q_IMPORT_CHANGELOG
:
3779 tagcache_import_changelog();
3784 build_tagcache("/");
3788 build_tagcache("/");
3789 #ifdef HAVE_TC_RAMCACHE
3792 check_deleted_files();
3798 if (check_done
|| !tc_stat
.ready
)
3801 #ifdef HAVE_TC_RAMCACHE
3802 if (!tc_stat
.ramcache
&& global_settings
.tagcache_ram
)
3805 if (tc_stat
.ramcache
&& global_settings
.tagcache_autoupdate
)
3806 build_tagcache("/");
3810 if (global_settings
.tagcache_autoupdate
)
3812 build_tagcache("/");
3813 /* Don't do auto removal without dircache (very slow). */
3814 #ifdef HAVE_DIRCACHE
3815 if (dircache_is_enabled())
3816 check_deleted_files();
3820 logf("tagcache check done");
3832 case SYS_USB_CONNECTED
:
3833 logf("USB: TagCache");
3834 usb_acknowledge(SYS_USB_CONNECTED_ACK
);
3835 usb_wait_for_disconnect(&tagcache_queue
);
3842 bool tagcache_prepare_shutdown(void)
3844 if (tagcache_get_commit_step() > 0)
3847 tagcache_stop_scan();
3848 while (read_lock
|| write_lock
)
3854 void tagcache_shutdown(void)
3856 #ifdef HAVE_EEPROM_SETTINGS
3857 if (tc_stat
.ramcache
)
3858 tagcache_dumpsave();
3862 static int get_progress(void)
3864 int total_count
= -1;
3866 #ifdef HAVE_DIRCACHE
3867 if (dircache_is_enabled())
3869 total_count
= dircache_get_entry_count();
3873 #ifdef HAVE_TC_RAMCACHE
3875 if (hdr
&& tc_stat
.ramcache
)
3876 total_count
= hdr
->h
.tch
.entry_count
;
3880 if (total_count
< 0)
3883 return processed_dir_count
* 100 / total_count
;
3886 struct tagcache_stat
* tagcache_get_stat(void)
3888 tc_stat
.progress
= get_progress();
3889 tc_stat
.processed_entries
= processed_dir_count
;
3894 void tagcache_start_scan(void)
3896 queue_post(&tagcache_queue
, Q_START_SCAN
, 0);
3899 bool tagcache_update(void)
3904 queue_post(&tagcache_queue
, Q_UPDATE
, 0);
3905 gui_syncsplash(HZ
*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH
));
3910 bool tagcache_rebuild(void)
3912 queue_post(&tagcache_queue
, Q_REBUILD
, 0);
3913 gui_syncsplash(HZ
*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH
));
3918 void tagcache_stop_scan(void)
3920 queue_post(&tagcache_queue
, Q_STOP_SCAN
, 0);
3923 #ifdef HAVE_TC_RAMCACHE
3924 bool tagcache_is_ramcache(void)
3926 return tc_stat
.ramcache
;
3930 #endif /* !__PCTOOL__ */
3933 void tagcache_init(void)
3935 memset(&tc_stat
, 0, sizeof(struct tagcache_stat
));
3936 memset(¤t_tcmh
, 0, sizeof(struct master_header
));
3937 filenametag_fd
= -1;
3938 write_lock
= read_lock
= 0;
3941 queue_init(&tagcache_queue
, true);
3942 create_thread(tagcache_thread
, tagcache_stack
,
3943 sizeof(tagcache_stack
), tagcache_thread_name
3944 IF_PRIO(, PRIORITY_BACKGROUND
));
3946 tc_stat
.initialized
= true;
3950 tc_stat
.ready
= check_all_headers();
3955 void tagcache_reverse_scan(void)
3957 logf("Checking for deleted files");
3958 check_deleted_files();
3962 bool tagcache_is_initialized(void)
3964 return tc_stat
.initialized
;
3966 bool tagcache_is_usable(void)
3968 return tc_stat
.initialized
&& tc_stat
.ready
;
3970 int tagcache_get_commit_step(void)
3972 return tc_stat
.commit_step
;