?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
PP502x: Add some important information about CPU/COP_CTL register to the header glean...
[Rockbox.git] / apps / tagcache.c
blob6d738bc5fb6cb46f4668ccbc5ed5959a950789a5
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include <stdlib.h>
58 #include <ctype.h>
59 #include "config.h"
60 #include "thread.h"
61 #include "kernel.h"
62 #include "system.h"
63 #include "logf.h"
64 #include "string.h"
65 #include "usb.h"
66 #include "metadata.h"
67 #include "id3.h"
68 #include "tagcache.h"
69 #include "buffer.h"
70 #include "crc32.h"
71 #include "misc.h"
72 #include "settings.h"
73 #include "dir.h"
74 #include "structec.h"
75 #ifndef __PCTOOL__
76 #include "atoi.h"
77 #include "splash.h"
78 #include "lang.h"
79 #include "eeprom_settings.h"
80 #endif
81
82 #ifdef __PCTOOL__
83 #define yield() do { } while(0)
84 #define sim_sleep(timeout) do { } while(0)
85 #define do_timed_yield() do { } while(0)
86 #endif
87
88
89 #ifndef __PCTOOL__
90 /* Tag Cache thread. */
91 static struct event_queue tagcache_queue;
92 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
93 static const char tagcache_thread_name[] = "tagcache";
94 #endif
95
96 #define UNTAGGED "<Untagged>"
97
98 /* Previous path when scanning directory tree recursively. */
99 static char curpath[TAG_MAXLEN+32];
100 static long curpath_size = sizeof(curpath);
101
102 /* Used when removing duplicates. */
103 static char *tempbuf; /* Allocated when needed. */
104 static long tempbufidx; /* Current location in buffer. */
105 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
106 static long tempbuf_left; /* Buffer space left. */
107 static long tempbuf_pos;
108
109 /* Tags we want to get sorted (loaded to the tempbuf). */
110 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
111 tag_composer, tag_comment, tag_albumartist, tag_title };
112
113 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
114 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
115 tag_composer, tag_comment, tag_albumartist };
116
117 /* Numeric tags (we can use these tags with conditional clauses). */
118 static const int numeric_tags[] = { tag_year, tag_tracknumber, tag_length,
119 tag_bitrate, tag_playcount, tag_rating, tag_playtime, tag_lastplayed, tag_commitid,
120 tag_virt_length_min, tag_virt_length_sec,
121 tag_virt_playtime_min, tag_virt_playtime_sec,
122 tag_virt_entryage, tag_virt_autoscore };
123
124 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
125 static const char *tags_str[] = { "artist", "album", "genre", "title",
126 "filename", "composer", "comment", "albumartist", "year", "tracknumber",
127 "bitrate", "length", "playcount", "rating", "playtime", "lastplayed", "commitid" };
128
129 /* Status information of the tagcache. */
130 static struct tagcache_stat tc_stat;
131
132 /* Queue commands. */
133 enum tagcache_queue {
134 Q_STOP_SCAN = 0,
135 Q_START_SCAN,
136 Q_IMPORT_CHANGELOG,
137 Q_UPDATE,
138 Q_REBUILD,
139
140 /* Internal tagcache command queue. */
141 CMD_UPDATE_MASTER_HEADER,
142 CMD_UPDATE_NUMERIC,
143 };
144
145 struct tagcache_command_entry {
146 long command;
147 long idx_id;
148 long tag;
149 long data;
150 };
151
152 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
153 static volatile int command_queue_widx = 0;
154 static volatile int command_queue_ridx = 0;
155 static struct mutex command_queue_mutex;
156 /* Timestamp of the last added event, so we can wait a bit before committing the
157 * whole queue at once. */
158 static long command_queue_timestamp = 0;
159
160 /* Tag database structures. */
161
162 /* Variable-length tag entry in tag files. */
163 struct tagfile_entry {
164 short tag_length; /* Length of the data in bytes including '\0' */
165 short idx_id; /* Corresponding entry location in index file of not unique tags */
166 char tag_data[0]; /* Begin of the tag data */
167 };
168
169 /* Fixed-size tag entry in master db index. */
170 struct index_entry {
171 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
172 long flag; /* Status flags */
173 };
174
175 /* Header is the same in every file. */
176 struct tagcache_header {
177 long magic; /* Header version number */
178 long datasize; /* Data size in bytes */
179 long entry_count; /* Number of entries in this file */
180 };
181
182 struct master_header {
183 struct tagcache_header tch;
184 long serial; /* Increasing counting number */
185 long commitid; /* Number of commits so far */
186 long dirty;
187 };
188
189 /* For the endianess correction */
190 static const char *tagfile_entry_ec = "ss";
191 static const char *index_entry_ec = "llllllllllllllllll"; /* (1 + TAG_COUNT) * l */
192 static const char *tagcache_header_ec = "lll";
193 static const char *master_header_ec = "llllll";
194
195 static struct master_header current_tcmh;
196
197 #ifdef HAVE_TC_RAMCACHE
198 /* Header is created when loading database to ram. */
199 struct ramcache_header {
200 struct master_header h; /* Header from the master index */
201 struct index_entry *indices; /* Master index file content */
202 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
203 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
204 };
205
206 # ifdef HAVE_EEPROM_SETTINGS
207 struct statefile_header {
208 struct ramcache_header *hdr;
209 struct tagcache_stat tc_stat;
210 };
211 # endif
212
213 /* Pointer to allocated ramcache_header */
214 static struct ramcache_header *hdr;
215 #endif
216
217 /**
218 * Full tag entries stored in a temporary file waiting
219 * for commit to the cache. */
220 struct temp_file_entry {
221 long tag_offset[TAG_COUNT];
222 short tag_length[TAG_COUNT];
223 long flag;
224
225 long data_length;
226 };
227
228 struct tempbuf_id_list {
229 long id;
230 struct tempbuf_id_list *next;
231 };
232
233 struct tempbuf_searchidx {
234 long idx_id;
235 char *str;
236 int seek;
237 struct tempbuf_id_list idlist;
238 };
239
240 /* Lookup buffer for fixing messed up index while after sorting. */
241 static long commit_entry_count;
242 static long lookup_buffer_depth;
243 static struct tempbuf_searchidx **lookup;
244
245 /* Used when building the temporary file. */
246 static int cachefd = -1, filenametag_fd;
247 static int total_entry_count = 0;
248 static int data_size = 0;
249 static int processed_dir_count;
250
251 /* Thread safe locking */
252 static volatile int write_lock;
253 static volatile int read_lock;
254
255 const char* tagcache_tag_to_str(int tag)
256 {
257 return tags_str[tag];
258 }
259
260 bool tagcache_is_numeric_tag(int type)
261 {
262 int i;
263
264 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
265 {
266 if (type == numeric_tags[i])
267 return true;
268 }
269
270 return false;
271 }
272
273 bool tagcache_is_unique_tag(int type)
274 {
275 int i;
276
277 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
278 {
279 if (type == unique_tags[i])
280 return true;
281 }
282
283 return false;
284 }
285
286 bool tagcache_is_sorted_tag(int type)
287 {
288 int i;
289
290 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
291 {
292 if (type == sorted_tags[i])
293 return true;
294 }
295
296 return false;
297 }
298
299 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
300 {
301 int fd;
302 char buf[MAX_PATH];
303 int rc;
304
305 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
306 return -1;
307
308 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
309
310 fd = open(buf, write ? O_RDWR : O_RDONLY);
311 if (fd < 0)
312 {
313 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
314 tc_stat.ready = false;
315 return fd;
316 }
317
318 /* Check the header. */
319 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
320 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
321 {
322 logf("header error");
323 tc_stat.ready = false;
324 close(fd);
325 return -2;
326 }
327
328 return fd;
329 }
330
331 #ifndef __PCTOOL__
332 static bool do_timed_yield(void)
333 {
334 /* Sorting can lock up for quite a while, so yield occasionally */
335 static long wakeup_tick = 0;
336 if (current_tick >= wakeup_tick)
337 {
338 wakeup_tick = current_tick + (HZ/4);
339 yield();
340 return true;
341 }
342 return false;
343 }
344 #endif
345
346 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
347 static long find_entry_ram(const char *filename,
348 const struct dirent *dc)
349 {
350 static long last_pos = 0;
351 int i;
352
353 /* Check if we tagcache is loaded into ram. */
354 if (!tc_stat.ramcache)
355 return -1;
356
357 if (dc == NULL)
358 dc = dircache_get_entry_ptr(filename);
359
360 if (dc == NULL)
361 {
362 logf("tagcache: file not found.");
363 return -1;
364 }
365
366 try_again:
367
368 if (last_pos > 0)
369 i = last_pos;
370 else
371 i = 0;
372
373 for (; i < hdr->h.tch.entry_count; i++)
374 {
375 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
376 {
377 last_pos = MAX(0, i - 3);
378 return i;
379 }
380
381 do_timed_yield();
382 }
383
384 if (last_pos > 0)
385 {
386 last_pos = 0;
387 goto try_again;
388 }
389
390 return -1;
391 }
392 #endif
393
394 static long find_entry_disk(const char *filename)
395 {
396 struct tagcache_header tch;
397 static long last_pos = -1;
398 long pos_history[POS_HISTORY_COUNT];
399 long pos_history_idx = 0;
400 bool found = false;
401 struct tagfile_entry tfe;
402 int fd;
403 char buf[TAG_MAXLEN+32];
404 int i;
405 int pos = -1;
406
407 if (!tc_stat.ready)
408 return -2;
409
410 fd = filenametag_fd;
411 if (fd < 0)
412 {
413 last_pos = -1;
414 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
415 return -1;
416 }
417
418 check_again:
419
420 if (last_pos > 0)
421 lseek(fd, last_pos, SEEK_SET);
422 else
423 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
424
425 while (true)
426 {
427 pos = lseek(fd, 0, SEEK_CUR);
428 for (i = pos_history_idx-1; i >= 0; i--)
429 pos_history[i+1] = pos_history[i];
430 pos_history[0] = pos;
431
432 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
433 != sizeof(struct tagfile_entry))
434 {
435 break ;
436 }
437
438 if (tfe.tag_length >= (long)sizeof(buf))
439 {
440 logf("too long tag #1");
441 close(fd);
442 last_pos = -1;
443 return -2;
444 }
445
446 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
447 {
448 logf("read error #2");
449 close(fd);
450 last_pos = -1;
451 return -3;
452 }
453
454 if (!strcasecmp(filename, buf))
455 {
456 last_pos = pos_history[pos_history_idx];
457 found = true;
458 break ;
459 }
460
461 if (pos_history_idx < POS_HISTORY_COUNT - 1)
462 pos_history_idx++;
463 }
464
465 /* Not found? */
466 if (!found)
467 {
468 if (last_pos > 0)
469 {
470 last_pos = -1;
471 logf("seek again");
472 goto check_again;
473 }
474
475 if (fd != filenametag_fd)
476 close(fd);
477 return -4;
478 }
479
480 if (fd != filenametag_fd)
481 close(fd);
482
483 return tfe.idx_id;
484 }
485
486 static int find_index(const char *filename)
487 {
488 long idx_id = -1;
489
490 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
491 if (tc_stat.ramcache && dircache_is_enabled())
492 idx_id = find_entry_ram(filename, NULL);
493 #endif
494
495 if (idx_id < 0)
496 idx_id = find_entry_disk(filename);
497
498 return idx_id;
499 }
500
501 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
502 {
503 int idx_id;
504
505 if (!tc_stat.ready)
506 return false;
507
508 idx_id = find_index(filename);
509 if (idx_id < 0)
510 return false;
511
512 if (!tagcache_search(tcs, tag_filename))
513 return false;
514
515 tcs->entry_count = 0;
516 tcs->idx_id = idx_id;
517
518 return true;
519 }
520
521 static bool get_index(int masterfd, int idxid,
522 struct index_entry *idx, bool use_ram)
523 {
524 if (idxid < 0)
525 {
526 logf("Incorrect idxid: %d", idxid);
527 return false;
528 }
529
530 #ifdef HAVE_TC_RAMCACHE
531 if (tc_stat.ramcache && use_ram)
532 {
533 if (hdr->indices[idxid].flag & FLAG_DELETED)
534 return false;
535
536 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
537 return true;
538 }
539 #else
540 (void)use_ram;
541 #endif
542
543 lseek(masterfd, idxid * sizeof(struct index_entry)
544 + sizeof(struct master_header), SEEK_SET);
545 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
546 != sizeof(struct index_entry))
547 {
548 logf("read error #3");
549 return false;
550 }
551
552 if (idx->flag & FLAG_DELETED)
553 return false;
554
555 return true;
556 }
557
558 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
559 {
560 /* We need to exclude all memory only flags & tags when writing to disk. */
561 if (idx->flag & FLAG_DIRCACHE)
562 {
563 logf("memory only flags!");
564 return false;
565 }
566
567 #ifdef HAVE_TC_RAMCACHE
568 /* Only update numeric data. Writing the whole index to RAM by memcpy
569 * destroys dircache pointers!
570 */
571 if (tc_stat.ramcache)
572 {
573 int tag;
574 struct index_entry *idx_ram = &hdr->indices[idxid];
575
576 for (tag = 0; tag < TAG_COUNT; tag++)
577 {
578 if (tagcache_is_numeric_tag(tag))
579 {
580 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
581 }
582 }
583
584 /* Don't touch the dircache flag. */
585 idx_ram->flag = idx->flag | (idx_ram->flag & FLAG_DIRCACHE);
586 }
587 #endif
588
589 lseek(masterfd, idxid * sizeof(struct index_entry)
590 + sizeof(struct master_header), SEEK_SET);
591 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
592 != sizeof(struct index_entry))
593 {
594 logf("write error #3");
595 logf("idxid: %d", idxid);
596 return false;
597 }
598
599 return true;
600 }
601
602 static bool open_files(struct tagcache_search *tcs, int tag)
603 {
604 if (tcs->idxfd[tag] < 0)
605 {
606 char fn[MAX_PATH];
607
608 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
609 tcs->idxfd[tag] = open(fn, O_RDONLY);
610 }
611
612 if (tcs->idxfd[tag] < 0)
613 {
614 logf("File not open!");
615 return false;
616 }
617
618 return true;
619 }
620
621 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
622 int tag, char *buf, long size)
623 {
624 struct tagfile_entry tfe;
625 long seek;
626
627 *buf = '\0';
628
629 if (tagcache_is_numeric_tag(tag))
630 return false;
631
632 seek = idx->tag_seek[tag];
633 if (seek < 0)
634 {
635 logf("Retrieve failed");
636 return false;
637 }
638
639 #ifdef HAVE_TC_RAMCACHE
640 if (tcs->ramsearch)
641 {
642 struct tagfile_entry *ep;
643
644 # ifdef HAVE_DIRCACHE
645 if (tag == tag_filename && idx->flag & FLAG_DIRCACHE)
646 {
647 dircache_copy_path((struct dirent *)seek,
648 buf, size);
649 return true;
650 }
651 else
652 # endif
653 if (tag != tag_filename)
654 {
655 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
656 strncpy(buf, ep->tag_data, size-1);
657
658 return true;
659 }
660 }
661 #endif
662
663 if (!open_files(tcs, tag))
664 return false;
665
666 lseek(tcs->idxfd[tag], seek, SEEK_SET);
667 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
668 != sizeof(struct tagfile_entry))
669 {
670 logf("read error #5");
671 return false;
672 }
673
674 if (tfe.tag_length >= size)
675 {
676 logf("too small buffer");
677 return false;
678 }
679
680 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
681 tfe.tag_length)
682 {
683 logf("read error #6");
684 return false;
685 }
686
687 buf[tfe.tag_length] = '\0';
688
689 return true;
690 }
691
692 static long check_virtual_tags(int tag, const struct index_entry *idx)
693 {
694 long data = 0;
695
696 switch (tag)
697 {
698 case tag_virt_length_sec:
699 data = (idx->tag_seek[tag_length]/1000) % 60;
700 break;
701
702 case tag_virt_length_min:
703 data = (idx->tag_seek[tag_length]/1000) / 60;
704 break;
705
706 case tag_virt_playtime_sec:
707 data = (idx->tag_seek[tag_playtime]/1000) % 60;
708 break;
709
710 case tag_virt_playtime_min:
711 data = (idx->tag_seek[tag_playtime]/1000) / 60;
712 break;
713
714 case tag_virt_autoscore:
715 if (idx->tag_seek[tag_length] == 0
716 || idx->tag_seek[tag_playcount] == 0)
717 {
718 data = 0;
719 }
720 else
721 {
722 data = 100 * idx->tag_seek[tag_playtime]
723 / idx->tag_seek[tag_length]
724 / idx->tag_seek[tag_playcount];
725 }
726 break;
727
728 /* How many commits before the file has been added to the DB. */
729 case tag_virt_entryage:
730 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
731 break;
732
733 default:
734 data = idx->tag_seek[tag];
735 }
736
737 return data;
738 }
739
740 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
741 {
742 struct index_entry idx;
743
744 if (!tc_stat.ready)
745 return false;
746
747 if (!tagcache_is_numeric_tag(tag))
748 return -1;
749
750 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
751 return -2;
752
753 return check_virtual_tags(tag, &idx);
754 }
755
756 inline static bool str_ends_with(const char *str1, const char *str2)
757 {
758 int str_len = strlen(str1);
759 int clause_len = strlen(str2);
760
761 if (clause_len > str_len)
762 return false;
763
764 return !strcasecmp(&str1[str_len - clause_len], str2);
765 }
766
767 inline static bool str_oneof(const char *str, const char *list)
768 {
769 const char *sep;
770 int l, len = strlen(str);
771
772 while (*list)
773 {
774 sep = strchr(list, '|');
775 l = sep ? (long)sep - (long)list : (int)strlen(list);
776 if ((l==len) && !strncasecmp(str, list, len))
777 return true;
778 list += sep ? l + 1 : l;
779 }
780
781 return false;
782 }
783
784 static bool check_against_clause(long numeric, const char *str,
785 const struct tagcache_search_clause *clause)
786 {
787 if (clause->numeric)
788 {
789 switch (clause->type)
790 {
791 case clause_is:
792 return numeric == clause->numeric_data;
793 case clause_is_not:
794 return numeric != clause->numeric_data;
795 case clause_gt:
796 return numeric > clause->numeric_data;
797 case clause_gteq:
798 return numeric >= clause->numeric_data;
799 case clause_lt:
800 return numeric < clause->numeric_data;
801 case clause_lteq:
802 return numeric <= clause->numeric_data;
803 default:
804 logf("Incorrect numeric tag: %d", clause->type);
805 }
806 }
807 else
808 {
809 switch (clause->type)
810 {
811 case clause_is:
812 return !strcasecmp(clause->str, str);
813 case clause_is_not:
814 return strcasecmp(clause->str, str);
815 case clause_gt:
816 return 0>strcasecmp(clause->str, str);
817 case clause_gteq:
818 return 0>=strcasecmp(clause->str, str);
819 case clause_lt:
820 return 0<strcasecmp(clause->str, str);
821 case clause_lteq:
822 return 0<=strcasecmp(clause->str, str);
823 case clause_contains:
824 return (strcasestr(str, clause->str) != NULL);
825 case clause_not_contains:
826 return (strcasestr(str, clause->str) == NULL);
827 case clause_begins_with:
828 return (strcasestr(str, clause->str) == str);
829 case clause_not_begins_with:
830 return (strcasestr(str, clause->str) != str);
831 case clause_ends_with:
832 return str_ends_with(str, clause->str);
833 case clause_not_ends_with:
834 return !str_ends_with(str, clause->str);
835 case clause_oneof:
836 return str_oneof(str, clause->str);
837
838 default:
839 logf("Incorrect tag: %d", clause->type);
840 }
841 }
842
843 return false;
844 }
845
846 static bool check_clauses(struct tagcache_search *tcs,
847 struct index_entry *idx,
848 struct tagcache_search_clause **clause, int count)
849 {
850 int i;
851
852 #ifdef HAVE_TC_RAMCACHE
853 if (tcs->ramsearch)
854 {
855 /* Go through all conditional clauses. */
856 for (i = 0; i < count; i++)
857 {
858 struct tagfile_entry *tfe;
859 int seek;
860 char buf[256];
861 char *str = NULL;
862
863 seek = check_virtual_tags(clause[i]->tag, idx);
864
865 if (!tagcache_is_numeric_tag(clause[i]->tag))
866 {
867 if (clause[i]->tag == tag_filename)
868 {
869 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
870 str = buf;
871 }
872 else
873 {
874 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
875 str = tfe->tag_data;
876 }
877 }
878
879 if (!check_against_clause(seek, str, clause[i]))
880 return false;
881 }
882 }
883 else
884 #endif
885 {
886 /* Check for conditions. */
887 for (i = 0; i < count; i++)
888 {
889 struct tagfile_entry tfe;
890 int seek;
891 char str[256];
892
893 seek = check_virtual_tags(clause[i]->tag, idx);
894
895 memset(str, 0, sizeof str);
896 if (!tagcache_is_numeric_tag(clause[i]->tag))
897 {
898 int fd = tcs->idxfd[clause[i]->tag];
899 lseek(fd, seek, SEEK_SET);
900 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
901 if (tfe.tag_length >= (int)sizeof(str))
902 {
903 logf("Too long tag read!");
904 break ;
905 }
906
907 read(fd, str, tfe.tag_length);
908
909 /* Check if entry has been deleted. */
910 if (str[0] == '\0')
911 break;
912 }
913
914 if (!check_against_clause(seek, str, clause[i]))
915 return false;
916 }
917 }
918
919 return true;
920 }
921
922 bool tagcache_check_clauses(struct tagcache_search *tcs,
923 struct tagcache_search_clause **clause, int count)
924 {
925 struct index_entry idx;
926
927 if (count == 0)
928 return true;
929
930 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
931 return false;
932
933 return check_clauses(tcs, &idx, clause, count);
934 }
935
936 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
937 {
938 int i;
939
940 /* If uniq buffer is not defined we must return true for search to work. */
941 if (tcs->unique_list == NULL
942 || (!tagcache_is_unique_tag(tcs->type)
943 && !tagcache_is_numeric_tag(tcs->type)))
944 {
945 return true;
946 }
947
948 for (i = 0; i < tcs->unique_list_count; i++)
949 {
950 /* Return false if entry is found. */
951 if (tcs->unique_list[i] == id)
952 return false;
953 }
954
955 if (tcs->unique_list_count < tcs->unique_list_capacity)
956 {
957 tcs->unique_list[i] = id;
958 tcs->unique_list_count++;
959 }
960
961 return true;
962 }
963
964 static bool build_lookup_list(struct tagcache_search *tcs)
965 {
966 struct index_entry entry;
967 int i;
968
969 tcs->seek_list_count = 0;
970
971 #ifdef HAVE_TC_RAMCACHE
972 if (tcs->ramsearch)
973 {
974 int j;
975
976 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
977 {
978 struct index_entry *idx = &hdr->indices[i];
979 if (tcs->seek_list_count == SEEK_LIST_SIZE)
980 break ;
981
982 /* Skip deleted files. */
983 if (idx->flag & FLAG_DELETED)
984 continue;
985
986 /* Go through all filters.. */
987 for (j = 0; j < tcs->filter_count; j++)
988 {
989 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
990 {
991 break ;
992 }
993 }
994
995 if (j < tcs->filter_count)
996 continue ;
997
998 /* Check for conditions. */
999 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1000 continue;
1001
1002 /* Add to the seek list if not already in uniq buffer. */
1003 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1004 continue;
1005
1006 /* Lets add it. */
1007 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1008 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1009 tcs->seek_list_count++;
1010 }
1011
1012 tcs->seek_pos = i;
1013
1014 return tcs->seek_list_count > 0;
1015 }
1016 #endif
1017
1018 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1019 sizeof(struct master_header), SEEK_SET);
1020
1021 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1022 == sizeof(struct index_entry))
1023 {
1024 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1025 break ;
1026
1027 tcs->seek_pos++;
1028
1029 /* Check if entry has been deleted. */
1030 if (entry.flag & FLAG_DELETED)
1031 continue;
1032
1033 /* Go through all filters.. */
1034 for (i = 0; i < tcs->filter_count; i++)
1035 {
1036 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1037 break ;
1038 }
1039
1040 if (i < tcs->filter_count)
1041 continue ;
1042
1043 /* Check for conditions. */
1044 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1045 continue;
1046
1047 /* Add to the seek list if not already in uniq buffer. */
1048 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1049 continue;
1050
1051 /* Lets add it. */
1052 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1053 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1054 tcs->seek_list_count++;
1055
1056 yield();
1057 }
1058
1059 return tcs->seek_list_count > 0;
1060 }
1061
1062
1063 static void remove_files(void)
1064 {
1065 int i;
1066 char buf[MAX_PATH];
1067
1068 tc_stat.ready = false;
1069 tc_stat.ramcache = false;
1070 tc_stat.econ = false;
1071 remove(TAGCACHE_FILE_MASTER);
1072 for (i = 0; i < TAG_COUNT; i++)
1073 {
1074 if (tagcache_is_numeric_tag(i))
1075 continue;
1076
1077 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1078 remove(buf);
1079 }
1080 }
1081
1082
1083 static int open_master_fd(struct master_header *hdr, bool write)
1084 {
1085 int fd;
1086 int rc;
1087
1088 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
1089 if (fd < 0)
1090 {
1091 logf("master file open failed for R/W");
1092 tc_stat.ready = false;
1093 return fd;
1094 }
1095
1096 tc_stat.econ = false;
1097
1098 /* Check the header. */
1099 rc = read(fd, hdr, sizeof(struct master_header));
1100 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
1101 {
1102 /* Success. */
1103 return fd;
1104 }
1105
1106 /* Trying to read again, this time with endianess correction enabled. */
1107 lseek(fd, 0, SEEK_SET);
1108
1109 rc = ecread(fd, hdr, 1, master_header_ec, true);
1110 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
1111 {
1112 logf("header error");
1113 tc_stat.ready = false;
1114 close(fd);
1115 return -2;
1116 }
1117
1118 tc_stat.econ = true;
1119
1120 return fd;
1121 }
1122
1123 static bool check_all_headers(void)
1124 {
1125 struct master_header myhdr;
1126 struct tagcache_header tch;
1127 int tag;
1128 int fd;
1129
1130 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1131 return false;
1132
1133 close(fd);
1134 if (myhdr.dirty)
1135 {
1136 logf("tagcache is dirty!");
1137 return false;
1138 }
1139
1140 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1141
1142 for (tag = 0; tag < TAG_COUNT; tag++)
1143 {
1144 if (tagcache_is_numeric_tag(tag))
1145 continue;
1146
1147 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1148 return false;
1149
1150 close(fd);
1151 }
1152
1153 return true;
1154 }
1155
1156 bool tagcache_search(struct tagcache_search *tcs, int tag)
1157 {
1158 struct tagcache_header tag_hdr;
1159 struct master_header master_hdr;
1160 int i;
1161
1162 if (tcs->initialized)
1163 tagcache_search_finish(tcs);
1164
1165 while (read_lock)
1166 sleep(1);
1167
1168 memset(tcs, 0, sizeof(struct tagcache_search));
1169 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1170 return false;
1171
1172 tcs->position = sizeof(struct tagcache_header);
1173 tcs->type = tag;
1174 tcs->seek_pos = 0;
1175 tcs->seek_list_count = 0;
1176 tcs->filter_count = 0;
1177 tcs->masterfd = -1;
1178
1179 for (i = 0; i < TAG_COUNT; i++)
1180 tcs->idxfd[i] = -1;
1181
1182 #ifndef HAVE_TC_RAMCACHE
1183 tcs->ramsearch = false;
1184 #else
1185 tcs->ramsearch = tc_stat.ramcache;
1186 if (tcs->ramsearch)
1187 {
1188 tcs->entry_count = hdr->entry_count[tcs->type];
1189 }
1190 else
1191 #endif
1192 {
1193 if (!tagcache_is_numeric_tag(tcs->type))
1194 {
1195 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1196 if (tcs->idxfd[tcs->type] < 0)
1197 return false;
1198 }
1199
1200 /* Always open as R/W so we can pass tcs to functions that modify data also
1201 * without failing. */
1202 tcs->masterfd = open_master_fd(&master_hdr, true);
1203
1204 if (tcs->masterfd < 0)
1205 return false;
1206 }
1207
1208 tcs->valid = true;
1209 tcs->initialized = true;
1210 write_lock++;
1211
1212 return true;
1213 }
1214
1215 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1216 void *buffer, long length)
1217 {
1218 tcs->unique_list = (unsigned long *)buffer;
1219 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1220 tcs->unique_list_count = 0;
1221 }
1222
1223 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1224 int tag, int seek)
1225 {
1226 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1227 return false;
1228
1229 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1230 return false;
1231
1232 tcs->filter_tag[tcs->filter_count] = tag;
1233 tcs->filter_seek[tcs->filter_count] = seek;
1234 tcs->filter_count++;
1235
1236 return true;
1237 }
1238
1239 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1240 struct tagcache_search_clause *clause)
1241 {
1242 int i;
1243
1244 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1245 {
1246 logf("Too many clauses");
1247 return false;
1248 }
1249
1250 /* Check if there is already a similar filter in present (filters are
1251 * much faster than clauses).
1252 */
1253 for (i = 0; i < tcs->filter_count; i++)
1254 {
1255 if (tcs->filter_tag[i] == clause->tag)
1256 return true;
1257 }
1258
1259 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1260 {
1261 char buf[MAX_PATH];
1262
1263 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1264 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1265 }
1266
1267 tcs->clause[tcs->clause_count] = clause;
1268 tcs->clause_count++;
1269
1270 return true;
1271 }
1272
1273 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1274 ? (flag & FLAG_DIRCACHE) : 1)
1275
1276 static bool get_next(struct tagcache_search *tcs)
1277 {
1278 static char buf[TAG_MAXLEN+32];
1279 struct tagfile_entry entry;
1280 long flag = 0;
1281
1282 if (!tcs->valid || !tc_stat.ready)
1283 return false;
1284
1285 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1286 #ifdef HAVE_TC_RAMCACHE
1287 && !tcs->ramsearch
1288 #endif
1289 )
1290 return false;
1291
1292 /* Relative fetch. */
1293 if (tcs->filter_count > 0 || tcs->clause_count > 0
1294 || tagcache_is_numeric_tag(tcs->type))
1295 {
1296 /* Check for end of list. */
1297 if (tcs->seek_list_count == 0)
1298 {
1299 /* Try to fetch more. */
1300 if (!build_lookup_list(tcs))
1301 {
1302 tcs->valid = false;
1303 return false;
1304 }
1305 }
1306
1307 tcs->seek_list_count--;
1308 flag = tcs->seek_flags[tcs->seek_list_count];
1309
1310 /* Seek stream to the correct position and continue to direct fetch. */
1311 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1312 && !tagcache_is_numeric_tag(tcs->type))
1313 {
1314 if (!open_files(tcs, tcs->type))
1315 return false;
1316
1317 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1318 }
1319 else
1320 tcs->position = tcs->seek_list[tcs->seek_list_count];
1321 }
1322
1323 if (tagcache_is_numeric_tag(tcs->type))
1324 {
1325 snprintf(buf, sizeof(buf), "%d", tcs->position);
1326 tcs->result_seek = tcs->position;
1327 tcs->result = buf;
1328 tcs->result_len = strlen(buf) + 1;
1329 return true;
1330 }
1331
1332 /* Direct fetch. */
1333 #ifdef HAVE_TC_RAMCACHE
1334 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1335 {
1336 struct tagfile_entry *ep;
1337
1338 if (tcs->entry_count == 0)
1339 {
1340 tcs->valid = false;
1341 return false;
1342 }
1343 tcs->entry_count--;
1344
1345 tcs->result_seek = tcs->position;
1346
1347 # ifdef HAVE_DIRCACHE
1348 if (tcs->type == tag_filename)
1349 {
1350 dircache_copy_path((struct dirent *)tcs->position,
1351 buf, sizeof buf);
1352 tcs->result = buf;
1353 tcs->result_len = strlen(buf) + 1;
1354 tcs->idx_id = FLAG_GET_ATTR(flag);
1355 tcs->ramresult = false;
1356
1357 return true;
1358 }
1359 # endif
1360
1361 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1362 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1363 tcs->result = ep->tag_data;
1364 tcs->result_len = strlen(tcs->result) + 1;
1365 tcs->idx_id = ep->idx_id;
1366 tcs->ramresult = true;
1367
1368 return true;
1369 }
1370 else
1371 #endif
1372 {
1373 if (!open_files(tcs, tcs->type))
1374 return false;
1375
1376 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1377 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1378 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1379 {
1380 /* End of data. */
1381 tcs->valid = false;
1382 return false;
1383 }
1384 }
1385
1386 if (entry.tag_length > (long)sizeof(buf))
1387 {
1388 tcs->valid = false;
1389 logf("too long tag #2");
1390 return false;
1391 }
1392
1393 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1394 {
1395 tcs->valid = false;
1396 logf("read error #4");
1397 return false;
1398 }
1399
1400 tcs->result = buf;
1401 tcs->result_len = strlen(tcs->result) + 1;
1402 tcs->idx_id = entry.idx_id;
1403 tcs->ramresult = false;
1404
1405 return true;
1406 }
1407
1408 bool tagcache_get_next(struct tagcache_search *tcs)
1409 {
1410 while (get_next(tcs))
1411 {
1412 if (tcs->result_len > 1)
1413 return true;
1414 }
1415
1416 return false;
1417 }
1418
1419 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1420 int tag, char *buf, long size)
1421 {
1422 struct index_entry idx;
1423
1424 *buf = '\0';
1425 if (!get_index(tcs->masterfd, idxid, &idx, true))
1426 return false;
1427
1428 return retrieve(tcs, &idx, tag, buf, size);
1429 }
1430
1431 static bool update_master_header(void)
1432 {
1433 struct master_header myhdr;
1434 int fd;
1435
1436 if (!tc_stat.ready)
1437 return false;
1438
1439 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1440 return false;
1441
1442 myhdr.serial = current_tcmh.serial;
1443 myhdr.commitid = current_tcmh.commitid;
1444
1445 /* Write it back */
1446 lseek(fd, 0, SEEK_SET);
1447 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1448 close(fd);
1449
1450 #ifdef HAVE_TC_RAMCACHE
1451 if (hdr)
1452 {
1453 hdr->h.serial = current_tcmh.serial;
1454 hdr->h.commitid = current_tcmh.commitid;
1455 }
1456 #endif
1457
1458 return true;
1459 }
1460
1461 #if 0
1462
1463 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1464 {
1465 struct tagentry *entry;
1466
1467 if (tcs->type != tag_title)
1468 return ;
1469
1470 /* We will need reserve buffer for this. */
1471 if (tcs->ramcache)
1472 {
1473 struct tagfile_entry *ep;
1474
1475 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1476 tcs->seek_list[tcs->seek_list_count];
1477 }
1478
1479 entry = find_entry_ram();
1480
1481 }
1482 #endif
1483
1484 void tagcache_search_finish(struct tagcache_search *tcs)
1485 {
1486 int i;
1487
1488 if (!tcs->initialized)
1489 return;
1490
1491 if (tcs->masterfd >= 0)
1492 {
1493 close(tcs->masterfd);
1494 tcs->masterfd = -1;
1495 }
1496
1497 for (i = 0; i < TAG_COUNT; i++)
1498 {
1499 if (tcs->idxfd[i] >= 0)
1500 {
1501 close(tcs->idxfd[i]);
1502 tcs->idxfd[i] = -1;
1503 }
1504 }
1505
1506 tcs->ramsearch = false;
1507 tcs->valid = false;
1508 tcs->initialized = 0;
1509 if (write_lock > 0)
1510 write_lock--;
1511 }
1512
1513 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1514 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1515 {
1516 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1517 }
1518
1519 static long get_tag_numeric(const struct index_entry *entry, int tag)
1520 {
1521 return entry->tag_seek[tag];
1522 }
1523
1524 static char* get_tag_string(const struct index_entry *entry, int tag)
1525 {
1526 char* s = get_tag(entry, tag)->tag_data;
1527 return strcmp(s, UNTAGGED) ? s : NULL;
1528 }
1529
1530 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1531 {
1532 struct index_entry *entry;
1533 int idx_id;
1534
1535 if (!tc_stat.ready)
1536 return false;
1537
1538 /* Find the corresponding entry in tagcache. */
1539 idx_id = find_entry_ram(filename, NULL);
1540 if (idx_id < 0 || !tc_stat.ramcache)
1541 return false;
1542
1543 entry = &hdr->indices[idx_id];
1544
1545 id3->title = get_tag_string(entry, tag_title);
1546 id3->artist = get_tag_string(entry, tag_artist);
1547 id3->album = get_tag_string(entry, tag_album);
1548 id3->genre_string = get_tag_string(entry, tag_genre);
1549 id3->composer = get_tag_string(entry, tag_composer);
1550 id3->comment = get_tag_string(entry, tag_comment);
1551 id3->albumartist = get_tag_string(entry, tag_albumartist);
1552
1553 id3->playcount = get_tag_numeric(entry, tag_playcount);
1554 id3->rating = get_tag_numeric(entry, tag_rating);
1555 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1556 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1557 id3->year = get_tag_numeric(entry, tag_year);
1558
1559 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1560 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1561 if (id3->bitrate == 0)
1562 id3->bitrate = 1;
1563
1564 return true;
1565 }
1566 #endif
1567
1568 static inline void write_item(const char *item)
1569 {
1570 int len = strlen(item) + 1;
1571
1572 data_size += len;
1573 write(cachefd, item, len);
1574 }
1575
1576 static int check_if_empty(char **tag)
1577 {
1578 int length;
1579
1580 if (*tag == NULL || *tag[0] == '\0')
1581 {
1582 *tag = UNTAGGED;
1583 return sizeof(UNTAGGED); /* Tag length */
1584 }
1585
1586 length = strlen(*tag);
1587 if (length > TAG_MAXLEN)
1588 {
1589 logf("over length tag: %s", *tag);
1590 length = TAG_MAXLEN;
1591 *tag[length] = '\0';
1592 }
1593
1594 return length + 1;
1595 }
1596
1597 #define ADD_TAG(entry,tag,data) \
1598 /* Adding tag */ \
1599 entry.tag_offset[tag] = offset; \
1600 entry.tag_length[tag] = check_if_empty(data); \
1601 offset += entry.tag_length[tag]
1602
1603 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1604 static void add_tagcache(char *path, const struct dirent *dc)
1605 #else
1606 static void add_tagcache(char *path)
1607 #endif
1608 {
1609 struct track_info track;
1610 struct temp_file_entry entry;
1611 bool ret;
1612 int fd;
1613 char tracknumfix[3];
1614 int offset = 0;
1615 int path_length = strlen(path);
1616 bool has_albumartist;
1617
1618 if (cachefd < 0)
1619 return ;
1620
1621 /* Check for overlength file path. */
1622 if (path_length > TAG_MAXLEN)
1623 {
1624 /* Path can't be shortened. */
1625 logf("Too long path: %s", path);
1626 return ;
1627 }
1628
1629 /* Check if the file is supported. */
1630 if (probe_file_format(path) == AFMT_UNKNOWN)
1631 return ;
1632
1633 /* Check if the file is already cached. */
1634 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1635 if (tc_stat.ramcache && dircache_is_enabled())
1636 {
1637 if (find_entry_ram(path, dc) >= 0)
1638 return ;
1639 }
1640 else
1641 #endif
1642 {
1643 if (filenametag_fd >= 0)
1644 {
1645 if (find_entry_disk(path) >= 0)
1646 return ;
1647 }
1648 }
1649
1650 fd = open(path, O_RDONLY);
1651 if (fd < 0)
1652 {
1653 logf("open fail: %s", path);
1654 return ;
1655 }
1656
1657 memset(&track, 0, sizeof(struct track_info));
1658 memset(&entry, 0, sizeof(struct temp_file_entry));
1659 memset(&tracknumfix, 0, sizeof(tracknumfix));
1660 ret = get_metadata(&track, fd, path, false);
1661 close(fd);
1662
1663 if (!ret)
1664 return ;
1665
1666 logf("-> %s", path);
1667
1668 /* Generate track number if missing. */
1669 if (track.id3.tracknum <= 0)
1670 {
1671 const char *p = strrchr(path, '.');
1672
1673 if (p == NULL)
1674 p = &path[strlen(path)-1];
1675
1676 while (*p != '/')
1677 {
1678 if (isdigit(*p) && isdigit(*(p-1)))
1679 {
1680 tracknumfix[1] = *p--;
1681 tracknumfix[0] = *p;
1682 break;
1683 }
1684 p--;
1685 }
1686
1687 if (tracknumfix[0] != '\0')
1688 {
1689 track.id3.tracknum = atoi(tracknumfix);
1690 /* Set a flag to indicate track number has been generated. */
1691 entry.flag |= FLAG_TRKNUMGEN;
1692 }
1693 else
1694 {
1695 /* Unable to generate track number. */
1696 track.id3.tracknum = -1;
1697 }
1698 }
1699
1700 /* Numeric tags */
1701 entry.tag_offset[tag_year] = track.id3.year;
1702 entry.tag_offset[tag_tracknumber] = track.id3.tracknum;
1703 entry.tag_offset[tag_length] = track.id3.length;
1704 entry.tag_offset[tag_bitrate] = track.id3.bitrate;
1705
1706 /* String tags. */
1707 has_albumartist = track.id3.albumartist != NULL
1708 && strlen(track.id3.albumartist) > 0;
1709
1710 ADD_TAG(entry, tag_filename, &path);
1711 ADD_TAG(entry, tag_title, &track.id3.title);
1712 ADD_TAG(entry, tag_artist, &track.id3.artist);
1713 ADD_TAG(entry, tag_album, &track.id3.album);
1714 ADD_TAG(entry, tag_genre, &track.id3.genre_string);
1715 ADD_TAG(entry, tag_composer, &track.id3.composer);
1716 ADD_TAG(entry, tag_comment, &track.id3.comment);
1717 if (has_albumartist)
1718 {
1719 ADD_TAG(entry, tag_albumartist, &track.id3.albumartist);
1720 }
1721 else
1722 {
1723 ADD_TAG(entry, tag_albumartist, &track.id3.artist);
1724 }
1725 entry.data_length = offset;
1726
1727 /* Write the header */
1728 write(cachefd, &entry, sizeof(struct temp_file_entry));
1729
1730 /* And tags also... Correct order is critical */
1731 write_item(path);
1732 write_item(track.id3.title);
1733 write_item(track.id3.artist);
1734 write_item(track.id3.album);
1735 write_item(track.id3.genre_string);
1736 write_item(track.id3.composer);
1737 write_item(track.id3.comment);
1738 if (has_albumartist)
1739 {
1740 write_item(track.id3.albumartist);
1741 }
1742 else
1743 {
1744 write_item(track.id3.artist);
1745 }
1746 total_entry_count++;
1747 }
1748
1749 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1750 {
1751 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1752 int len = strlen(str)+1;
1753 int i;
1754 unsigned crc32;
1755 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1756 char buf[TAG_MAXLEN+32];
1757
1758 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1759 buf[i] = tolower(str[i]);
1760 buf[i] = '\0';
1761
1762 crc32 = crc_32(buf, i, 0xffffffff);
1763
1764 if (unique)
1765 {
1766 /* Check if the crc does not exist -> entry does not exist for sure. */
1767 for (i = 0; i < tempbufidx; i++)
1768 {
1769 if (crcbuf[-i] != crc32)
1770 continue;
1771
1772 if (!strcasecmp(str, index[i].str))
1773 {
1774 if (id < 0 || id >= lookup_buffer_depth)
1775 {
1776 logf("lookup buf overf.: %d", id);
1777 return false;
1778 }
1779
1780 lookup[id] = &index[i];
1781 return true;
1782 }
1783 }
1784 }
1785
1786 /* Insert to CRC buffer. */
1787 crcbuf[-tempbufidx] = crc32;
1788 tempbuf_left -= 4;
1789
1790 /* Insert it to the buffer. */
1791 tempbuf_left -= len;
1792 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1793 return false;
1794
1795 if (id >= lookup_buffer_depth)
1796 {
1797 logf("lookup buf overf. #2: %d", id);
1798 return false;
1799 }
1800
1801 if (id >= 0)
1802 {
1803 lookup[id] = &index[tempbufidx];
1804 index[tempbufidx].idlist.id = id;
1805 }
1806 else
1807 index[tempbufidx].idlist.id = -1;
1808
1809 index[tempbufidx].idlist.next = NULL;
1810 index[tempbufidx].idx_id = idx_id;
1811 index[tempbufidx].seek = -1;
1812 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1813 memcpy(index[tempbufidx].str, str, len);
1814 tempbuf_pos += len;
1815 tempbufidx++;
1816
1817 return true;
1818 }
1819
1820 static int compare(const void *p1, const void *p2)
1821 {
1822 do_timed_yield();
1823
1824 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1825 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1826
1827 if (strcmp(e1->str, UNTAGGED) == 0)
1828 {
1829 if (strcmp(e2->str, UNTAGGED) == 0)
1830 return 0;
1831 return -1;
1832 }
1833 else if (strcmp(e2->str, UNTAGGED) == 0)
1834 return 1;
1835
1836 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1837 }
1838
1839 static int tempbuf_sort(int fd)
1840 {
1841 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1842 struct tagfile_entry fe;
1843 int i;
1844 int length;
1845
1846 /* Generate reverse lookup entries. */
1847 for (i = 0; i < lookup_buffer_depth; i++)
1848 {
1849 struct tempbuf_id_list *idlist;
1850
1851 if (!lookup[i])
1852 continue;
1853
1854 if (lookup[i]->idlist.id == i)
1855 continue;
1856
1857 idlist = &lookup[i]->idlist;
1858 while (idlist->next != NULL)
1859 idlist = idlist->next;
1860
1861 tempbuf_left -= sizeof(struct tempbuf_id_list);
1862 if (tempbuf_left - 4 < 0)
1863 return -1;
1864
1865 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1866 if (tempbuf_pos & 0x03)
1867 {
1868 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1869 tempbuf_left -= 3;
1870 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1871 }
1872 tempbuf_pos += sizeof(struct tempbuf_id_list);
1873
1874 idlist = idlist->next;
1875 idlist->id = i;
1876 idlist->next = NULL;
1877
1878 do_timed_yield();
1879 }
1880
1881 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1882 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1883
1884 for (i = 0; i < tempbufidx; i++)
1885 {
1886 struct tempbuf_id_list *idlist = &index[i].idlist;
1887
1888 /* Fix the lookup list. */
1889 while (idlist != NULL)
1890 {
1891 if (idlist->id >= 0)
1892 lookup[idlist->id] = &index[i];
1893 idlist = idlist->next;
1894 }
1895
1896 index[i].seek = lseek(fd, 0, SEEK_CUR);
1897 length = strlen(index[i].str) + 1;
1898 fe.tag_length = length;
1899 fe.idx_id = index[i].idx_id;
1900
1901 /* Check the chunk alignment. */
1902 if ((fe.tag_length + sizeof(struct tagfile_entry))
1903 % TAGFILE_ENTRY_CHUNK_LENGTH)
1904 {
1905 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1906 ((fe.tag_length + sizeof(struct tagfile_entry))
1907 % TAGFILE_ENTRY_CHUNK_LENGTH);
1908 }
1909
1910 #ifdef TAGCACHE_STRICT_ALIGN
1911 /* Make sure the entry is long aligned. */
1912 if (index[i].seek & 0x03)
1913 {
1914 logf("tempbuf_sort: alignment error!");
1915 return -3;
1916 }
1917 #endif
1918
1919 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
1920 sizeof(struct tagfile_entry))
1921 {
1922 logf("tempbuf_sort: write error #1");
1923 return -1;
1924 }
1925
1926 if (write(fd, index[i].str, length) != length)
1927 {
1928 logf("tempbuf_sort: write error #2");
1929 return -2;
1930 }
1931
1932 /* Write some padding. */
1933 if (fe.tag_length - length > 0)
1934 write(fd, "XXXXXXXX", fe.tag_length - length);
1935 }
1936
1937 return i;
1938 }
1939
1940 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
1941 {
1942 if (id < 0 || id >= lookup_buffer_depth)
1943 return NULL;
1944
1945 return lookup[id];
1946 }
1947
1948
1949 inline static int tempbuf_find_location(int id)
1950 {
1951 struct tempbuf_searchidx *entry;
1952
1953 entry = tempbuf_locate(id);
1954 if (entry == NULL)
1955 return -1;
1956
1957 return entry->seek;
1958 }
1959
1960 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
1961 {
1962 struct master_header tcmh;
1963 struct index_entry idx;
1964 int masterfd;
1965 int masterfd_pos;
1966 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
1967 int max_entries;
1968 int entries_processed = 0;
1969 int i, j;
1970
1971 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
1972
1973 logf("Building numeric indices...");
1974 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
1975
1976 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
1977 return false;
1978
1979 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
1980 SEEK_CUR);
1981 if (masterfd_pos == filesize(masterfd))
1982 {
1983 logf("we can't append!");
1984 close(masterfd);
1985 return false;
1986 }
1987
1988 while (entries_processed < h->entry_count)
1989 {
1990 int count = MIN(h->entry_count - entries_processed, max_entries);
1991
1992 /* Read in as many entries as possible. */
1993 for (i = 0; i < count; i++)
1994 {
1995 /* Read in numeric data. */
1996 if (read(tmpfd, &entrybuf[i], sizeof(struct temp_file_entry)) !=
1997 sizeof(struct temp_file_entry))
1998 {
1999 logf("read fail #1");
2000 close(masterfd);
2001 return false;
2002 }
2003
2004 /* Skip string data. */
2005 lseek(tmpfd, entrybuf[i].data_length, SEEK_CUR);
2006 }
2007
2008 /* Commit the data to the index. */
2009 for (i = 0; i < count; i++)
2010 {
2011 int loc = lseek(masterfd, 0, SEEK_CUR);
2012
2013 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2014 != sizeof(struct index_entry))
2015 {
2016 logf("read fail #2");
2017 close(masterfd);
2018 return false;
2019 }
2020
2021 for (j = 0; j < TAG_COUNT; j++)
2022 {
2023 if (!tagcache_is_numeric_tag(j))
2024 continue;
2025
2026 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2027 }
2028 idx.flag = entrybuf[i].flag;
2029
2030 if (tc_stat.ready && current_tcmh.commitid > 0)
2031 {
2032 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2033 idx.flag |= FLAG_DIRTYNUM;
2034 }
2035
2036 /* Write back the updated index. */
2037 lseek(masterfd, loc, SEEK_SET);
2038 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2039 != sizeof(struct index_entry))
2040 {
2041 logf("write fail");
2042 close(masterfd);
2043 return false;
2044 }
2045 }
2046
2047 entries_processed += count;
2048 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2049 }
2050
2051 close(masterfd);
2052
2053 return true;
2054 }
2055
2056 /**
2057 * Return values:
2058 * > 0 success
2059 * == 0 temporary failure
2060 * < 0 fatal error
2061 */
2062 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2063 {
2064 int i;
2065 struct tagcache_header tch;
2066 struct master_header tcmh;
2067 struct index_entry idxbuf[IDX_BUF_DEPTH];
2068 int idxbuf_pos;
2069 char buf[TAG_MAXLEN+32];
2070 int fd = -1, masterfd;
2071 bool error = false;
2072 int init;
2073 int masterfd_pos;
2074
2075 logf("Building index: %d", index_type);
2076
2077 /* Check the number of entries we need to allocate ram for. */
2078 commit_entry_count = h->entry_count + 1;
2079
2080 masterfd = open_master_fd(&tcmh, false);
2081 if (masterfd >= 0)
2082 {
2083 commit_entry_count += tcmh.tch.entry_count;
2084 close(masterfd);
2085 }
2086 else
2087 remove_files(); /* Just to be sure we are clean. */
2088
2089 /* Open the index file, which contains the tag names. */
2090 fd = open_tag_fd(&tch, index_type, true);
2091 if (fd >= 0)
2092 {
2093 logf("tch.datasize=%ld", tch.datasize);
2094 lookup_buffer_depth = 1 +
2095 /* First part */ commit_entry_count +
2096 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2097 }
2098 else
2099 {
2100 lookup_buffer_depth = 1 +
2101 /* First part */ commit_entry_count +
2102 /* Second part */ 0;
2103 }
2104
2105 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2106 logf("commit_entry_count=%ld", commit_entry_count);
2107
2108 /* Allocate buffer for all index entries from both old and new
2109 * tag files. */
2110 tempbufidx = 0;
2111 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2112
2113 /* Allocate lookup buffer. The first portion of commit_entry_count
2114 * contains the new tags in the temporary file and the second
2115 * part for locating entries already in the db.
2116 *
2117 * New tags Old tags
2118 * +---------+---------------------------+
2119 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2120 * +---------+---------------------------+
2121 *
2122 * Old tags are inserted to a temporary buffer with position:
2123 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2124 * And new tags with index:
2125 * tempbuf_insert(idx, ...);
2126 *
2127 * The buffer is sorted and written into tag file:
2128 * tempbuf_sort(...);
2129 * leaving master index locations messed up.
2130 *
2131 * That is fixed using the lookup buffer for old tags:
2132 * new_seek = tempbuf_find_location(old_seek, ...);
2133 * and for new tags:
2134 * new_seek = tempbuf_find_location(idx);
2135 */
2136 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2137 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2138 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2139
2140 /* And calculate the remaining data space used mainly for storing
2141 * tag data (strings). */
2142 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2143 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2144 {
2145 logf("Buffer way too small!");
2146 return 0;
2147 }
2148
2149 if (fd >= 0)
2150 {
2151 /**
2152 * If tag file contains unique tags (sorted index), we will load
2153 * it entirely into memory so we can resort it later for use with
2154 * chunked browsing.
2155 */
2156 if (tagcache_is_sorted_tag(index_type))
2157 {
2158 logf("loading tags...");
2159 for (i = 0; i < tch.entry_count; i++)
2160 {
2161 struct tagfile_entry entry;
2162 int loc = lseek(fd, 0, SEEK_CUR);
2163 bool ret;
2164
2165 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2166 != sizeof(struct tagfile_entry))
2167 {
2168 logf("read error #7");
2169 close(fd);
2170 return -2;
2171 }
2172
2173 if (entry.tag_length >= (int)sizeof(buf))
2174 {
2175 logf("too long tag #3");
2176 close(fd);
2177 return -2;
2178 }
2179
2180 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2181 {
2182 logf("read error #8");
2183 close(fd);
2184 return -2;
2185 }
2186
2187 /* Skip deleted entries. */
2188 if (buf[0] == '\0')
2189 continue;
2190
2191 /**
2192 * Save the tag and tag id in the memory buffer. Tag id
2193 * is saved so we can later reindex the master lookup
2194 * table when the index gets resorted.
2195 */
2196 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2197 + commit_entry_count, entry.idx_id,
2198 tagcache_is_unique_tag(index_type));
2199 if (!ret)
2200 {
2201 close(fd);
2202 return -3;
2203 }
2204 do_timed_yield();
2205 }
2206 logf("done");
2207 }
2208 else
2209 tempbufidx = tch.entry_count;
2210 }
2211 else
2212 {
2213 /**
2214 * Creating new index file to store the tags. No need to preload
2215 * anything whether the index type is sorted or not.
2216 */
2217 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2218 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2219 if (fd < 0)
2220 {
2221 logf("%s open fail", buf);
2222 return -2;
2223 }
2224
2225 tch.magic = TAGCACHE_MAGIC;
2226 tch.entry_count = 0;
2227 tch.datasize = 0;
2228
2229 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2230 != sizeof(struct tagcache_header))
2231 {
2232 logf("header write failed");
2233 close(fd);
2234 return -2;
2235 }
2236 }
2237
2238 /* Loading the tag lookup file as "master file". */
2239 logf("Loading index file");
2240 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2241
2242 if (masterfd < 0)
2243 {
2244 logf("Creating new DB");
2245 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2246
2247 if (masterfd < 0)
2248 {
2249 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2250 close(fd);
2251 return -2;
2252 }
2253
2254 /* Write the header (write real values later). */
2255 memset(&tcmh, 0, sizeof(struct master_header));
2256 tcmh.tch = *h;
2257 tcmh.tch.entry_count = 0;
2258 tcmh.tch.datasize = 0;
2259 tcmh.dirty = true;
2260 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2261 init = true;
2262 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2263 }
2264 else
2265 {
2266 /**
2267 * Master file already exists so we need to process the current
2268 * file first.
2269 */
2270 init = false;
2271
2272 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2273 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2274 {
2275 logf("header error");
2276 close(fd);
2277 close(masterfd);
2278 return -2;
2279 }
2280
2281 /**
2282 * If we reach end of the master file, we need to expand it to
2283 * hold new tags. If the current index is not sorted, we can
2284 * simply append new data to end of the file.
2285 * However, if the index is sorted, we need to update all tag
2286 * pointers in the master file for the current index.
2287 */
2288 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2289 SEEK_CUR);
2290 if (masterfd_pos == filesize(masterfd))
2291 {
2292 logf("appending...");
2293 init = true;
2294 }
2295 }
2296
2297 /**
2298 * Load new unique tags in memory to be sorted later and added
2299 * to the master lookup file.
2300 */
2301 if (tagcache_is_sorted_tag(index_type))
2302 {
2303 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2304 /* h is the header of the temporary file containing new tags. */
2305 logf("inserting new tags...");
2306 for (i = 0; i < h->entry_count; i++)
2307 {
2308 struct temp_file_entry entry;
2309
2310 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2311 sizeof(struct temp_file_entry))
2312 {
2313 logf("read fail #3");
2314 error = true;
2315 goto error_exit;
2316 }
2317
2318 /* Read data. */
2319 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2320 {
2321 logf("too long entry!");
2322 error = true;
2323 goto error_exit;
2324 }
2325
2326 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2327 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2328 entry.tag_length[index_type])
2329 {
2330 logf("read fail #4");
2331 error = true;
2332 goto error_exit;
2333 }
2334
2335 if (tagcache_is_unique_tag(index_type))
2336 error = !tempbuf_insert(buf, i, -1, true);
2337 else
2338 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2339
2340 if (error)
2341 {
2342 logf("insert error");
2343 goto error_exit;
2344 }
2345
2346 /* Skip to next. */
2347 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2348 entry.tag_length[index_type], SEEK_CUR);
2349 do_timed_yield();
2350 }
2351 logf("done");
2352
2353 /* Sort the buffer data and write it to the index file. */
2354 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2355 i = tempbuf_sort(fd);
2356 if (i < 0)
2357 goto error_exit;
2358 logf("sorted %d tags", i);
2359
2360 /**
2361 * Now update all indexes in the master lookup file.
2362 */
2363 logf("updating indices...");
2364 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2365 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2366 {
2367 int j;
2368 int loc = lseek(masterfd, 0, SEEK_CUR);
2369
2370 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2371
2372 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2373 != (int)sizeof(struct index_entry)*idxbuf_pos)
2374 {
2375 logf("read fail #5");
2376 error = true;
2377 goto error_exit ;
2378 }
2379 lseek(masterfd, loc, SEEK_SET);
2380
2381 for (j = 0; j < idxbuf_pos; j++)
2382 {
2383 if (idxbuf[j].flag & FLAG_DELETED)
2384 {
2385 /* We can just ignore deleted entries. */
2386 idxbuf[j].tag_seek[index_type] = 0;
2387 continue;
2388 }
2389
2390 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2391 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2392 + commit_entry_count);
2393
2394 if (idxbuf[j].tag_seek[index_type] < 0)
2395 {
2396 logf("update error: %d/%ld", i+j, tcmh.tch.entry_count);
2397 error = true;
2398 goto error_exit;
2399 }
2400
2401 do_timed_yield();
2402 }
2403
2404 /* Write back the updated index. */
2405 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2406 index_entry_ec, tc_stat.econ) !=
2407 (int)sizeof(struct index_entry)*idxbuf_pos)
2408 {
2409 logf("write fail");
2410 error = true;
2411 goto error_exit;
2412 }
2413 }
2414 logf("done");
2415 }
2416
2417 /**
2418 * Walk through the temporary file containing the new tags.
2419 */
2420 // build_normal_index(h, tmpfd, masterfd, idx);
2421 logf("updating new indices...");
2422 lseek(masterfd, masterfd_pos, SEEK_SET);
2423 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2424 lseek(fd, 0, SEEK_END);
2425 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2426 {
2427 int j;
2428
2429 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2430 if (init)
2431 {
2432 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2433 }
2434 else
2435 {
2436 int loc = lseek(masterfd, 0, SEEK_CUR);
2437
2438 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2439 != (int)sizeof(struct index_entry)*idxbuf_pos)
2440 {
2441 logf("read fail #6");
2442 error = true;
2443 break ;
2444 }
2445 lseek(masterfd, loc, SEEK_SET);
2446 }
2447
2448 /* Read entry headers. */
2449 for (j = 0; j < idxbuf_pos; j++)
2450 {
2451 if (!tagcache_is_sorted_tag(index_type))
2452 {
2453 struct temp_file_entry entry;
2454 struct tagfile_entry fe;
2455
2456 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2457 sizeof(struct temp_file_entry))
2458 {
2459 logf("read fail #7");
2460 error = true;
2461 break ;
2462 }
2463
2464 /* Read data. */
2465 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2466 {
2467 logf("too long entry!");
2468 logf("length=%d", entry.tag_length[index_type]);
2469 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2470 error = true;
2471 break ;
2472 }
2473
2474 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2475 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2476 entry.tag_length[index_type])
2477 {
2478 logf("read fail #8");
2479 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2480 logf("length=0x%02x", entry.tag_length[index_type]);
2481 error = true;
2482 break ;
2483 }
2484
2485 /* Write to index file. */
2486 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2487 fe.tag_length = entry.tag_length[index_type];
2488 fe.idx_id = tcmh.tch.entry_count + i + j;
2489 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2490 write(fd, buf, fe.tag_length);
2491 tempbufidx++;
2492
2493 /* Skip to next. */
2494 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2495 entry.tag_length[index_type], SEEK_CUR);
2496 }
2497 else
2498 {
2499 /* Locate the correct entry from the sorted array. */
2500 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2501 if (idxbuf[j].tag_seek[index_type] < 0)
2502 {
2503 logf("entry not found (%d)", j);
2504 error = true;
2505 break ;
2506 }
2507 }
2508 }
2509
2510 /* Write index. */
2511 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2512 index_entry_ec, tc_stat.econ) !=
2513 (int)sizeof(struct index_entry)*idxbuf_pos)
2514 {
2515 logf("tagcache: write fail #4");
2516 error = true;
2517 break ;
2518 }
2519
2520 do_timed_yield();
2521 }
2522 logf("done");
2523
2524 /* Finally write the header. */
2525 tch.magic = TAGCACHE_MAGIC;
2526 tch.entry_count = tempbufidx;
2527 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2528 lseek(fd, 0, SEEK_SET);
2529 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2530
2531 if (index_type != tag_filename)
2532 h->datasize += tch.datasize;
2533 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2534 error_exit:
2535
2536 close(fd);
2537 close(masterfd);
2538
2539 if (error)
2540 return -2;
2541
2542 return 1;
2543 }
2544
2545 static bool commit(void)
2546 {
2547 struct tagcache_header tch;
2548 struct master_header tcmh;
2549 int i, len, rc;
2550 int tmpfd;
2551 int masterfd;
2552 #ifdef HAVE_DIRCACHE
2553 bool dircache_buffer_stolen = false;
2554 #endif
2555 bool local_allocation = false;
2556
2557 logf("committing tagcache");
2558
2559 while (write_lock)
2560 sleep(1);
2561
2562 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2563 if (tmpfd < 0)
2564 {
2565 logf("nothing to commit");
2566 return true;
2567 }
2568
2569
2570 /* Load the header. */
2571 len = sizeof(struct tagcache_header);
2572 rc = read(tmpfd, &tch, len);
2573
2574 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2575 {
2576 logf("incorrect header");
2577 close(tmpfd);
2578 remove(TAGCACHE_FILE_TEMP);
2579 return false;
2580 }
2581
2582 if (tch.entry_count == 0)
2583 {
2584 logf("nothing to commit");
2585 close(tmpfd);
2586 remove(TAGCACHE_FILE_TEMP);
2587 return true;
2588 }
2589
2590 #ifdef HAVE_EEPROM_SETTINGS
2591 remove(TAGCACHE_STATEFILE);
2592 #endif
2593
2594 /* At first be sure to unload the ramcache! */
2595 #ifdef HAVE_TC_RAMCACHE
2596 tc_stat.ramcache = false;
2597 #endif
2598
2599 read_lock++;
2600
2601 /* Try to steal every buffer we can :) */
2602 if (tempbuf_size == 0)
2603 local_allocation = true;
2604
2605 #ifdef HAVE_DIRCACHE
2606 if (tempbuf_size == 0)
2607 {
2608 /* Try to steal the dircache buffer. */
2609 tempbuf = dircache_steal_buffer(&tempbuf_size);
2610 tempbuf_size &= ~0x03;
2611
2612 if (tempbuf_size > 0)
2613 {
2614 dircache_buffer_stolen = true;
2615 }
2616 }
2617 #endif
2618
2619 #ifdef HAVE_TC_RAMCACHE
2620 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2621 {
2622 tempbuf = (char *)(hdr + 1);
2623 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2624 tempbuf_size &= ~0x03;
2625 }
2626 #endif
2627
2628 /* And finally fail if there are no buffers available. */
2629 if (tempbuf_size == 0)
2630 {
2631 logf("delaying commit until next boot");
2632 tc_stat.commit_delayed = true;
2633 close(tmpfd);
2634 read_lock--;
2635 return false;
2636 }
2637
2638 logf("commit %ld entries...", tch.entry_count);
2639
2640 /* Mark DB dirty so it will stay disabled if commit fails. */
2641 current_tcmh.dirty = true;
2642 update_master_header();
2643
2644 /* Now create the index files. */
2645 tc_stat.commit_step = 0;
2646 tch.datasize = 0;
2647 tc_stat.commit_delayed = false;
2648
2649 for (i = 0; i < TAG_COUNT; i++)
2650 {
2651 int ret;
2652
2653 if (tagcache_is_numeric_tag(i))
2654 continue;
2655
2656 tc_stat.commit_step++;
2657 ret = build_index(i, &tch, tmpfd);
2658 if (ret <= 0)
2659 {
2660 close(tmpfd);
2661 logf("tagcache failed init");
2662 if (ret < 0)
2663 remove_files();
2664 else
2665 tc_stat.commit_delayed = true;
2666 tc_stat.commit_step = 0;
2667 read_lock--;
2668 return false;
2669 }
2670 }
2671
2672 if (!build_numeric_indices(&tch, tmpfd))
2673 {
2674 logf("Failure to commit numeric indices");
2675 close(tmpfd);
2676 remove_files();
2677 tc_stat.commit_step = 0;
2678 read_lock--;
2679 return false;
2680 }
2681
2682 close(tmpfd);
2683 tc_stat.commit_step = 0;
2684
2685 /* Update the master index headers. */
2686 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2687 {
2688 read_lock--;
2689 return false;
2690 }
2691
2692 tcmh.tch.entry_count += tch.entry_count;
2693 tcmh.tch.datasize = sizeof(struct master_header)
2694 + sizeof(struct index_entry) * tcmh.tch.entry_count
2695 + tch.datasize;
2696 tcmh.dirty = false;
2697 tcmh.commitid++;
2698
2699 lseek(masterfd, 0, SEEK_SET);
2700 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2701 close(masterfd);
2702
2703 logf("tagcache committed");
2704 remove(TAGCACHE_FILE_TEMP);
2705 tc_stat.ready = check_all_headers();
2706 tc_stat.readyvalid = true;
2707
2708 if (local_allocation)
2709 {
2710 tempbuf = NULL;
2711 tempbuf_size = 0;
2712 }
2713
2714 #ifdef HAVE_DIRCACHE
2715 /* Rebuild the dircache, if we stole the buffer. */
2716 if (dircache_buffer_stolen)
2717 dircache_build(0);
2718 #endif
2719
2720 #ifdef HAVE_TC_RAMCACHE
2721 /* Reload tagcache. */
2722 if (tc_stat.ramcache_allocated > 0)
2723 tagcache_start_scan();
2724 #endif
2725
2726 read_lock--;
2727
2728 return true;
2729 }
2730
2731 static void allocate_tempbuf(void)
2732 {
2733 /* Yeah, malloc would be really nice now :) */
2734 #ifdef __PCTOOL__
2735 tempbuf_size = 32*1024*1024;
2736 tempbuf = malloc(tempbuf_size);
2737 #else
2738 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2739 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2740 audiobuf += tempbuf_size;
2741 #endif
2742 }
2743
2744 static void free_tempbuf(void)
2745 {
2746 if (tempbuf_size == 0)
2747 return ;
2748
2749 #ifdef __PCTOOL__
2750 free(tempbuf);
2751 #else
2752 audiobuf -= tempbuf_size;
2753 #endif
2754 tempbuf = NULL;
2755 tempbuf_size = 0;
2756 }
2757
2758 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2759 {
2760 struct index_entry idx;
2761
2762 if (!tc_stat.ready)
2763 return false;
2764
2765 if (!tagcache_is_numeric_tag(tag))
2766 return false;
2767
2768 if (!get_index(masterfd, idx_id, &idx, false))
2769 return false;
2770
2771 idx.tag_seek[tag] = data;
2772 idx.flag |= FLAG_DIRTYNUM;
2773
2774 return write_index(masterfd, idx_id, &idx);
2775 }
2776
2777 #if 0
2778 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
2779 int tag, long data)
2780 {
2781 struct master_header myhdr;
2782
2783 if (tcs->masterfd < 0)
2784 {
2785 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
2786 return false;
2787 }
2788
2789 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
2790 }
2791 #endif
2792
2793 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
2794
2795 static bool command_queue_is_full(void)
2796 {
2797 int next;
2798
2799 next = command_queue_widx + 1;
2800 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2801 next = 0;
2802
2803 return (next == command_queue_ridx);
2804 }
2805
2806 void run_command_queue(bool force)
2807 {
2808 struct master_header myhdr;
2809 int masterfd;
2810
2811 if (COMMAND_QUEUE_IS_EMPTY)
2812 return;
2813
2814 if (!force && !command_queue_is_full()
2815 && current_tick - TAGCACHE_COMMAND_QUEUE_COMMIT_DELAY
2816 < command_queue_timestamp)
2817 {
2818 return;
2819 }
2820
2821 mutex_lock(&command_queue_mutex);
2822
2823 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
2824 return;
2825
2826 while (command_queue_ridx != command_queue_widx)
2827 {
2828 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
2829
2830 switch (ce->command)
2831 {
2832 case CMD_UPDATE_MASTER_HEADER:
2833 {
2834 close(masterfd);
2835 update_master_header();
2836
2837 /* Re-open the masterfd. */
2838 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
2839 return;
2840
2841 break;
2842 }
2843 case CMD_UPDATE_NUMERIC:
2844 {
2845 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
2846 break;
2847 }
2848 }
2849
2850 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2851 command_queue_ridx = 0;
2852 }
2853
2854 close(masterfd);
2855
2856 mutex_unlock(&command_queue_mutex);
2857 }
2858
2859 static void queue_command(int cmd, long idx_id, int tag, long data)
2860 {
2861 while (1)
2862 {
2863 int next;
2864
2865 mutex_lock(&command_queue_mutex);
2866 next = command_queue_widx + 1;
2867 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
2868 next = 0;
2869
2870 /* Make sure queue is not full. */
2871 if (next != command_queue_ridx)
2872 {
2873 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
2874
2875 ce->command = cmd;
2876 ce->idx_id = idx_id;
2877 ce->tag = tag;
2878 ce->data = data;
2879
2880 command_queue_widx = next;
2881 command_queue_timestamp = current_tick;
2882 mutex_unlock(&command_queue_mutex);
2883 break;
2884 }
2885
2886 /* Queue is full, try again later... */
2887 mutex_unlock(&command_queue_mutex);
2888 sleep(1);
2889 }
2890 }
2891
2892 long tagcache_increase_serial(void)
2893 {
2894 long old;
2895
2896 if (!tc_stat.ready)
2897 return -2;
2898
2899 while (read_lock)
2900 sleep(1);
2901
2902 old = current_tcmh.serial++;
2903 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
2904
2905 return old;
2906 }
2907
2908 void tagcache_update_numeric(int idx_id, int tag, long data)
2909 {
2910 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
2911 }
2912
2913 long tagcache_get_serial(void)
2914 {
2915 return current_tcmh.serial;
2916 }
2917
2918 long tagcache_get_commitid(void)
2919 {
2920 return current_tcmh.commitid;
2921 }
2922
2923 static bool write_tag(int fd, const char *tagstr, const char *datastr)
2924 {
2925 char buf[512];
2926 int i;
2927
2928 snprintf(buf, sizeof buf, "%s=\"", tagstr);
2929 for (i = strlen(buf); i < (long)sizeof(buf)-3; i++)
2930 {
2931 if (*datastr == '\0')
2932 break;
2933
2934 if (*datastr == '"')
2935 {
2936 buf[i] = '\\';
2937 datastr++;
2938 continue;
2939 }
2940
2941 buf[i] = *(datastr++);
2942 }
2943
2944 strcpy(&buf[i], "\" ");
2945
2946 write(fd, buf, i + 2);
2947
2948 return true;
2949 }
2950
2951 static bool read_tag(char *dest, long size,
2952 const char *src, const char *tagstr)
2953 {
2954 int pos;
2955 char current_tag[32];
2956
2957 while (*src != '\0')
2958 {
2959 /* Skip all whitespace */
2960 while (*src == ' ')
2961 src++;
2962
2963 if (*src == '\0')
2964 break;
2965
2966 pos = 0;
2967 /* Read in tag name */
2968 while (*src != '=' && *src != ' ')
2969 {
2970 current_tag[pos] = *src;
2971 src++;
2972 pos++;
2973
2974 if (*src == '\0' || pos >= (long)sizeof(current_tag))
2975 return false;
2976 }
2977 current_tag[pos] = '\0';
2978
2979 /* Read in tag data */
2980
2981 /* Find the start. */
2982 while (*src != '"' && *src != '\0')
2983 src++;
2984
2985 if (*src == '\0' || *(++src) == '\0')
2986 return false;
2987
2988 /* Read the data. */
2989 for (pos = 0; pos < size; pos++)
2990 {
2991 if (*src == '\0')
2992 break;
2993
2994 if (*src == '\\' && *(src+1) == '"')
2995 {
2996 dest[pos] = '"';
2997 src += 2;
2998 continue;
2999 }
3000
3001 dest[pos] = *src;
3002
3003 if (*src == '"')
3004 {
3005 src++;
3006 break;
3007 }
3008
3009 if (*src == '\0')
3010 break;
3011
3012 src++;
3013 }
3014 dest[pos] = '\0';
3015
3016 if (!strcasecmp(tagstr, current_tag))
3017 return true;
3018 }
3019
3020 return false;
3021 }
3022
3023 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3024 {
3025 struct index_entry idx;
3026 char tag_data[TAG_MAXLEN+32];
3027 int idx_id;
3028 long masterfd = (long)parameters;
3029 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3030 tag_commitid };
3031 int i;
3032 (void)line_n;
3033
3034 if (*buf == '#')
3035 return 0;
3036
3037 logf("%d/%s", line_n, buf);
3038 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3039 {
3040 logf("filename missing");
3041 logf("-> %s", buf);
3042 return 0;
3043 }
3044
3045 idx_id = find_index(tag_data);
3046 if (idx_id < 0)
3047 {
3048 logf("entry not found");
3049 return 0;
3050 }
3051
3052 if (!get_index(masterfd, idx_id, &idx, false))
3053 {
3054 logf("failed to retrieve index entry");
3055 return 0;
3056 }
3057
3058 /* Stop if tag has already been modified. */
3059 if (idx.flag & FLAG_DIRTYNUM)
3060 return 0;
3061
3062 logf("import: %s", tag_data);
3063
3064 idx.flag |= FLAG_DIRTYNUM;
3065 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3066 {
3067 int data;
3068
3069 if (!read_tag(tag_data, sizeof tag_data, buf,
3070 tagcache_tag_to_str(import_tags[i])))
3071 {
3072 continue;
3073 }
3074
3075 data = atoi(tag_data);
3076 if (data < 0)
3077 continue;
3078
3079 idx.tag_seek[import_tags[i]] = data;
3080
3081 if (import_tags[i] == tag_lastplayed && data > current_tcmh.serial)
3082 current_tcmh.serial = data;
3083 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3084 current_tcmh.commitid = data + 1;
3085 }
3086
3087 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3088 }
3089
3090 #ifndef __PCTOOL__
3091 bool tagcache_import_changelog(void)
3092 {
3093 struct master_header myhdr;
3094 struct tagcache_header tch;
3095 int clfd;
3096 long masterfd;
3097 char buf[2048];
3098
3099 if (!tc_stat.ready)
3100 return false;
3101
3102 while (read_lock)
3103 sleep(1);
3104
3105 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3106 if (clfd < 0)
3107 {
3108 logf("failure to open changelog");
3109 return false;
3110 }
3111
3112 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3113 {
3114 close(clfd);
3115 return false;
3116 }
3117
3118 write_lock++;
3119
3120 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3121
3122 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3123 parse_changelog_line);
3124
3125 close(clfd);
3126 close(masterfd);
3127
3128 if (filenametag_fd >= 0)
3129 close(filenametag_fd);
3130
3131 write_lock--;
3132
3133 update_master_header();
3134
3135 return true;
3136 }
3137 #endif
3138
3139 bool tagcache_create_changelog(struct tagcache_search *tcs)
3140 {
3141 struct master_header myhdr;
3142 struct index_entry idx;
3143 char buf[TAG_MAXLEN+32];
3144 char temp[32];
3145 int clfd;
3146 int i, j;
3147
3148 if (!tc_stat.ready)
3149 return false;
3150
3151 if (!tagcache_search(tcs, tag_filename))
3152 return false;
3153
3154 /* Initialize the changelog */
3155 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3156 if (clfd < 0)
3157 {
3158 logf("failure to open changelog");
3159 return false;
3160 }
3161
3162 if (tcs->masterfd < 0)
3163 {
3164 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3165 return false;
3166 }
3167 else
3168 {
3169 lseek(tcs->masterfd, 0, SEEK_SET);
3170 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3171 }
3172
3173 write(clfd, "## Changelog version 1\n", 23);
3174
3175 for (i = 0; i < myhdr.tch.entry_count; i++)
3176 {
3177 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3178 != sizeof(struct index_entry))
3179 {
3180 logf("read error #9");
3181 tagcache_search_finish(tcs);
3182 close(clfd);
3183 return false;
3184 }
3185
3186 /* Skip until the entry found has been modified. */
3187 if (! (idx.flag & FLAG_DIRTYNUM) )
3188 continue;
3189
3190 /* Skip deleted entries too. */
3191 if (idx.flag & FLAG_DELETED)
3192 continue;
3193
3194 /* Now retrieve all tags. */
3195 for (j = 0; j < TAG_COUNT; j++)
3196 {
3197 if (tagcache_is_numeric_tag(j))
3198 {
3199 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3200 write_tag(clfd, tagcache_tag_to_str(j), temp);
3201 continue;
3202 }
3203
3204 tcs->type = j;
3205 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3206 write_tag(clfd, tagcache_tag_to_str(j), buf);
3207 }
3208
3209 write(clfd, "\n", 1);
3210 do_timed_yield();
3211 }
3212
3213 close(clfd);
3214
3215 tagcache_search_finish(tcs);
3216
3217 return true;
3218 }
3219
3220 static bool delete_entry(long idx_id)
3221 {
3222 int fd;
3223 int tag, i;
3224 struct index_entry idx, myidx;
3225 struct master_header myhdr;
3226 char buf[TAG_MAXLEN+32];
3227 int in_use[TAG_COUNT];
3228
3229 logf("delete_entry(): %ld", idx_id);
3230
3231 #ifdef HAVE_TC_RAMCACHE
3232 /* At first mark the entry removed from ram cache. */
3233 if (tc_stat.ramcache)
3234 hdr->indices[idx_id].flag |= FLAG_DELETED;
3235 #endif
3236
3237 if ( (fd = open_master_fd(&myhdr, true) ) < 0)
3238 return false;
3239
3240 lseek(fd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3241 if (ecread(fd, &myidx, 1, index_entry_ec, tc_stat.econ)
3242 != sizeof(struct index_entry))
3243 {
3244 logf("delete_entry(): read error");
3245 close(fd);
3246 return false;
3247 }
3248
3249 myidx.flag |= FLAG_DELETED;
3250 lseek(fd, -sizeof(struct index_entry), SEEK_CUR);
3251 if (ecwrite(fd, &myidx, 1, index_entry_ec, tc_stat.econ)
3252 != sizeof(struct index_entry))
3253 {
3254 logf("delete_entry(): write_error");
3255 close(fd);
3256 return false;
3257 }
3258
3259 /* Now check which tags are no longer in use (if any) */
3260 for (tag = 0; tag < TAG_COUNT; tag++)
3261 in_use[tag] = 0;
3262
3263 lseek(fd, sizeof(struct master_header), SEEK_SET);
3264 for (i = 0; i < myhdr.tch.entry_count; i++)
3265 {
3266 struct index_entry *idxp;
3267
3268 #ifdef HAVE_TC_RAMCACHE
3269 /* Use RAM DB if available for greater speed */
3270 if (tc_stat.ramcache)
3271 idxp = &hdr->indices[i];
3272 else
3273 #endif
3274 {
3275 if (ecread(fd, &idx, 1, index_entry_ec, tc_stat.econ)
3276 != sizeof(struct index_entry))
3277 {
3278 logf("delete_entry(): read error #2");
3279 close(fd);
3280 return false;
3281 }
3282 idxp = &idx;
3283 }
3284
3285 if (idxp->flag & FLAG_DELETED)
3286 continue;
3287
3288 for (tag = 0; tag < TAG_COUNT; tag++)
3289 {
3290 if (tagcache_is_numeric_tag(tag))
3291 continue;
3292
3293 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3294 in_use[tag]++;
3295 }
3296 }
3297
3298 close(fd);
3299
3300 /* Now delete all tags no longer in use. */
3301 for (tag = 0; tag < TAG_COUNT; tag++)
3302 {
3303 if (tagcache_is_numeric_tag(tag))
3304 continue;
3305
3306 if (in_use[tag])
3307 {
3308 logf("in use: %d/%d", tag, in_use[tag]);
3309 continue;
3310 }
3311
3312 #ifdef HAVE_TC_RAMCACHE
3313 /* Delete from ram. */
3314 if (tc_stat.ramcache && tag != tag_filename)
3315 {
3316 struct tagfile_entry *tagentry = get_tag(&myidx, tag);
3317 tagentry->tag_data[0] = '\0';
3318 }
3319 #endif
3320
3321 /* Open the index file, which contains the tag names. */
3322 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
3323 fd = open(buf, O_RDWR);
3324
3325 if (fd < 0)
3326 {
3327 logf("open failed");
3328 return false;
3329 }
3330
3331 /* Skip the header block */
3332 lseek(fd, myidx.tag_seek[tag] + sizeof(struct tagfile_entry), SEEK_SET);
3333
3334 /* Debug, print 10 first characters of the tag
3335 read(fd, buf, 10);
3336 buf[10]='\0';
3337 logf("TAG:%s", buf);
3338 lseek(fd, -10, SEEK_CUR);
3339 */
3340
3341 /* Write first data byte in tag as \0 */
3342 write(fd, "", 1);
3343
3344 /* Now tag data has been removed */
3345 close(fd);
3346 }
3347
3348 return true;
3349 }
3350
3351 #ifndef __PCTOOL__
3352 /**
3353 * Returns true if there is an event waiting in the queue
3354 * that requires the current operation to be aborted.
3355 */
3356 static bool check_event_queue(void)
3357 {
3358 struct event ev;
3359
3360 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3361 switch (ev.id)
3362 {
3363 case Q_STOP_SCAN:
3364 case SYS_POWEROFF:
3365 case SYS_USB_CONNECTED:
3366 /* Put the event back into the queue. */
3367 queue_post(&tagcache_queue, ev.id, ev.data);
3368 return true;
3369 }
3370
3371 return false;
3372 }
3373 #endif
3374
3375 #ifdef HAVE_TC_RAMCACHE
3376 static bool allocate_tagcache(void)
3377 {
3378 struct master_header tcmh;
3379 int fd;
3380
3381 /* Load the header. */
3382 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3383 {
3384 hdr = NULL;
3385 return false;
3386 }
3387
3388 close(fd);
3389
3390 /**
3391 * Now calculate the required cache size plus
3392 * some extra space for alignment fixes.
3393 */
3394 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3395 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3396 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3397 memset(hdr, 0, sizeof(struct ramcache_header));
3398 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3399 hdr->indices = (struct index_entry *)(hdr + 1);
3400 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3401
3402 return true;
3403 }
3404
3405 # ifdef HAVE_EEPROM_SETTINGS
3406 static bool tagcache_dumpload(void)
3407 {
3408 struct statefile_header shdr;
3409 int fd, rc;
3410 long offpos;
3411 int i;
3412
3413 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3414 if (fd < 0)
3415 {
3416 logf("no tagcache statedump");
3417 return false;
3418 }
3419
3420 /* Check the statefile memory placement */
3421 hdr = buffer_alloc(0);
3422 rc = read(fd, &shdr, sizeof(struct statefile_header));
3423 if (rc != sizeof(struct statefile_header)
3424 /* || (long)hdr != (long)shdr.hdr */)
3425 {
3426 logf("incorrect statefile");
3427 hdr = NULL;
3428 close(fd);
3429 return false;
3430 }
3431
3432 offpos = (long)hdr - (long)shdr.hdr;
3433
3434 /* Lets allocate real memory and load it */
3435 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3436 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3437 close(fd);
3438
3439 if (rc != shdr.tc_stat.ramcache_allocated)
3440 {
3441 logf("read failure!");
3442 hdr = NULL;
3443 return false;
3444 }
3445
3446 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3447
3448 /* Now fix the pointers */
3449 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3450 for (i = 0; i < TAG_COUNT; i++)
3451 hdr->tags[i] += offpos;
3452
3453 return true;
3454 }
3455
3456 static bool tagcache_dumpsave(void)
3457 {
3458 struct statefile_header shdr;
3459 int fd;
3460
3461 if (!tc_stat.ramcache)
3462 return false;
3463
3464 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3465 if (fd < 0)
3466 {
3467 logf("failed to create a statedump");
3468 return false;
3469 }
3470
3471 /* Create the header */
3472 shdr.hdr = hdr;
3473 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3474 write(fd, &shdr, sizeof(struct statefile_header));
3475
3476 /* And dump the data too */
3477 write(fd, hdr, tc_stat.ramcache_allocated);
3478 close(fd);
3479
3480 return true;
3481 }
3482 # endif
3483
3484 static bool load_tagcache(void)
3485 {
3486 struct tagcache_header *tch;
3487 long bytesleft = tc_stat.ramcache_allocated;
3488 struct index_entry *idx;
3489 int rc, fd;
3490 char *p;
3491 int i, tag;
3492
3493 # ifdef HAVE_DIRCACHE
3494 while (dircache_is_initializing())
3495 sleep(1);
3496 # endif
3497
3498 logf("loading tagcache to ram...");
3499
3500 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3501 if (fd < 0)
3502 {
3503 logf("tagcache open failed");
3504 return false;
3505 }
3506
3507 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3508 != sizeof(struct master_header)
3509 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3510 {
3511 logf("incorrect header");
3512 return false;
3513 }
3514
3515 idx = hdr->indices;
3516
3517 /* Load the master index table. */
3518 for (i = 0; i < hdr->h.tch.entry_count; i++)
3519 {
3520 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3521 if (rc != sizeof(struct index_entry))
3522 {
3523 logf("read error #10");
3524 close(fd);
3525 return false;
3526 }
3527
3528 bytesleft -= sizeof(struct index_entry);
3529 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3530 {
3531 logf("too big tagcache.");
3532 close(fd);
3533 return false;
3534 }
3535
3536 idx++;
3537 }
3538
3539 close(fd);
3540
3541 /* Load the tags. */
3542 p = (char *)idx;
3543 for (tag = 0; tag < TAG_COUNT; tag++)
3544 {
3545 struct tagfile_entry *fe;
3546 char buf[TAG_MAXLEN+32];
3547
3548 if (tagcache_is_numeric_tag(tag))
3549 continue ;
3550
3551 //p = ((void *)p+1);
3552 p = (char *)((long)p & ~0x03) + 0x04;
3553 hdr->tags[tag] = p;
3554
3555 /* Check the header. */
3556 tch = (struct tagcache_header *)p;
3557 p += sizeof(struct tagcache_header);
3558
3559 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3560 return false;
3561
3562 for (hdr->entry_count[tag] = 0;
3563 hdr->entry_count[tag] < tch->entry_count;
3564 hdr->entry_count[tag]++)
3565 {
3566 long pos;
3567
3568 if (do_timed_yield())
3569 {
3570 /* Abort if we got a critical event in queue */
3571 if (check_event_queue())
3572 return false;
3573 }
3574
3575 fe = (struct tagfile_entry *)p;
3576 pos = lseek(fd, 0, SEEK_CUR);
3577 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3578 if (rc != sizeof(struct tagfile_entry))
3579 {
3580 /* End of lookup table. */
3581 logf("read error #11");
3582 close(fd);
3583 return false;
3584 }
3585
3586 /* We have a special handling for the filename tags. */
3587 if (tag == tag_filename)
3588 {
3589 # ifdef HAVE_DIRCACHE
3590 const struct dirent *dc;
3591 # endif
3592
3593 // FIXME: This is wrong!
3594 // idx = &hdr->indices[hdr->entry_count[i]];
3595 idx = &hdr->indices[fe->idx_id];
3596
3597 if (fe->tag_length >= (long)sizeof(buf)-1)
3598 {
3599 read(fd, buf, 10);
3600 buf[10] = '\0';
3601 logf("TAG:%s", buf);
3602 logf("too long filename");
3603 close(fd);
3604 return false;
3605 }
3606
3607 rc = read(fd, buf, fe->tag_length);
3608 if (rc != fe->tag_length)
3609 {
3610 logf("read error #12");
3611 close(fd);
3612 return false;
3613 }
3614
3615 /* Check if the entry has already been removed */
3616 if (idx->flag & FLAG_DELETED)
3617 continue;
3618
3619 /* This flag must not be used yet. */
3620 if (idx->flag & FLAG_DIRCACHE)
3621 {
3622 logf("internal error!");
3623 close(fd);
3624 return false;
3625 }
3626
3627 if (idx->tag_seek[tag] != pos)
3628 {
3629 logf("corrupt data structures!");
3630 close(fd);
3631 return false;
3632 }
3633
3634 # ifdef HAVE_DIRCACHE
3635 if (dircache_is_enabled())
3636 {
3637 dc = dircache_get_entry_ptr(buf);
3638 if (dc == NULL)
3639 {
3640 logf("Entry no longer valid.");
3641 logf("-> %s", buf);
3642 delete_entry(fe->idx_id);
3643 continue ;
3644 }
3645
3646 idx->flag |= FLAG_DIRCACHE;
3647 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3648 idx->tag_seek[tag_filename] = (long)dc;
3649 }
3650 else
3651 # endif
3652 {
3653
3654 # if 0 /* Maybe we could enable this for flash players. Too slow otherwise. */
3655 /* Check if entry has been removed. */
3656 if (global_settings.tagcache_autoupdate)
3657 {
3658 int testfd;
3659
3660 testfd = open(buf, O_RDONLY);
3661 if (testfd < 0)
3662 {
3663 logf("Entry no longer valid.");
3664 logf("-> %s", buf);
3665 delete_entry(fe->idx_id);
3666 continue;
3667 }
3668 close(testfd);
3669 }
3670 # endif
3671 }
3672
3673 continue ;
3674 }
3675
3676 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3677 if (bytesleft < 0)
3678 {
3679 logf("too big tagcache #2");
3680 logf("tl: %d", fe->tag_length);
3681 logf("bl: %ld", bytesleft);
3682 close(fd);
3683 return false;
3684 }
3685
3686 p = fe->tag_data;
3687 rc = read(fd, fe->tag_data, fe->tag_length);
3688 p += rc;
3689
3690 if (rc != fe->tag_length)
3691 {
3692 logf("read error #13");
3693 logf("rc=0x%04x", rc); // 0x431
3694 logf("len=0x%04x", fe->tag_length); // 0x4000
3695 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
3696 logf("tag=0x%02x", tag); // 0x00
3697 close(fd);
3698 return false;
3699 }
3700 }
3701 close(fd);
3702 }
3703
3704 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
3705 logf("tagcache loaded into ram!");
3706
3707 return true;
3708 }
3709 #endif /* HAVE_TC_RAMCACHE */
3710
3711 static bool check_deleted_files(void)
3712 {
3713 int fd, testfd;
3714 char buf[TAG_MAXLEN+32];
3715 struct tagfile_entry tfe;
3716
3717 logf("reverse scan...");
3718 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
3719 fd = open(buf, O_RDONLY);
3720
3721 if (fd < 0)
3722 {
3723 logf("%s open fail", buf);
3724 return false;
3725 }
3726
3727 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
3728 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3729 == sizeof(struct tagfile_entry)
3730 #ifndef __PCTOOL__
3731 && !check_event_queue()
3732 #endif
3733 )
3734 {
3735 if (tfe.tag_length >= (long)sizeof(buf)-1)
3736 {
3737 logf("too long tag");
3738 close(fd);
3739 return false;
3740 }
3741
3742 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3743 {
3744 logf("read error #14");
3745 close(fd);
3746 return false;
3747 }
3748
3749 /* Check if the file has already deleted from the db. */
3750 if (*buf == '\0')
3751 continue;
3752
3753 /* Now check if the file exists. */
3754 testfd = open(buf, O_RDONLY);
3755 if (testfd < 0)
3756 {
3757 logf("Entry no longer valid.");
3758 logf("-> %s / %d", buf, tfe.tag_length);
3759 delete_entry(tfe.idx_id);
3760 }
3761 close(testfd);
3762 }
3763
3764 close(fd);
3765
3766 logf("done");
3767
3768 return true;
3769 }
3770
3771 static bool check_dir(const char *dirname)
3772 {
3773 DIR *dir;
3774 int len;
3775 int success = false;
3776
3777 dir = opendir(dirname);
3778 if (!dir)
3779 {
3780 logf("tagcache: opendir() failed");
3781 return false;
3782 }
3783
3784 /* Recursively scan the dir. */
3785 #ifdef __PCTOOL__
3786 while (1)
3787 #else
3788 while (!check_event_queue())
3789 #endif
3790 {
3791 struct dirent *entry;
3792
3793 entry = readdir(dir);
3794
3795 if (entry == NULL)
3796 {
3797 success = true;
3798 break ;
3799 }
3800
3801 if (!strcmp((char *)entry->d_name, ".") ||
3802 !strcmp((char *)entry->d_name, ".."))
3803 continue;
3804
3805 yield();
3806
3807 len = strlen(curpath);
3808 snprintf(&curpath[len], curpath_size - len, "/%s",
3809 entry->d_name);
3810
3811 processed_dir_count++;
3812 if (entry->attribute & ATTR_DIRECTORY)
3813 check_dir(curpath);
3814 else
3815 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
3816 add_tagcache(curpath, dir->internal_entry);
3817 #else
3818 add_tagcache(curpath);
3819 #endif
3820
3821 curpath[len] = '\0';
3822 }
3823
3824 closedir(dir);
3825
3826 return success;
3827 }
3828
3829 void build_tagcache(const char *path)
3830 {
3831 struct tagcache_header header;
3832 bool ret;
3833
3834 curpath[0] = '\0';
3835 data_size = 0;
3836 total_entry_count = 0;
3837 processed_dir_count = 0;
3838
3839 #ifdef HAVE_DIRCACHE
3840 while (dircache_is_initializing())
3841 sleep(1);
3842 #endif
3843
3844 logf("updating tagcache");
3845
3846 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
3847 if (cachefd >= 0)
3848 {
3849 logf("skipping, cache already waiting for commit");
3850 close(cachefd);
3851 return ;
3852 }
3853
3854 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
3855 if (cachefd < 0)
3856 {
3857 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
3858 return ;
3859 }
3860
3861 filenametag_fd = open_tag_fd(&header, tag_filename, false);
3862
3863 cpu_boost(true);
3864
3865 logf("Scanning files...");
3866 /* Scan for new files. */
3867 memset(&header, 0, sizeof(struct tagcache_header));
3868 write(cachefd, &header, sizeof(struct tagcache_header));
3869
3870 if (strcmp("/", path) != 0)
3871 strcpy(curpath, path);
3872 ret = check_dir(path);
3873
3874 /* Write the header. */
3875 header.magic = TAGCACHE_MAGIC;
3876 header.datasize = data_size;
3877 header.entry_count = total_entry_count;
3878 lseek(cachefd, 0, SEEK_SET);
3879 write(cachefd, &header, sizeof(struct tagcache_header));
3880 close(cachefd);
3881
3882 if (filenametag_fd >= 0)
3883 {
3884 close(filenametag_fd);
3885 filenametag_fd = -1;
3886 }
3887
3888 if (!ret)
3889 {
3890 logf("Aborted.");
3891 cpu_boost(false);
3892 return ;
3893 }
3894
3895 /* Commit changes to the database. */
3896 #ifdef __PCTOOL__
3897 allocate_tempbuf();
3898 #endif
3899 if (commit())
3900 {
3901 remove(TAGCACHE_FILE_TEMP);
3902 logf("tagcache built!");
3903 }
3904 #ifdef __PCTOOL__
3905 free_tempbuf();
3906 #endif
3907
3908 #ifdef HAVE_TC_RAMCACHE
3909 if (hdr)
3910 {
3911 /* Import runtime statistics if we just initialized the db. */
3912 if (hdr->h.serial == 0)
3913 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
3914 }
3915 #endif
3916
3917 cpu_boost(false);
3918 }
3919
3920 #ifdef HAVE_TC_RAMCACHE
3921 static void load_ramcache(void)
3922 {
3923 if (!hdr)
3924 return ;
3925
3926 cpu_boost(true);
3927
3928 /* At first we should load the cache (if exists). */
3929 tc_stat.ramcache = load_tagcache();
3930
3931 if (!tc_stat.ramcache)
3932 {
3933 /* If loading failed, it must indicate some problem with the db
3934 * so disable it entirely to prevent further issues. */
3935 tc_stat.ready = false;
3936 hdr = NULL;
3937 }
3938
3939 cpu_boost(false);
3940 }
3941
3942 void tagcache_unload_ramcache(void)
3943 {
3944 tc_stat.ramcache = false;
3945 /* Just to make sure there is no statefile present. */
3946 // remove(TAGCACHE_STATEFILE);
3947 }
3948 #endif
3949
3950 #ifndef __PCTOOL__
3951 static void tagcache_thread(void)
3952 {
3953 struct event ev;
3954 bool check_done = false;
3955
3956 /* If the previous cache build/update was interrupted, commit
3957 * the changes first in foreground. */
3958 cpu_boost(true);
3959 allocate_tempbuf();
3960 commit();
3961 free_tempbuf();
3962
3963 #ifdef HAVE_TC_RAMCACHE
3964 # ifdef HAVE_EEPROM_SETTINGS
3965 if (firmware_settings.initialized && firmware_settings.disk_clean)
3966 check_done = tagcache_dumpload();
3967
3968 remove(TAGCACHE_STATEFILE);
3969 # endif
3970
3971 /* Allocate space for the tagcache if found on disk. */
3972 if (global_settings.tagcache_ram && !tc_stat.ramcache)
3973 allocate_tagcache();
3974 #endif
3975
3976 cpu_boost(false);
3977 tc_stat.initialized = true;
3978
3979 /* Don't delay bootup with the header check but do it on background. */
3980 sleep(HZ);
3981 tc_stat.ready = check_all_headers();
3982 tc_stat.readyvalid = true;
3983
3984 while (1)
3985 {
3986 run_command_queue(false);
3987
3988 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
3989
3990 switch (ev.id)
3991 {
3992 case Q_IMPORT_CHANGELOG:
3993 tagcache_import_changelog();
3994 break;
3995
3996 case Q_REBUILD:
3997 remove_files();
3998 build_tagcache("/");
3999 break;
4000
4001 case Q_UPDATE:
4002 build_tagcache("/");
4003 #ifdef HAVE_TC_RAMCACHE
4004 load_ramcache();
4005 #endif
4006 check_deleted_files();
4007 break ;
4008
4009 case Q_START_SCAN:
4010 check_done = false;
4011 case SYS_TIMEOUT:
4012 if (check_done || !tc_stat.ready)
4013 break ;
4014
4015 #ifdef HAVE_TC_RAMCACHE
4016 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4017 {
4018 load_ramcache();
4019 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4020 build_tagcache("/");
4021 }
4022 else
4023 #endif
4024 if (global_settings.tagcache_autoupdate)
4025 {
4026 build_tagcache("/");
4027 /* Don't do auto removal without dircache (very slow). */
4028 #ifdef HAVE_DIRCACHE
4029 if (dircache_is_enabled())
4030 check_deleted_files();
4031 #endif
4032 }
4033
4034 logf("tagcache check done");
4035
4036 check_done = true;
4037 break ;
4038
4039 case Q_STOP_SCAN:
4040 break ;
4041
4042 case SYS_POWEROFF:
4043 break ;
4044
4045 #ifndef SIMULATOR
4046 case SYS_USB_CONNECTED:
4047 logf("USB: TagCache");
4048 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4049 usb_wait_for_disconnect(&tagcache_queue);
4050 break ;
4051 #endif
4052 }
4053 }
4054 }
4055
4056 bool tagcache_prepare_shutdown(void)
4057 {
4058 if (tagcache_get_commit_step() > 0)
4059 return false;
4060
4061 tagcache_stop_scan();
4062 while (read_lock || write_lock)
4063 sleep(1);
4064
4065 return true;
4066 }
4067
4068 void tagcache_shutdown(void)
4069 {
4070 /* Flush the command queue. */
4071 run_command_queue(true);
4072
4073 #ifdef HAVE_EEPROM_SETTINGS
4074 if (tc_stat.ramcache)
4075 tagcache_dumpsave();
4076 #endif
4077 }
4078
4079 static int get_progress(void)
4080 {
4081 int total_count = -1;
4082
4083 #ifdef HAVE_DIRCACHE
4084 if (dircache_is_enabled())
4085 {
4086 total_count = dircache_get_entry_count();
4087 }
4088 else
4089 #endif
4090 #ifdef HAVE_TC_RAMCACHE
4091 {
4092 if (hdr && tc_stat.ramcache)
4093 total_count = hdr->h.tch.entry_count;
4094 }
4095 #endif
4096
4097 if (total_count < 0)
4098 return -1;
4099
4100 return processed_dir_count * 100 / total_count;
4101 }
4102
4103 struct tagcache_stat* tagcache_get_stat(void)
4104 {
4105 tc_stat.progress = get_progress();
4106 tc_stat.processed_entries = processed_dir_count;
4107
4108 return &tc_stat;
4109 }
4110
4111 void tagcache_start_scan(void)
4112 {
4113 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4114 }
4115
4116 bool tagcache_update(void)
4117 {
4118 if (!tc_stat.ready)
4119 return false;
4120
4121 queue_post(&tagcache_queue, Q_UPDATE, 0);
4122 return false;
4123 }
4124
4125 bool tagcache_rebuild()
4126 {
4127 queue_post(&tagcache_queue, Q_REBUILD, 0);
4128 return false;
4129 }
4130
4131 void tagcache_stop_scan(void)
4132 {
4133 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4134 }
4135
4136 #ifdef HAVE_TC_RAMCACHE
4137 bool tagcache_is_ramcache(void)
4138 {
4139 return tc_stat.ramcache;
4140 }
4141 #endif
4142
4143 #endif /* !__PCTOOL__ */
4144
4145
4146 void tagcache_init(void)
4147 {
4148 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4149 memset(&current_tcmh, 0, sizeof(struct master_header));
4150 filenametag_fd = -1;
4151 write_lock = read_lock = 0;
4152
4153 #ifndef __PCTOOL__
4154 mutex_init(&command_queue_mutex);
4155 queue_init(&tagcache_queue, true);
4156 create_thread(tagcache_thread, tagcache_stack,
4157 sizeof(tagcache_stack), tagcache_thread_name
4158 IF_PRIO(, PRIORITY_BACKGROUND)
4159 IF_COP(, CPU, false));
4160 #else
4161 tc_stat.initialized = true;
4162 allocate_tempbuf();
4163 commit();
4164 free_tempbuf();
4165 tc_stat.ready = check_all_headers();
4166 #endif
4167 }
4168
4169 #ifdef __PCTOOL__
4170 void tagcache_reverse_scan(void)
4171 {
4172 logf("Checking for deleted files");
4173 check_deleted_files();
4174 }
4175 #endif
4176
4177 bool tagcache_is_initialized(void)
4178 {
4179 return tc_stat.initialized;
4180 }
4181 bool tagcache_is_usable(void)
4182 {
4183 return tc_stat.initialized && tc_stat.ready;
4184 }
4185 int tagcache_get_commit_step(void)
4186 {
4187 return tc_stat.commit_step;
4188 }
4189 int tagcache_get_max_commit_step(void)
4190 {
4191 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4192 }
4193
Open Source Jukebox Firmware