?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Update several codec Makefiles so that the codec libs build again on Coldfire targets...
[Rockbox.git] / apps / tagcache.c
blobd8684bd989ffc3084e1026daa0f5df1f41dc5dbe
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * All files in this archive are subject to the GNU General Public License.
13 * See the file COPYING in the source tree root for full license agreement.
14 *
15 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 * KIND, either express or implied.
17 *
18 ****************************************************************************/
19
20 /*
21 * TagCache API
22 *
23 * ----------x---------x------------------x-----
24 * | | | External
25 * +---------------x-------+ | TagCache | Libraries
26 * | Modification routines | | Core |
27 * +-x---------x-----------+ | |
28 * | (R/W) | | | |
29 * | +------x-------------x-+ +-------------x-----+ |
30 * | | x==x Filters & clauses | |
31 * | | Search routines | +-------------------+ |
32 * | | x============================x DirCache
33 * | +-x--------------------+ | (optional)
34 * | | (R) |
35 * | | +-------------------------------+ +---------+ |
36 * | | | DB Commit (sort,unique,index) | | | |
37 * | | +-x--------------------------x--+ | Control | |
38 * | | | (R/W) | (R) | Thread | |
39 * | | | +----------------------+ | | | |
40 * | | | | TagCache DB Builder | | +---------+ |
41 * | | | +-x-------------x------+ | |
42 * | | | | (R) | (W) | |
43 * | | | | +--x--------x---------+ |
44 * | | | | | Temporary Commit DB | |
45 * | | | | +---------------------+ |
46 * +-x----x-------x--+ |
47 * | TagCache RAM DB x==\(W) +-----------------+ |
48 * +-----------------+ \===x | |
49 * | | | | (R) | Ram DB Loader x============x DirCache
50 * +-x----x---x---x---+ /==x | | (optional)
51 * | Tagcache Disk DB x==/ +-----------------+ |
52 * +------------------+ |
53 *
54 */
55
56 #include <stdio.h>
57 #include "config.h"
58 #include "thread.h"
59 #include "kernel.h"
60 #include "system.h"
61 #include "logf.h"
62 #include "string.h"
63 #include "usb.h"
64 #include "dircache.h"
65 #include "metadata.h"
66 #include "id3.h"
67 #include "settings.h"
68 #include "splash.h"
69 #include "lang.h"
70 #include "tagcache.h"
71 #include "buffer.h"
72 #include "atoi.h"
73 #include "crc32.h"
74 #include "eeprom_settings.h"
75 #include "misc.h"
76
77 /* Tag Cache thread. */
78 static struct event_queue tagcache_queue;
79 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
80 static const char tagcache_thread_name[] = "tagcache";
81
82 /* Previous path when scanning directory tree recursively. */
83 static char curpath[MAX_PATH*2];
84 static long curpath_size = sizeof(curpath);
85
86 /* Used when removing duplicates. */
87 static char *tempbuf; /* Allocated when needed. */
88 static long tempbufidx; /* Current location in buffer. */
89 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
90 static long tempbuf_left; /* Buffer space left. */
91 static long tempbuf_pos;
92
93 /* Tags we want to get sorted (loaded to the tempbuf). */
94 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre, tag_composer, tag_title };
95
96 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
97 static const int unique_tags[] = { tag_artist, tag_album, tag_genre, tag_composer };
98
99 /* Numeric tags (we can use these tags with conditional clauses). */
100 static const int numeric_tags[] = { tag_year, tag_tracknumber, tag_length, tag_bitrate,
101 tag_playcount, tag_playtime, tag_lastplayed, tag_virt_autoscore };
102
103 static const char *tags_str[] = { "artist", "album", "genre", "title",
104 "filename", "composer", "year", "tracknumber", "bitrate", "length",
105 "playcount", "playtime", "lastplayed" };
106
107 /* Status information of the tagcache. */
108 static struct tagcache_stat stat;
109
110 /* Queue commands. */
111 enum tagcache_queue {
112 Q_STOP_SCAN = 0,
113 Q_START_SCAN,
114 Q_IMPORT_CHANGELOG,
115 Q_UPDATE,
116 Q_REBUILD,
117 };
118
119
120 /* Tag database structures. */
121
122 /* Variable-length tag entry in tag files. */
123 struct tagfile_entry {
124 short tag_length; /* Length of the data in bytes including '\0' */
125 short idx_id; /* Corresponding entry location in index file of not unique tags */
126 char tag_data[0]; /* Begin of the tag data */
127 };
128
129 /* Fixed-size tag entry in master db index. */
130 struct index_entry {
131 long tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
132 long flag; /* Status flags */
133 };
134
135 /* Header is the same in every file. */
136 struct tagcache_header {
137 long magic; /* Header version number */
138 long datasize; /* Data size in bytes */
139 long entry_count; /* Number of entries in this file */
140 };
141
142 struct master_header {
143 struct tagcache_header tch;
144 long serial; /* Increasing counting number */
145 };
146
147 static long current_serial;
148
149 #ifdef HAVE_TC_RAMCACHE
150 /* Header is created when loading database to ram. */
151 struct ramcache_header {
152 struct master_header h; /* Header from the master index */
153 struct index_entry *indices; /* Master index file content */
154 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
155 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
156 };
157
158 # ifdef HAVE_EEPROM_SETTINGS
159 struct statefile_header {
160 struct ramcache_header *hdr;
161 struct tagcache_stat stat;
162 };
163 # endif
164
165 /* Pointer to allocated ramcache_header */
166 static struct ramcache_header *hdr;
167 #endif
168
169 /**
170 * Full tag entries stored in a temporary file waiting
171 * for commit to the cache. */
172 struct temp_file_entry {
173 long tag_offset[TAG_COUNT];
174 short tag_length[TAG_COUNT];
175 long flag;
176
177 long data_length;
178 };
179
180 struct tempbuf_id_list {
181 long id;
182 struct tempbuf_id_list *next;
183 };
184
185 struct tempbuf_searchidx {
186 long idx_id;
187 char *str;
188 int seek;
189 struct tempbuf_id_list idlist;
190 };
191
192 /* Lookup buffer for fixing messed up index while after sorting. */
193 static long commit_entry_count;
194 static long lookup_buffer_depth;
195 struct tempbuf_searchidx **lookup;
196
197 /* Used when building the temporary file. */
198 static int cachefd = -1, filenametag_fd;
199 static int total_entry_count = 0;
200 static int data_size = 0;
201 static int processed_dir_count;
202
203 /* Thread safe locking */
204 static volatile int write_lock;
205 static volatile int read_lock;
206
207 int tagcache_str_to_tag(const char *str)
208 {
209 int i;
210
211 for (i = 0; i < (long)(sizeof(tags_str)/sizeof(tags_str[0])); i++)
212 {
213 if (!strcasecmp(tags_str[i], str))
214 return i;
215 }
216
217 return -1;
218 }
219
220 const char* tagcache_tag_to_str(int tag)
221 {
222 return tags_str[tag];
223 }
224
225 bool tagcache_is_numeric_tag(int type)
226 {
227 int i;
228
229 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
230 {
231 if (type == numeric_tags[i])
232 return true;
233 }
234
235 return false;
236 }
237
238 bool tagcache_is_unique_tag(int type)
239 {
240 int i;
241
242 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
243 {
244 if (type == unique_tags[i])
245 return true;
246 }
247
248 return false;
249 }
250
251 bool tagcache_is_sorted_tag(int type)
252 {
253 int i;
254
255 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
256 {
257 if (type == sorted_tags[i])
258 return true;
259 }
260
261 return false;
262 }
263
264 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
265 {
266 int fd;
267 char buf[MAX_PATH];
268 int rc;
269
270 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
271 return -1;
272
273 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
274
275 fd = open(buf, write ? O_RDWR : O_RDONLY);
276 if (fd < 0)
277 {
278 logf("tag file open failed: %d", tag);
279 stat.ready = false;
280 return fd;
281 }
282
283 /* Check the header. */
284 rc = read(fd, hdr, sizeof(struct tagcache_header));
285 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
286 {
287 logf("header error");
288 stat.ready = false;
289 close(fd);
290 return -2;
291 }
292
293 return fd;
294 }
295
296 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
297 static long find_entry_ram(const char *filename,
298 const struct dircache_entry *dc)
299 {
300 static long last_pos = 0;
301 int counter = 0;
302 int i;
303
304 /* Check if we tagcache is loaded into ram. */
305 if (!stat.ramcache)
306 return -1;
307
308 if (dc == NULL)
309 dc = dircache_get_entry_ptr(filename);
310
311 if (dc == NULL)
312 {
313 logf("tagcache: file not found.");
314 return -1;
315 }
316
317 try_again:
318
319 if (last_pos > 0)
320 i = last_pos;
321 else
322 i = 0;
323
324 for (; i < hdr->h.tch.entry_count; i++)
325 {
326 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
327 {
328 last_pos = MAX(0, i - 3);
329 return i;
330 }
331
332 if (++counter == 100)
333 {
334 yield();
335 counter = 0;
336 }
337 }
338
339 if (last_pos > 0)
340 {
341 last_pos = 0;
342 goto try_again;
343 }
344
345 return -1;
346 }
347 #endif
348
349 static long find_entry_disk(const char *filename)
350 {
351 struct tagcache_header tch;
352 static long last_pos = -1;
353 long pos_history[POS_HISTORY_COUNT];
354 long pos_history_idx = 0;
355 bool found = false;
356 struct tagfile_entry tfe;
357 int fd;
358 char buf[MAX_PATH];
359 int i;
360 int pos = -1;
361
362 if (!stat.ready)
363 return -2;
364
365 fd = filenametag_fd;
366 if (fd < 0)
367 {
368 last_pos = -1;
369 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
370 return -1;
371 }
372
373 check_again:
374
375 if (last_pos > 0)
376 lseek(fd, last_pos, SEEK_SET);
377 else
378 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
379
380 while (true)
381 {
382 pos = lseek(fd, 0, SEEK_CUR);
383 for (i = pos_history_idx-1; i >= 0; i--)
384 pos_history[i+1] = pos_history[i];
385 pos_history[0] = pos;
386
387 if (read(fd, &tfe, sizeof(struct tagfile_entry)) !=
388 sizeof(struct tagfile_entry))
389 {
390 break ;
391 }
392
393 if (tfe.tag_length >= (long)sizeof(buf))
394 {
395 logf("too long tag #1");
396 close(fd);
397 last_pos = -1;
398 return -2;
399 }
400
401 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
402 {
403 logf("read error #2");
404 close(fd);
405 last_pos = -1;
406 return -3;
407 }
408
409 if (!strcasecmp(filename, buf))
410 {
411 last_pos = pos_history[pos_history_idx];
412 found = true;
413 break ;
414 }
415
416 if (pos_history_idx < POS_HISTORY_COUNT - 1)
417 pos_history_idx++;
418 }
419
420 /* Not found? */
421 if (!found)
422 {
423 if (last_pos > 0)
424 {
425 last_pos = -1;
426 logf("seek again");
427 goto check_again;
428 }
429
430 if (fd != filenametag_fd)
431 close(fd);
432 return -4;
433 }
434
435 if (fd != filenametag_fd)
436 close(fd);
437
438 return tfe.idx_id;
439 }
440
441 static int find_index(const char *filename)
442 {
443 long idx_id = -1;
444
445 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
446 if (stat.ramcache && dircache_is_enabled())
447 idx_id = find_entry_ram(filename, NULL);
448 #endif
449
450 if (idx_id < 0)
451 idx_id = find_entry_disk(filename);
452
453 return idx_id;
454 }
455
456 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
457 {
458 int idx_id;
459
460 if (!stat.ready)
461 return false;
462
463 idx_id = find_index(filename);
464 if (idx_id < 0)
465 return false;
466
467 if (!tagcache_search(tcs, tag_filename))
468 return false;
469
470 tcs->entry_count = 0;
471 tcs->idx_id = idx_id;
472
473 return true;
474 }
475
476 static bool get_index(int masterfd, int idxid,
477 struct index_entry *idx, bool use_ram)
478 {
479 if (idxid < 0)
480 {
481 logf("Incorrect idxid: %d", idxid);
482 return false;
483 }
484
485 #ifdef HAVE_TC_RAMCACHE
486 if (stat.ramcache && use_ram)
487 {
488 if (hdr->indices[idxid].flag & FLAG_DELETED)
489 return false;
490
491 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
492 return true;
493 }
494 #else
495 (void)use_ram;
496 #endif
497
498 lseek(masterfd, idxid * sizeof(struct index_entry)
499 + sizeof(struct master_header), SEEK_SET);
500 if (read(masterfd, idx, sizeof(struct index_entry)) !=
501 sizeof(struct index_entry))
502 {
503 logf("read error #3");
504 return false;
505 }
506
507 if (idx->flag & FLAG_DELETED)
508 return false;
509
510 return true;
511 }
512
513 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
514 {
515 /* We need to exclude all memory only flags & tags when writing to disk. */
516 if (idx->flag & FLAG_DIRCACHE)
517 {
518 logf("memory only flags!");
519 return false;
520 }
521
522 #ifdef HAVE_TC_RAMCACHE
523 if (stat.ramcache)
524 {
525 memcpy(&hdr->indices[idxid], idx, sizeof(struct index_entry));
526 }
527 #endif
528
529 lseek(masterfd, idxid * sizeof(struct index_entry)
530 + sizeof(struct master_header), SEEK_SET);
531 if (write(masterfd, idx, sizeof(struct index_entry)) !=
532 sizeof(struct index_entry))
533 {
534 logf("write error #3");
535 logf("idxid: %d", idxid);
536 return false;
537 }
538
539 return true;
540 }
541
542 static bool open_files(struct tagcache_search *tcs, int tag)
543 {
544 if (tcs->idxfd[tag] < 0)
545 {
546 char fn[MAX_PATH];
547
548 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
549 tcs->idxfd[tag] = open(fn, O_RDONLY);
550 }
551
552 if (tcs->idxfd[tag] < 0)
553 {
554 logf("File not open!");
555 return false;
556 }
557
558 return true;
559 }
560
561 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
562 int tag, char *buf, long size)
563 {
564 struct tagfile_entry tfe;
565 long seek;
566
567 *buf = '\0';
568
569 if (tagcache_is_numeric_tag(tag))
570 return false;
571
572 seek = idx->tag_seek[tag];
573 if (seek < 0)
574 {
575 logf("Retrieve failed");
576 return false;
577 }
578
579 #ifdef HAVE_TC_RAMCACHE
580 if (tcs->ramsearch)
581 {
582 struct tagfile_entry *ep;
583
584 # ifdef HAVE_DIRCACHE
585 if (tag == tag_filename && idx->flag & FLAG_DIRCACHE)
586 {
587 dircache_copy_path((struct dircache_entry *)seek,
588 buf, size);
589 return true;
590 }
591 else
592 # endif
593 if (tag != tag_filename)
594 {
595 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
596 strncpy(buf, ep->tag_data, size-1);
597
598 return true;
599 }
600 }
601 #endif
602
603 if (!open_files(tcs, tag))
604 return false;
605
606 lseek(tcs->idxfd[tag], seek, SEEK_SET);
607 if (read(tcs->idxfd[tag], &tfe, sizeof(struct tagfile_entry)) !=
608 sizeof(struct tagfile_entry))
609 {
610 logf("read error #5");
611 return false;
612 }
613
614 if (tfe.tag_length >= size)
615 {
616 logf("too small buffer");
617 return false;
618 }
619
620 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
621 tfe.tag_length)
622 {
623 logf("read error #6");
624 return false;
625 }
626
627 buf[tfe.tag_length] = '\0';
628
629 return true;
630 }
631
632 static long check_virtual_tags(int tag, const struct index_entry *idx)
633 {
634 long data = 0;
635
636 switch (tag)
637 {
638 case tag_virt_autoscore:
639 if (idx->tag_seek[tag_length] == 0
640 || idx->tag_seek[tag_playcount] == 0)
641 {
642 data = 0;
643 }
644 else
645 {
646 data = 100 * idx->tag_seek[tag_playtime]
647 / idx->tag_seek[tag_length]
648 / idx->tag_seek[tag_playcount];
649 }
650 break;
651
652 default:
653 data = idx->tag_seek[tag];
654 }
655
656 return data;
657 }
658
659 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
660 {
661 struct index_entry idx;
662
663 if (!stat.ready)
664 return false;
665
666 if (!tagcache_is_numeric_tag(tag))
667 return -1;
668
669 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
670 return -2;
671
672 return check_virtual_tags(tag, &idx);
673 }
674
675 inline static bool str_ends_with(const char *str1, const char *str2)
676 {
677 int str_len = strlen(str1);
678 int clause_len = strlen(str2);
679
680 if (clause_len > str_len)
681 return false;
682
683 return !strcasecmp(&str1[str_len - clause_len], str2);
684 }
685
686 inline static bool str_oneof(const char *str, const char *list)
687 {
688 const char *sep;
689 int l, len = strlen(str);
690
691 while (*list)
692 {
693 sep = strchr(list, '|');
694 l = sep ? (long)sep - (long)list : (int)strlen(list);
695 if ((l==len) && !strncasecmp(str, list, len))
696 return true;
697 list += sep ? l + 1 : l;
698 }
699
700 return false;
701 }
702
703 static bool check_against_clause(long numeric, const char *str,
704 const struct tagcache_search_clause *clause)
705 {
706 if (clause->numeric)
707 {
708 switch (clause->type)
709 {
710 case clause_is:
711 return numeric == clause->numeric_data;
712 case clause_is_not:
713 return numeric != clause->numeric_data;
714 case clause_gt:
715 return numeric > clause->numeric_data;
716 case clause_gteq:
717 return numeric >= clause->numeric_data;
718 case clause_lt:
719 return numeric < clause->numeric_data;
720 case clause_lteq:
721 return numeric <= clause->numeric_data;
722 default:
723 logf("Incorrect numeric tag: %d", clause->type);
724 }
725 }
726 else
727 {
728 switch (clause->type)
729 {
730 case clause_is:
731 return !strcasecmp(clause->str, str);
732 case clause_is_not:
733 return strcasecmp(clause->str, str);
734 case clause_gt:
735 return 0>strcasecmp(clause->str, str);
736 case clause_gteq:
737 return 0>=strcasecmp(clause->str, str);
738 case clause_lt:
739 return 0<strcasecmp(clause->str, str);
740 case clause_lteq:
741 return 0<=strcasecmp(clause->str, str);
742 case clause_contains:
743 return (strcasestr(str, clause->str) != NULL);
744 case clause_not_contains:
745 return (strcasestr(str, clause->str) == NULL);
746 case clause_begins_with:
747 return (strcasestr(str, clause->str) == str);
748 case clause_not_begins_with:
749 return (strcasestr(str, clause->str) != str);
750 case clause_ends_with:
751 return str_ends_with(str, clause->str);
752 case clause_not_ends_with:
753 return !str_ends_with(str, clause->str);
754 case clause_oneof:
755 return str_oneof(str, clause->str);
756
757 default:
758 logf("Incorrect tag: %d", clause->type);
759 }
760 }
761
762 return false;
763 }
764
765 bool check_clauses(struct tagcache_search *tcs,
766 struct index_entry *idx,
767 struct tagcache_search_clause **clause, int count)
768 {
769 int i;
770
771 #ifdef HAVE_TC_RAMCACHE
772 if (tcs->ramsearch)
773 {
774 /* Go through all conditional clauses. */
775 for (i = 0; i < count; i++)
776 {
777 struct tagfile_entry *tfe;
778 int seek;
779 char buf[256];
780 char *str = NULL;
781
782 seek = check_virtual_tags(clause[i]->tag, idx);
783
784 if (!tagcache_is_numeric_tag(clause[i]->tag))
785 {
786 if (clause[i]->tag == tag_filename)
787 {
788 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
789 str = buf;
790 }
791 else
792 {
793 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
794 str = tfe->tag_data;
795 }
796 }
797
798 if (!check_against_clause(seek, str, clause[i]))
799 return false;
800 }
801 }
802 else
803 #endif
804 {
805 /* Check for conditions. */
806 for (i = 0; i < count; i++)
807 {
808 struct tagfile_entry tfe;
809 int seek;
810 char str[256];
811
812 seek = check_virtual_tags(clause[i]->tag, idx);
813
814 memset(str, 0, sizeof str);
815 if (!tagcache_is_numeric_tag(clause[i]->tag))
816 {
817 int fd = tcs->idxfd[clause[i]->tag];
818 lseek(fd, seek, SEEK_SET);
819 read(fd, &tfe, sizeof(struct tagfile_entry));
820 if (tfe.tag_length >= (int)sizeof(str))
821 {
822 logf("Too long tag read!");
823 break ;
824 }
825
826 read(fd, str, tfe.tag_length);
827
828 /* Check if entry has been deleted. */
829 if (str[0] == '\0')
830 break;
831 }
832
833 if (!check_against_clause(seek, str, clause[i]))
834 return false;
835 }
836 }
837
838 return true;
839 }
840
841 bool tagcache_check_clauses(struct tagcache_search *tcs,
842 struct tagcache_search_clause **clause, int count)
843 {
844 struct index_entry idx;
845
846 if (count == 0)
847 return true;
848
849 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
850 return false;
851
852 return check_clauses(tcs, &idx, clause, count);
853 }
854
855 static bool add_uniqbuf(struct tagcache_search *tcs, long id)
856 {
857 int i;
858
859 /* If uniq buffer is not defined we must return true for search to work. */
860 if (tcs->unique_list == NULL
861 || (!tagcache_is_unique_tag(tcs->type)
862 && !tagcache_is_numeric_tag(tcs->type)))
863 {
864 return true;
865 }
866
867 for (i = 0; i < tcs->unique_list_count; i++)
868 {
869 /* Return false if entry is found. */
870 if (tcs->unique_list[i] == id)
871 return false;
872 }
873
874 if (tcs->unique_list_count < tcs->unique_list_capacity)
875 {
876 tcs->unique_list[i] = id;
877 tcs->unique_list_count++;
878 }
879
880 return true;
881 }
882
883 static bool build_lookup_list(struct tagcache_search *tcs)
884 {
885 struct index_entry entry;
886 int i;
887
888 tcs->seek_list_count = 0;
889
890 #ifdef HAVE_TC_RAMCACHE
891 if (tcs->ramsearch)
892 {
893 int j;
894
895 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
896 {
897 struct index_entry *idx = &hdr->indices[i];
898 if (tcs->seek_list_count == SEEK_LIST_SIZE)
899 break ;
900
901 /* Skip deleted files. */
902 if (idx->flag & FLAG_DELETED)
903 continue;
904
905 /* Go through all filters.. */
906 for (j = 0; j < tcs->filter_count; j++)
907 {
908 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
909 {
910 break ;
911 }
912 }
913
914 if (j < tcs->filter_count)
915 continue ;
916
917 /* Check for conditions. */
918 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
919 continue;
920
921 /* Add to the seek list if not already in uniq buffer. */
922 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
923 continue;
924
925 /* Lets add it. */
926 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
927 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
928 tcs->seek_list_count++;
929 }
930
931 tcs->seek_pos = i;
932
933 return tcs->seek_list_count > 0;
934 }
935 #endif
936
937 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
938 sizeof(struct master_header), SEEK_SET);
939
940 while (read(tcs->masterfd, &entry, sizeof(struct index_entry)) ==
941 sizeof(struct index_entry))
942 {
943 /* Check if entry has been deleted. */
944 if (entry.flag & FLAG_DELETED)
945 continue;
946
947 if (tcs->seek_list_count == SEEK_LIST_SIZE)
948 break ;
949
950 /* Go through all filters.. */
951 for (i = 0; i < tcs->filter_count; i++)
952 {
953 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
954 break ;
955 }
956
957 tcs->seek_pos++;
958
959 if (i < tcs->filter_count)
960 continue ;
961
962 /* Check for conditions. */
963 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
964 continue;
965
966 /* Add to the seek list if not already in uniq buffer. */
967 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
968 continue;
969
970 /* Lets add it. */
971 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
972 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
973 tcs->seek_list_count++;
974
975 yield();
976 }
977
978 return tcs->seek_list_count > 0;
979 }
980
981
982 static void remove_files(void)
983 {
984 int i;
985 char buf[MAX_PATH];
986
987 stat.ready = false;
988 stat.ramcache = false;
989 remove(TAGCACHE_FILE_MASTER);
990 for (i = 0; i < TAG_COUNT; i++)
991 {
992 if (tagcache_is_numeric_tag(i))
993 continue;
994
995 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
996 remove(buf);
997 }
998 }
999
1000
1001 static int open_master_fd(struct master_header *hdr, bool write)
1002 {
1003 int fd;
1004 int rc;
1005
1006 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
1007 if (fd < 0)
1008 {
1009 logf("master file open failed for R/W");
1010 stat.ready = false;
1011 return fd;
1012 }
1013
1014 /* Check the header. */
1015 rc = read(fd, hdr, sizeof(struct master_header));
1016 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
1017 {
1018 logf("header error");
1019 stat.ready = false;
1020 close(fd);
1021 return -2;
1022 }
1023
1024 return fd;
1025 }
1026
1027 bool tagcache_search(struct tagcache_search *tcs, int tag)
1028 {
1029 struct tagcache_header tag_hdr;
1030 struct master_header master_hdr;
1031 int i;
1032
1033 if (tcs->initialized)
1034 tagcache_search_finish(tcs);
1035
1036 while (read_lock)
1037 sleep(1);
1038
1039 memset(tcs, 0, sizeof(struct tagcache_search));
1040 if (stat.commit_step > 0 || !stat.ready)
1041 return false;
1042
1043 tcs->position = sizeof(struct tagcache_header);
1044 tcs->type = tag;
1045 tcs->seek_pos = 0;
1046 tcs->seek_list_count = 0;
1047 tcs->filter_count = 0;
1048 tcs->masterfd = -1;
1049
1050 for (i = 0; i < TAG_COUNT; i++)
1051 tcs->idxfd[i] = -1;
1052
1053 #ifndef HAVE_TC_RAMCACHE
1054 tcs->ramsearch = false;
1055 #else
1056 tcs->ramsearch = stat.ramcache;
1057 if (tcs->ramsearch)
1058 {
1059 tcs->entry_count = hdr->entry_count[tcs->type];
1060 }
1061 else
1062 #endif
1063 {
1064 if (!tagcache_is_numeric_tag(tcs->type))
1065 {
1066 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1067 if (tcs->idxfd[tcs->type] < 0)
1068 return false;
1069 }
1070
1071 /* Always open as R/W so we can pass tcs to functions that modify data also
1072 * without failing. */
1073 tcs->masterfd = open_master_fd(&master_hdr, true);
1074
1075 if (tcs->masterfd < 0)
1076 return false;
1077 }
1078
1079 tcs->valid = true;
1080 tcs->initialized = true;
1081 write_lock++;
1082
1083 return true;
1084 }
1085
1086 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1087 void *buffer, long length)
1088 {
1089 tcs->unique_list = (unsigned long *)buffer;
1090 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1091 tcs->unique_list_count = 0;
1092 }
1093
1094 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1095 int tag, int seek)
1096 {
1097 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1098 return false;
1099
1100 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1101 return false;
1102
1103 tcs->filter_tag[tcs->filter_count] = tag;
1104 tcs->filter_seek[tcs->filter_count] = seek;
1105 tcs->filter_count++;
1106
1107 return true;
1108 }
1109
1110 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1111 struct tagcache_search_clause *clause)
1112 {
1113 int i;
1114
1115 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1116 {
1117 logf("Too many clauses");
1118 return false;
1119 }
1120
1121 /* Check if there is already a similar filter in present (filters are
1122 * much faster than clauses).
1123 */
1124 for (i = 0; i < tcs->filter_count; i++)
1125 {
1126 if (tcs->filter_tag[i] == clause->tag)
1127 return true;
1128 }
1129
1130 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1131 {
1132 char buf[MAX_PATH];
1133
1134 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1135 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1136 }
1137
1138 tcs->clause[tcs->clause_count] = clause;
1139 tcs->clause_count++;
1140
1141 return true;
1142 }
1143
1144 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1145 ? (flag & FLAG_DIRCACHE) : 1)
1146
1147 static bool get_next(struct tagcache_search *tcs)
1148 {
1149 static char buf[MAX_PATH];
1150 struct tagfile_entry entry;
1151 long flag = 0;
1152
1153 if (!tcs->valid || !stat.ready)
1154 return false;
1155
1156 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1157 #ifdef HAVE_TC_RAMCACHE
1158 && !tcs->ramsearch
1159 #endif
1160 )
1161 return false;
1162
1163 /* Relative fetch. */
1164 if (tcs->filter_count > 0 || tcs->clause_count > 0
1165 || tagcache_is_numeric_tag(tcs->type))
1166 {
1167 /* Check for end of list. */
1168 if (tcs->seek_list_count == 0)
1169 {
1170 /* Try to fetch more. */
1171 if (!build_lookup_list(tcs))
1172 {
1173 tcs->valid = false;
1174 return false;
1175 }
1176 }
1177
1178 tcs->seek_list_count--;
1179 flag = tcs->seek_flags[tcs->seek_list_count];
1180
1181 /* Seek stream to the correct position and continue to direct fetch. */
1182 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1183 && !tagcache_is_numeric_tag(tcs->type))
1184 {
1185 if (!open_files(tcs, tcs->type))
1186 return false;
1187
1188 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1189 }
1190 else
1191 tcs->position = tcs->seek_list[tcs->seek_list_count];
1192 }
1193
1194 if (tagcache_is_numeric_tag(tcs->type))
1195 {
1196 snprintf(buf, sizeof(buf), "%d", tcs->position);
1197 tcs->result_seek = tcs->position;
1198 tcs->result = buf;
1199 tcs->result_len = strlen(buf) + 1;
1200 return true;
1201 }
1202
1203 /* Direct fetch. */
1204 #ifdef HAVE_TC_RAMCACHE
1205 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1206 {
1207 struct tagfile_entry *ep;
1208
1209 if (tcs->entry_count == 0)
1210 {
1211 tcs->valid = false;
1212 return false;
1213 }
1214 tcs->entry_count--;
1215
1216 tcs->result_seek = tcs->position;
1217
1218 # ifdef HAVE_DIRCACHE
1219 if (tcs->type == tag_filename)
1220 {
1221 dircache_copy_path((struct dircache_entry *)tcs->position,
1222 buf, sizeof buf);
1223 tcs->result = buf;
1224 tcs->result_len = strlen(buf) + 1;
1225 tcs->idx_id = FLAG_GET_ATTR(flag);
1226 tcs->ramresult = false;
1227
1228 return true;
1229 }
1230 # endif
1231
1232 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1233 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1234 tcs->result = ep->tag_data;
1235 tcs->result_len = strlen(tcs->result) + 1;
1236 tcs->idx_id = ep->idx_id;
1237 tcs->ramresult = true;
1238
1239 return true;
1240 }
1241 else
1242 #endif
1243 {
1244 if (!open_files(tcs, tcs->type))
1245 return false;
1246
1247 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1248 if (read(tcs->idxfd[tcs->type], &entry, sizeof(struct tagfile_entry)) !=
1249 sizeof(struct tagfile_entry))
1250 {
1251 /* End of data. */
1252 tcs->valid = false;
1253 return false;
1254 }
1255 }
1256
1257 if (entry.tag_length > (long)sizeof(buf))
1258 {
1259 tcs->valid = false;
1260 logf("too long tag #2");
1261 return false;
1262 }
1263
1264 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1265 {
1266 tcs->valid = false;
1267 logf("read error #4");
1268 return false;
1269 }
1270
1271 tcs->result = buf;
1272 tcs->result_len = strlen(tcs->result) + 1;
1273 tcs->idx_id = entry.idx_id;
1274 tcs->ramresult = false;
1275
1276 return true;
1277 }
1278
1279 bool tagcache_get_next(struct tagcache_search *tcs)
1280 {
1281 while (get_next(tcs))
1282 {
1283 if (tcs->result_len > 1)
1284 return true;
1285 }
1286
1287 return false;
1288 }
1289
1290 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1291 int tag, char *buf, long size)
1292 {
1293 struct index_entry idx;
1294
1295 *buf = '\0';
1296 if (!get_index(tcs->masterfd, idxid, &idx, true))
1297 return false;
1298
1299 return retrieve(tcs, &idx, tag, buf, size);
1300 }
1301
1302 #if 0
1303
1304 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1305 {
1306 struct tagentry *entry;
1307
1308 if (tcs->type != tag_title)
1309 return ;
1310
1311 /* We will need reserve buffer for this. */
1312 if (tcs->ramcache)
1313 {
1314 struct tagfile_entry *ep;
1315
1316 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1317 tcs->seek_list[tcs->seek_list_count];
1318 }
1319
1320 entry = find_entry_ram();
1321
1322 }
1323 #endif
1324
1325 void tagcache_search_finish(struct tagcache_search *tcs)
1326 {
1327 int i;
1328
1329 if (!tcs->initialized)
1330 return;
1331
1332 if (tcs->masterfd >= 0)
1333 {
1334 close(tcs->masterfd);
1335 tcs->masterfd = -1;
1336 }
1337
1338 for (i = 0; i < TAG_COUNT; i++)
1339 {
1340 if (tcs->idxfd[i] >= 0)
1341 {
1342 close(tcs->idxfd[i]);
1343 tcs->idxfd[i] = -1;
1344 }
1345 }
1346
1347 tcs->ramsearch = false;
1348 tcs->valid = false;
1349 tcs->initialized = 0;
1350 if (write_lock > 0)
1351 write_lock--;
1352 }
1353
1354 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1355 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1356 {
1357 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1358 }
1359
1360 static long get_tag_numeric(const struct index_entry *entry, int tag)
1361 {
1362 return entry->tag_seek[tag];
1363 }
1364
1365 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1366 {
1367 struct index_entry *entry;
1368 int idx_id;
1369
1370 if (!stat.ready)
1371 return false;
1372
1373 /* Find the corresponding entry in tagcache. */
1374 idx_id = find_entry_ram(filename, NULL);
1375 if (idx_id < 0 || !stat.ramcache)
1376 return false;
1377
1378 entry = &hdr->indices[idx_id];
1379
1380 id3->title = get_tag(entry, tag_title)->tag_data;
1381 id3->artist = get_tag(entry, tag_artist)->tag_data;
1382 id3->album = get_tag(entry, tag_album)->tag_data;
1383 id3->genre_string = get_tag(entry, tag_genre)->tag_data;
1384 id3->composer = get_tag(entry, tag_composer)->tag_data;
1385 id3->year = get_tag_numeric(entry, tag_year);
1386 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1387 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1388 if (id3->bitrate == 0)
1389 id3->bitrate = 1;
1390
1391 return true;
1392 }
1393 #endif
1394
1395 static inline void write_item(const char *item)
1396 {
1397 int len = strlen(item) + 1;
1398
1399 data_size += len;
1400 write(cachefd, item, len);
1401 }
1402
1403 static int check_if_empty(char **tag)
1404 {
1405 int length;
1406
1407 if (*tag == NULL || *tag[0] == '\0')
1408 {
1409 *tag = "<Untagged>";
1410 return 11; /* Tag length */
1411 }
1412
1413 length = strlen(*tag);
1414 if (length >= MAX_PATH-32)
1415 {
1416 logf("over length tag: %s", *tag);
1417 *tag[MAX_PATH-32] = '\0';
1418 length = MAX_PATH-32;
1419 }
1420
1421 return length + 1;
1422 }
1423
1424 #define ADD_TAG(entry,tag,data) \
1425 /* Adding tag */ \
1426 entry.tag_offset[tag] = offset; \
1427 entry.tag_length[tag] = check_if_empty(data); \
1428 offset += entry.tag_length[tag]
1429
1430 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1431 static void add_tagcache(char *path, const struct dircache_entry *dc)
1432 #else
1433 static void add_tagcache(char *path)
1434 #endif
1435 {
1436 struct track_info track;
1437 struct temp_file_entry entry;
1438 bool ret;
1439 int fd;
1440 char tracknumfix[3];
1441 char *genrestr;
1442 int offset = 0;
1443
1444 if (cachefd < 0)
1445 return ;
1446
1447 /* Check if the file is supported. */
1448 if (probe_file_format(path) == AFMT_UNKNOWN)
1449 return ;
1450
1451 /* Check if the file is already cached. */
1452 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1453 if (stat.ramcache && dircache_is_enabled())
1454 {
1455 if (find_entry_ram(path, dc) >= 0)
1456 return ;
1457 }
1458 else
1459 #endif
1460 {
1461 if (filenametag_fd >= 0)
1462 {
1463 if (find_entry_disk(path) >= 0)
1464 return ;
1465 }
1466 }
1467
1468 fd = open(path, O_RDONLY);
1469 if (fd < 0)
1470 {
1471 logf("open fail: %s", path);
1472 return ;
1473 }
1474
1475 memset(&track, 0, sizeof(struct track_info));
1476 memset(&entry, 0, sizeof(struct temp_file_entry));
1477 memset(&tracknumfix, 0, sizeof(tracknumfix));
1478 ret = get_metadata(&track, fd, path, false);
1479 close(fd);
1480
1481 if (!ret)
1482 return ;
1483
1484 logf("-> %s", path);
1485
1486 /* Generate track number if missing. */
1487 if (track.id3.tracknum <= 0)
1488 {
1489 const char *p = strrchr(path, '.');
1490
1491 if (p == NULL)
1492 p = &path[strlen(path)-1];
1493
1494 while (*p != '/')
1495 {
1496 if (isdigit(*p) && isdigit(*(p-1)))
1497 {
1498 tracknumfix[1] = *p--;
1499 tracknumfix[0] = *p;
1500 break;
1501 }
1502 p--;
1503 }
1504
1505 if (tracknumfix[0] != '\0')
1506 {
1507 track.id3.tracknum = atoi(tracknumfix);
1508 /* Set a flag to indicate track number has been generated. */
1509 entry.flag |= FLAG_TRKNUMGEN;
1510 }
1511 else
1512 {
1513 /* Unable to generate track number. */
1514 track.id3.tracknum = -1;
1515 }
1516 }
1517
1518 genrestr = id3_get_genre(&track.id3);
1519
1520 /* Numeric tags */
1521 entry.tag_offset[tag_year] = track.id3.year;
1522 entry.tag_offset[tag_tracknumber] = track.id3.tracknum;
1523 entry.tag_offset[tag_length] = track.id3.length;
1524 entry.tag_offset[tag_bitrate] = track.id3.bitrate;
1525
1526 /* String tags. */
1527 ADD_TAG(entry, tag_filename, &path);
1528 ADD_TAG(entry, tag_title, &track.id3.title);
1529 ADD_TAG(entry, tag_artist, &track.id3.artist);
1530 ADD_TAG(entry, tag_album, &track.id3.album);
1531 ADD_TAG(entry, tag_genre, &genrestr);
1532 ADD_TAG(entry, tag_composer, &track.id3.composer);
1533 entry.data_length = offset;
1534
1535 /* Write the header */
1536 write(cachefd, &entry, sizeof(struct temp_file_entry));
1537
1538 /* And tags also... Correct order is critical */
1539 write_item(path);
1540 write_item(track.id3.title);
1541 write_item(track.id3.artist);
1542 write_item(track.id3.album);
1543 write_item(genrestr);
1544 write_item(track.id3.composer);
1545 total_entry_count++;
1546 }
1547
1548 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1549 {
1550 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1551 int len = strlen(str)+1;
1552 int i;
1553 unsigned crc32;
1554 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1555 char buf[MAX_PATH];
1556
1557 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1558 buf[i] = tolower(str[i]);
1559 buf[i] = '\0';
1560
1561 crc32 = crc_32(buf, i, 0xffffffff);
1562
1563 if (unique)
1564 {
1565 /* Check if the crc does not exist -> entry does not exist for sure. */
1566 for (i = 0; i < tempbufidx; i++)
1567 {
1568 if (crcbuf[-i] != crc32)
1569 continue;
1570
1571 if (!strcasecmp(str, index[i].str))
1572 {
1573 if (id < 0 || id >= lookup_buffer_depth)
1574 {
1575 logf("lookup buf overf.: %d", id);
1576 return false;
1577 }
1578
1579 lookup[id] = &index[i];
1580 return true;
1581 }
1582 }
1583 }
1584
1585 /* Insert to CRC buffer. */
1586 crcbuf[-tempbufidx] = crc32;
1587 tempbuf_left -= 4;
1588
1589 /* Insert it to the buffer. */
1590 tempbuf_left -= len;
1591 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1592 return false;
1593
1594 if (id >= lookup_buffer_depth)
1595 {
1596 logf("lookup buf overf. #2: %d", id);
1597 return false;
1598 }
1599
1600 if (id >= 0)
1601 {
1602 lookup[id] = &index[tempbufidx];
1603 index[tempbufidx].idlist.id = id;
1604 }
1605 else
1606 index[tempbufidx].idlist.id = -1;
1607
1608 index[tempbufidx].idlist.next = NULL;
1609 index[tempbufidx].idx_id = idx_id;
1610 index[tempbufidx].seek = -1;
1611 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1612 memcpy(index[tempbufidx].str, str, len);
1613 tempbuf_pos += len;
1614 tempbufidx++;
1615
1616 return true;
1617 }
1618
1619 static int compare(const void *p1, const void *p2)
1620 {
1621 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1622 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1623
1624 /*
1625 if (!strncasecmp("the ", e1, 4))
1626 e1 = &e1[4];
1627 if (!strncasecmp("the ", e2, 4))
1628 e2 = &e2[4];
1629 */
1630
1631 return strncasecmp(e1->str, e2->str, MAX_PATH);
1632 }
1633
1634 static int tempbuf_sort(int fd)
1635 {
1636 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1637 struct tagfile_entry fe;
1638 int i;
1639 int length;
1640
1641 /* Generate reverse lookup entries. */
1642 for (i = 0; i < lookup_buffer_depth; i++)
1643 {
1644 struct tempbuf_id_list *idlist;
1645
1646 if (!lookup[i])
1647 continue;
1648
1649 if (lookup[i]->idlist.id == i)
1650 continue;
1651
1652 idlist = &lookup[i]->idlist;
1653 while (idlist->next != NULL)
1654 idlist = idlist->next;
1655
1656 tempbuf_left -= sizeof(struct tempbuf_id_list);
1657 if (tempbuf_left - 4 < 0)
1658 return -1;
1659
1660 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1661 if (tempbuf_pos & 0x03)
1662 {
1663 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1664 tempbuf_left -= 3;
1665 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1666 }
1667 tempbuf_pos += sizeof(struct tempbuf_id_list);
1668
1669 idlist = idlist->next;
1670 idlist->id = i;
1671 idlist->next = NULL;
1672 }
1673
1674 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1675 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1676
1677 for (i = 0; i < tempbufidx; i++)
1678 {
1679 struct tempbuf_id_list *idlist = &index[i].idlist;
1680
1681 /* Fix the lookup list. */
1682 while (idlist != NULL)
1683 {
1684 if (idlist->id >= 0)
1685 lookup[idlist->id] = &index[i];
1686 idlist = idlist->next;
1687 }
1688
1689 index[i].seek = lseek(fd, 0, SEEK_CUR);
1690 length = strlen(index[i].str) + 1;
1691 fe.tag_length = length;
1692 fe.idx_id = index[i].idx_id;
1693
1694 /* Check the chunk alignment. */
1695 if ((fe.tag_length + sizeof(struct tagfile_entry))
1696 % TAGFILE_ENTRY_CHUNK_LENGTH)
1697 {
1698 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
1699 ((fe.tag_length + sizeof(struct tagfile_entry))
1700 % TAGFILE_ENTRY_CHUNK_LENGTH);
1701 }
1702
1703 #ifdef TAGCACHE_STRICT_ALIGN
1704 /* Make sure the entry is long aligned. */
1705 if (index[i].seek & 0x03)
1706 {
1707 logf("tempbuf_sort: alignment error!");
1708 return -3;
1709 }
1710 #endif
1711
1712 if (write(fd, &fe, sizeof(struct tagfile_entry)) !=
1713 sizeof(struct tagfile_entry))
1714 {
1715 logf("tempbuf_sort: write error #1");
1716 return -1;
1717 }
1718
1719 if (write(fd, index[i].str, length) != length)
1720 {
1721 logf("tempbuf_sort: write error #2");
1722 return -2;
1723 }
1724
1725 /* Write some padding. */
1726 if (fe.tag_length - length > 0)
1727 write(fd, "XXXXXXXX", fe.tag_length - length);
1728 }
1729
1730 return i;
1731 }
1732
1733 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
1734 {
1735 if (id < 0 || id >= lookup_buffer_depth)
1736 return NULL;
1737
1738 return lookup[id];
1739 }
1740
1741
1742 inline static int tempbuf_find_location(int id)
1743 {
1744 struct tempbuf_searchidx *entry;
1745
1746 entry = tempbuf_locate(id);
1747 if (entry == NULL)
1748 return -1;
1749
1750 return entry->seek;
1751 }
1752
1753 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
1754 {
1755 struct master_header tcmh;
1756 struct index_entry idx;
1757 int masterfd;
1758 int masterfd_pos;
1759 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
1760 int max_entries;
1761 int entries_processed = 0;
1762 int i, j;
1763
1764 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
1765
1766 logf("Building numeric indices...");
1767 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
1768
1769 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
1770 return false;
1771
1772 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
1773 SEEK_CUR);
1774 if (masterfd_pos == filesize(masterfd))
1775 {
1776 logf("we can't append!");
1777 close(masterfd);
1778 return false;
1779 }
1780
1781 while (entries_processed < h->entry_count)
1782 {
1783 int count = MIN(h->entry_count, max_entries);
1784
1785 /* Read in as many entries as possible. */
1786 for (i = 0; i < count; i++)
1787 {
1788 /* Read in numeric data. */
1789 if (read(tmpfd, &entrybuf[i], sizeof(struct temp_file_entry)) !=
1790 sizeof(struct temp_file_entry))
1791 {
1792 logf("read fail #1");
1793 close(masterfd);
1794 return false;
1795 }
1796
1797 /* Skip string data. */
1798 lseek(tmpfd, entrybuf[i].data_length, SEEK_CUR);
1799 }
1800
1801 /* Commit the data to the index. */
1802 for (i = 0; i < count; i++)
1803 {
1804 int loc = lseek(masterfd, 0, SEEK_CUR);
1805
1806 if (read(masterfd, &idx, sizeof(struct index_entry)) !=
1807 sizeof(struct index_entry))
1808 {
1809 logf("read fail #2");
1810 close(masterfd);
1811 return false;
1812 }
1813
1814 for (j = 0; j < TAG_COUNT; j++)
1815 {
1816 if (!tagcache_is_numeric_tag(j))
1817 continue;
1818
1819 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
1820 }
1821 idx.flag = entrybuf[i].flag;
1822
1823 /* Write back the updated index. */
1824 lseek(masterfd, loc, SEEK_SET);
1825 if (write(masterfd, &idx, sizeof(struct index_entry)) !=
1826 sizeof(struct index_entry))
1827 {
1828 logf("write fail");
1829 close(masterfd);
1830 return false;
1831 }
1832 }
1833
1834 entries_processed += count;
1835 logf("%d/%d entries processed", entries_processed, h->entry_count);
1836 }
1837
1838 close(masterfd);
1839
1840 return true;
1841 }
1842
1843 /**
1844 * Return values:
1845 * > 0 success
1846 * == 0 temporary failure
1847 * < 0 fatal error
1848 */
1849 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
1850 {
1851 int i;
1852 struct tagcache_header tch;
1853 struct master_header tcmh;
1854 struct index_entry idxbuf[IDX_BUF_DEPTH];
1855 int idxbuf_pos;
1856 char buf[MAX_PATH];
1857 int fd = -1, masterfd;
1858 bool error = false;
1859 int init;
1860 int masterfd_pos;
1861
1862 logf("Building index: %d", index_type);
1863
1864 /* Check the number of entries we need to allocate ram for. */
1865 commit_entry_count = h->entry_count + 1;
1866
1867 masterfd = open_master_fd(&tcmh, false);
1868 if (masterfd >= 0)
1869 {
1870 commit_entry_count += tcmh.tch.entry_count;
1871 close(masterfd);
1872 }
1873
1874 /* Open the index file, which contains the tag names. */
1875 fd = open_tag_fd(&tch, index_type, true);
1876 if (fd >= 0)
1877 {
1878 logf("tch.datasize=%d", tch.datasize);
1879 lookup_buffer_depth = 1 +
1880 /* First part */ commit_entry_count +
1881 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
1882 }
1883 else
1884 {
1885 lookup_buffer_depth = 1 +
1886 /* First part */ commit_entry_count +
1887 /* Second part */ 0;
1888 }
1889
1890 logf("lookup_buffer_depth=%d", lookup_buffer_depth);
1891 logf("commit_entry_count=%d", commit_entry_count);
1892
1893 /* Allocate buffer for all index entries from both old and new
1894 * tag files. */
1895 tempbufidx = 0;
1896 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
1897
1898 /* Allocate lookup buffer. The first portion of commit_entry_count
1899 * contains the new tags in the temporary file and the second
1900 * part for locating entries already in the db.
1901 *
1902 * New tags Old tags
1903 * +---------+---------------------------+
1904 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
1905 * +---------+---------------------------+
1906 *
1907 * Old tags are inserted to a temporary buffer with position:
1908 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
1909 * And new tags with index:
1910 * tempbuf_insert(idx, ...);
1911 *
1912 * The buffer is sorted and written into tag file:
1913 * tempbuf_sort(...);
1914 * leaving master index locations messed up.
1915 *
1916 * That is fixed using the lookup buffer for old tags:
1917 * new_seek = tempbuf_find_location(old_seek, ...);
1918 * and for new tags:
1919 * new_seek = tempbuf_find_location(idx);
1920 */
1921 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
1922 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
1923 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
1924
1925 /* And calculate the remaining data space used mainly for storing
1926 * tag data (strings). */
1927 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
1928 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
1929 {
1930 logf("Buffer way too small!");
1931 return 0;
1932 }
1933
1934 if (fd >= 0)
1935 {
1936 /**
1937 * If tag file contains unique tags (sorted index), we will load
1938 * it entirely into memory so we can resort it later for use with
1939 * chunked browsing.
1940 */
1941 if (tagcache_is_sorted_tag(index_type))
1942 {
1943 logf("loading tags...");
1944 for (i = 0; i < tch.entry_count; i++)
1945 {
1946 struct tagfile_entry entry;
1947 int loc = lseek(fd, 0, SEEK_CUR);
1948 bool ret;
1949
1950 if (read(fd, &entry, sizeof(struct tagfile_entry))
1951 != sizeof(struct tagfile_entry))
1952 {
1953 logf("read error #7");
1954 close(fd);
1955 return -2;
1956 }
1957
1958 if (entry.tag_length >= (int)sizeof(buf))
1959 {
1960 logf("too long tag #3");
1961 close(fd);
1962 return -2;
1963 }
1964
1965 if (read(fd, buf, entry.tag_length) != entry.tag_length)
1966 {
1967 logf("read error #8");
1968 close(fd);
1969 return -2;
1970 }
1971
1972 /* Skip deleted entries. */
1973 if (buf[0] == '\0')
1974 continue;
1975
1976 /**
1977 * Save the tag and tag id in the memory buffer. Tag id
1978 * is saved so we can later reindex the master lookup
1979 * table when the index gets resorted.
1980 */
1981 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
1982 + commit_entry_count, entry.idx_id,
1983 tagcache_is_unique_tag(index_type));
1984 if (!ret)
1985 {
1986 close(fd);
1987 return -3;
1988 }
1989 yield();
1990 }
1991 logf("done");
1992 }
1993 else
1994 tempbufidx = tch.entry_count;
1995 }
1996 else
1997 {
1998 /**
1999 * Creating new index file to store the tags. No need to preload
2000 * anything whether the index type is sorted or not.
2001 */
2002 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2003 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2004 if (fd < 0)
2005 {
2006 logf("%s open fail", buf);
2007 return -2;
2008 }
2009
2010 tch.magic = TAGCACHE_MAGIC;
2011 tch.entry_count = 0;
2012 tch.datasize = 0;
2013
2014 if (write(fd, &tch, sizeof(struct tagcache_header)) !=
2015 sizeof(struct tagcache_header))
2016 {
2017 logf("header write failed");
2018 close(fd);
2019 return -2;
2020 }
2021 }
2022
2023 /* Loading the tag lookup file as "master file". */
2024 logf("Loading index file");
2025 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2026
2027 if (masterfd < 0)
2028 {
2029 logf("Creating new index");
2030 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2031
2032 if (masterfd < 0)
2033 {
2034 logf("Failure to create index file");
2035 close(fd);
2036 return -2;
2037 }
2038
2039 /* Write the header (write real values later). */
2040 memset(&tcmh, 0, sizeof(struct master_header));
2041 tcmh.tch = *h;
2042 tcmh.tch.entry_count = 0;
2043 tcmh.tch.datasize = 0;
2044 write(masterfd, &tcmh, sizeof(struct master_header));
2045 init = true;
2046 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2047 current_serial = 0;
2048 }
2049 else
2050 {
2051 /**
2052 * Master file already exists so we need to process the current
2053 * file first.
2054 */
2055 init = false;
2056
2057 if (read(masterfd, &tcmh, sizeof(struct master_header)) !=
2058 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2059 {
2060 logf("header error");
2061 close(fd);
2062 close(masterfd);
2063 return -2;
2064 }
2065
2066 /**
2067 * If we reach end of the master file, we need to expand it to
2068 * hold new tags. If the current index is not sorted, we can
2069 * simply append new data to end of the file.
2070 * However, if the index is sorted, we need to update all tag
2071 * pointers in the master file for the current index.
2072 */
2073 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2074 SEEK_CUR);
2075 if (masterfd_pos == filesize(masterfd))
2076 {
2077 logf("appending...");
2078 init = true;
2079 }
2080 }
2081
2082 /**
2083 * Load new unique tags in memory to be sorted later and added
2084 * to the master lookup file.
2085 */
2086 if (tagcache_is_sorted_tag(index_type))
2087 {
2088 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2089 /* h is the header of the temporary file containing new tags. */
2090 logf("inserting new tags...");
2091 for (i = 0; i < h->entry_count; i++)
2092 {
2093 struct temp_file_entry entry;
2094
2095 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2096 sizeof(struct temp_file_entry))
2097 {
2098 logf("read fail #1");
2099 error = true;
2100 goto error_exit;
2101 }
2102
2103 /* Read data. */
2104 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2105 {
2106 logf("too long entry!");
2107 error = true;
2108 goto error_exit;
2109 }
2110
2111 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2112 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2113 entry.tag_length[index_type])
2114 {
2115 logf("read fail #3");
2116 error = true;
2117 goto error_exit;
2118 }
2119
2120 if (tagcache_is_unique_tag(index_type))
2121 error = !tempbuf_insert(buf, i, -1, true);
2122 else
2123 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2124
2125 if (error)
2126 {
2127 logf("insert error");
2128 goto error_exit;
2129 }
2130
2131 /* Skip to next. */
2132 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2133 entry.tag_length[index_type], SEEK_CUR);
2134 yield();
2135 }
2136 logf("done");
2137
2138 /* Sort the buffer data and write it to the index file. */
2139 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2140 i = tempbuf_sort(fd);
2141 if (i < 0)
2142 goto error_exit;
2143 logf("sorted %d tags", i);
2144
2145 /**
2146 * Now update all indexes in the master lookup file.
2147 */
2148 logf("updating indices...");
2149 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2150 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2151 {
2152 int j;
2153 int loc = lseek(masterfd, 0, SEEK_CUR);
2154
2155 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2156
2157 if (read(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
2158 (int)sizeof(struct index_entry)*idxbuf_pos)
2159 {
2160 logf("read fail #2");
2161 error = true;
2162 goto error_exit ;
2163 }
2164 lseek(masterfd, loc, SEEK_SET);
2165
2166 for (j = 0; j < idxbuf_pos; j++)
2167 {
2168 if (idxbuf[j].flag & FLAG_DELETED)
2169 {
2170 /* We can just ignore deleted entries. */
2171 idxbuf[j].tag_seek[index_type] = 0;
2172 continue;
2173 }
2174
2175 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2176 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2177 + commit_entry_count);
2178
2179 if (idxbuf[j].tag_seek[index_type] < 0)
2180 {
2181 logf("update error: %d/%d", i+j, tcmh.tch.entry_count);
2182 error = true;
2183 goto error_exit;
2184 }
2185
2186 yield();
2187 }
2188
2189 /* Write back the updated index. */
2190 if (write(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
2191 (int)sizeof(struct index_entry)*idxbuf_pos)
2192 {
2193 logf("write fail");
2194 error = true;
2195 goto error_exit;
2196 }
2197 }
2198 logf("done");
2199 }
2200
2201 /**
2202 * Walk through the temporary file containing the new tags.
2203 */
2204 // build_normal_index(h, tmpfd, masterfd, idx);
2205 logf("updating new indices...");
2206 lseek(masterfd, masterfd_pos, SEEK_SET);
2207 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2208 lseek(fd, 0, SEEK_END);
2209 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2210 {
2211 int j;
2212
2213 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2214 if (init)
2215 {
2216 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2217 }
2218 else
2219 {
2220 int loc = lseek(masterfd, 0, SEEK_CUR);
2221
2222 if (read(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
2223 (int)sizeof(struct index_entry)*idxbuf_pos)
2224 {
2225 logf("read fail #2");
2226 error = true;
2227 break ;
2228 }
2229 lseek(masterfd, loc, SEEK_SET);
2230 }
2231
2232 /* Read entry headers. */
2233 for (j = 0; j < idxbuf_pos; j++)
2234 {
2235 if (!tagcache_is_sorted_tag(index_type))
2236 {
2237 struct temp_file_entry entry;
2238 struct tagfile_entry fe;
2239
2240 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2241 sizeof(struct temp_file_entry))
2242 {
2243 logf("read fail #1");
2244 error = true;
2245 break ;
2246 }
2247
2248 /* Read data. */
2249 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2250 {
2251 logf("too long entry!");
2252 logf("length=%d", entry.tag_length[index_type]);
2253 logf("pos=0x%02x", lseek(tmpfd, 0, SEEK_CUR));
2254 error = true;
2255 break ;
2256 }
2257
2258 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2259 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2260 entry.tag_length[index_type])
2261 {
2262 logf("read fail #3");
2263 logf("offset=0x%02x", entry.tag_offset[index_type]);
2264 logf("length=0x%02x", entry.tag_length[index_type]);
2265 error = true;
2266 break ;
2267 }
2268
2269 /* Write to index file. */
2270 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2271 fe.tag_length = entry.tag_length[index_type];
2272 fe.idx_id = tcmh.tch.entry_count + i + j;
2273 write(fd, &fe, sizeof(struct tagfile_entry));
2274 write(fd, buf, fe.tag_length);
2275 tempbufidx++;
2276
2277 /* Skip to next. */
2278 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2279 entry.tag_length[index_type], SEEK_CUR);
2280 }
2281 else
2282 {
2283 /* Locate the correct entry from the sorted array. */
2284 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2285 if (idxbuf[j].tag_seek[index_type] < 0)
2286 {
2287 logf("entry not found (%d)");
2288 error = true;
2289 break ;
2290 }
2291 }
2292 }
2293
2294 /* Write index. */
2295 if (write(masterfd, idxbuf, sizeof(struct index_entry)*idxbuf_pos) !=
2296 (int)sizeof(struct index_entry)*idxbuf_pos)
2297 {
2298 logf("tagcache: write fail #4");
2299 error = true;
2300 break ;
2301 }
2302
2303 yield();
2304 }
2305 logf("done");
2306
2307 /* Finally write the header. */
2308 tch.magic = TAGCACHE_MAGIC;
2309 tch.entry_count = tempbufidx;
2310 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2311 lseek(fd, 0, SEEK_SET);
2312 write(fd, &tch, sizeof(struct tagcache_header));
2313
2314 if (index_type != tag_filename)
2315 h->datasize += tch.datasize;
2316 logf("s:%d/%d/%d", index_type, tch.datasize, h->datasize);
2317 error_exit:
2318
2319 close(fd);
2320 close(masterfd);
2321
2322 if (error)
2323 return -2;
2324
2325 return 1;
2326 }
2327
2328 static bool commit(void)
2329 {
2330 struct tagcache_header tch;
2331 struct master_header tcmh;
2332 int i, len, rc;
2333 int tmpfd;
2334 int masterfd;
2335 #ifdef HAVE_DIRCACHE
2336 bool dircache_buffer_stolen = false;
2337 #endif
2338 bool local_allocation = false;
2339
2340 logf("committing tagcache");
2341
2342 while (write_lock)
2343 sleep(1);
2344
2345 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2346 if (tmpfd < 0)
2347 {
2348 logf("nothing to commit");
2349 return true;
2350 }
2351
2352 /* Load the header. */
2353 len = sizeof(struct tagcache_header);
2354 rc = read(tmpfd, &tch, len);
2355
2356 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2357 {
2358 logf("incorrect header");
2359 close(tmpfd);
2360 remove(TAGCACHE_FILE_TEMP);
2361 return false;
2362 }
2363
2364 if (tch.entry_count == 0)
2365 {
2366 logf("nothing to commit");
2367 close(tmpfd);
2368 remove(TAGCACHE_FILE_TEMP);
2369 return true;
2370 }
2371
2372 #ifdef HAVE_EEPROM_SETTINGS
2373 remove(TAGCACHE_STATEFILE);
2374 #endif
2375
2376 /* At first be sure to unload the ramcache! */
2377 #ifdef HAVE_TC_RAMCACHE
2378 stat.ramcache = false;
2379 #endif
2380
2381 read_lock++;
2382
2383 /* Try to steal every buffer we can :) */
2384 if (tempbuf_size == 0)
2385 local_allocation = true;
2386
2387 #ifdef HAVE_DIRCACHE
2388 if (tempbuf_size == 0)
2389 {
2390 /* Try to steal the dircache buffer. */
2391 tempbuf = dircache_steal_buffer(&tempbuf_size);
2392 tempbuf_size &= ~0x03;
2393
2394 if (tempbuf_size > 0)
2395 {
2396 dircache_buffer_stolen = true;
2397 }
2398 }
2399 #endif
2400
2401 #ifdef HAVE_TC_RAMCACHE
2402 if (tempbuf_size == 0 && stat.ramcache_allocated > 0)
2403 {
2404 tempbuf = (char *)(hdr + 1);
2405 tempbuf_size = stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2406 tempbuf_size &= ~0x03;
2407 }
2408 #endif
2409
2410 /* And finally fail if there are no buffers available. */
2411 if (tempbuf_size == 0)
2412 {
2413 logf("delaying commit until next boot");
2414 stat.commit_delayed = true;
2415 close(tmpfd);
2416 read_lock--;
2417 return false;
2418 }
2419
2420 logf("commit %d entries...", tch.entry_count);
2421
2422 /* Now create the index files. */
2423 stat.commit_step = 0;
2424 tch.datasize = 0;
2425 stat.commit_delayed = false;
2426
2427 for (i = 0; i < TAG_COUNT; i++)
2428 {
2429 int ret;
2430
2431 if (tagcache_is_numeric_tag(i))
2432 continue;
2433
2434 stat.commit_step++;
2435 ret = build_index(i, &tch, tmpfd);
2436 if (ret <= 0)
2437 {
2438 close(tmpfd);
2439 logf("tagcache failed init");
2440 if (ret < 0)
2441 remove_files();
2442 else
2443 stat.commit_delayed = true;
2444 stat.commit_step = 0;
2445 read_lock--;
2446 return false;
2447 }
2448 }
2449
2450 build_numeric_indices(&tch, tmpfd);
2451 close(tmpfd);
2452 stat.commit_step = 0;
2453
2454 /* Update the master index headers. */
2455 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2456 {
2457 read_lock--;
2458 return false;
2459 }
2460
2461 tcmh.tch.entry_count += tch.entry_count;
2462 tcmh.tch.datasize = sizeof(struct master_header)
2463 + sizeof(struct index_entry) * tcmh.tch.entry_count
2464 + tch.datasize;
2465
2466 lseek(masterfd, 0, SEEK_SET);
2467 write(masterfd, &tcmh, sizeof(struct master_header));
2468 close(masterfd);
2469
2470 logf("tagcache committed");
2471 remove(TAGCACHE_FILE_TEMP);
2472 stat.ready = true;
2473
2474 if (local_allocation)
2475 {
2476 tempbuf = NULL;
2477 tempbuf_size = 0;
2478 }
2479
2480 #ifdef HAVE_DIRCACHE
2481 /* Rebuild the dircache, if we stole the buffer. */
2482 if (dircache_buffer_stolen)
2483 dircache_build(0);
2484 #endif
2485
2486 #ifdef HAVE_TC_RAMCACHE
2487 /* Reload tagcache. */
2488 if (stat.ramcache_allocated > 0)
2489 tagcache_start_scan();
2490 #endif
2491
2492 read_lock--;
2493
2494 return true;
2495 }
2496
2497 static void allocate_tempbuf(void)
2498 {
2499 /* Yeah, malloc would be really nice now :) */
2500 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2501 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2502 audiobuf += tempbuf_size;
2503 }
2504
2505 static void free_tempbuf(void)
2506 {
2507 if (tempbuf_size == 0)
2508 return ;
2509
2510 audiobuf -= tempbuf_size;
2511 tempbuf = NULL;
2512 tempbuf_size = 0;
2513 }
2514
2515 static bool update_current_serial(long serial)
2516 {
2517 struct master_header myhdr;
2518 int fd;
2519
2520 if ( (fd = open_master_fd(&myhdr, true)) < 0)
2521 return false;
2522
2523 myhdr.serial = serial;
2524 current_serial = serial;
2525
2526 #ifdef HAVE_TC_RAMCACHE
2527 if (hdr)
2528 hdr->h.serial = serial;
2529 #endif
2530
2531 /* Write it back */
2532 lseek(fd, 0, SEEK_SET);
2533 write(fd, &myhdr, sizeof(struct master_header));
2534 close(fd);
2535
2536 return true;
2537 }
2538
2539 long tagcache_increase_serial(void)
2540 {
2541 if (!stat.ready)
2542 return -2;
2543
2544 while (read_lock)
2545 sleep(1);
2546
2547 if (!update_current_serial(current_serial + 1))
2548 return -1;
2549
2550 return current_serial;
2551 }
2552
2553 long tagcache_get_serial(void)
2554 {
2555 return current_serial;
2556 }
2557
2558 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2559 {
2560 struct index_entry idx;
2561
2562 if (!stat.ready)
2563 return false;
2564
2565 if (!tagcache_is_numeric_tag(tag))
2566 return false;
2567
2568 if (!get_index(masterfd, idx_id, &idx, false))
2569 return false;
2570
2571 idx.tag_seek[tag] = data;
2572 idx.flag |= FLAG_DIRTYNUM;
2573
2574 return write_index(masterfd, idx_id, &idx);
2575 }
2576
2577 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
2578 int tag, long data)
2579 {
2580 struct master_header myhdr;
2581
2582 if (tcs->masterfd < 0)
2583 {
2584 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
2585 return false;
2586 }
2587
2588 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
2589 }
2590
2591 static bool write_tag(int fd, const char *tagstr, const char *datastr)
2592 {
2593 char buf[256];
2594 int i;
2595
2596 snprintf(buf, sizeof buf, "%s=\"", tagstr);
2597 for (i = strlen(buf); i < (long)sizeof(buf)-2; i++)
2598 {
2599 if (*datastr == '\0')
2600 break;
2601
2602 if (*datastr == '"')
2603 {
2604 buf[i] = '\\';
2605 datastr++;
2606 continue;
2607 }
2608
2609 buf[i] = *(datastr++);
2610 }
2611
2612 strcpy(&buf[i], "\" ");
2613
2614 write(fd, buf, i + 2);
2615
2616 return true;
2617 }
2618
2619 static bool read_tag(char *dest, long size,
2620 const char *src, const char *tagstr)
2621 {
2622 int pos;
2623 char current_tag[32];
2624
2625 while (*src != '\0')
2626 {
2627 /* Skip all whitespace */
2628 while (*src == ' ')
2629 src++;
2630
2631 if (*src == '\0')
2632 break;
2633
2634 pos = 0;
2635 /* Read in tag name */
2636 while (*src != '=' && *src != ' ')
2637 {
2638 current_tag[pos] = *src;
2639 src++;
2640 pos++;
2641
2642 if (*src == '\0' || pos >= (long)sizeof(current_tag))
2643 return false;
2644 }
2645 current_tag[pos] = '\0';
2646
2647 /* Read in tag data */
2648
2649 /* Find the start. */
2650 while (*src != '"' && *src != '\0')
2651 src++;
2652
2653 if (*src == '\0' || *(++src) == '\0')
2654 return false;
2655
2656 /* Read the data. */
2657 for (pos = 0; pos < size; pos++)
2658 {
2659 if (*src == '\0')
2660 break;
2661
2662 if (*src == '\\' && *(src+1) == '"')
2663 {
2664 dest[pos] = '"';
2665 src += 2;
2666 continue;
2667 }
2668
2669 dest[pos] = *src;
2670
2671 if (*src == '"')
2672 {
2673 src++;
2674 break;
2675 }
2676
2677 if (*src == '\0')
2678 break;
2679
2680 src++;
2681 }
2682 dest[pos] = '\0';
2683
2684 if (!strcasecmp(tagstr, current_tag))
2685 return true;
2686 }
2687
2688 return false;
2689 }
2690
2691 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
2692 {
2693 struct index_entry idx;
2694 char tag_data[MAX_PATH];
2695 int idx_id;
2696 long masterfd = (long)parameters;
2697 const int import_tags[] = { tag_playcount, tag_playtime, tag_lastplayed };
2698 int i;
2699 (void)line_n;
2700
2701 if (*buf == '#')
2702 return 0;
2703
2704 logf("%d/%s", line_n, buf);
2705 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
2706 {
2707 logf("filename missing");
2708 logf("-> %s", buf);
2709 return 0;
2710 }
2711
2712 idx_id = find_index(tag_data);
2713 if (idx_id < 0)
2714 {
2715 logf("entry not found");
2716 return 0;
2717 }
2718
2719 if (!get_index(masterfd, idx_id, &idx, false))
2720 {
2721 logf("failed to retrieve index entry");
2722 return 0;
2723 }
2724
2725 /* Stop if tag has already been modified. */
2726 if (idx.flag & FLAG_DIRTYNUM)
2727 return 0;
2728
2729 logf("import: %s", tag_data);
2730
2731 idx.flag |= FLAG_DIRTYNUM;
2732 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
2733 {
2734 int data;
2735
2736 if (!read_tag(tag_data, sizeof tag_data, buf,
2737 tagcache_tag_to_str(import_tags[i])))
2738 {
2739 continue;
2740 }
2741
2742 data = atoi(tag_data);
2743 if (data < 0)
2744 continue;
2745
2746 idx.tag_seek[import_tags[i]] = data;
2747
2748 if (import_tags[i] == tag_lastplayed && data > current_serial)
2749 current_serial = data;
2750 }
2751
2752 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
2753 }
2754
2755 bool tagcache_import_changelog(void)
2756 {
2757 struct master_header myhdr;
2758 struct tagcache_header tch;
2759 int clfd;
2760 long masterfd;
2761 char buf[2048];
2762
2763 if (!stat.ready)
2764 return false;
2765
2766 while (read_lock)
2767 sleep(1);
2768
2769 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
2770 if (clfd < 0)
2771 {
2772 logf("failure to open changelog");
2773 return false;
2774 }
2775
2776 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
2777 {
2778 close(clfd);
2779 return false;
2780 }
2781
2782 write_lock++;
2783
2784 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
2785
2786 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
2787 parse_changelog_line);
2788
2789 close(clfd);
2790 close(masterfd);
2791
2792 if (filenametag_fd >= 0)
2793 close(filenametag_fd);
2794
2795 write_lock--;
2796
2797 update_current_serial(current_serial);
2798
2799 return true;
2800 }
2801
2802 bool tagcache_create_changelog(struct tagcache_search *tcs)
2803 {
2804 struct master_header myhdr;
2805 struct index_entry idx;
2806 char buf[MAX_PATH];
2807 char temp[32];
2808 int clfd;
2809 int i, j;
2810
2811 if (!stat.ready)
2812 return false;
2813
2814 if (!tagcache_search(tcs, tag_filename))
2815 return false;
2816
2817 /* Initialize the changelog */
2818 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
2819 if (clfd < 0)
2820 {
2821 logf("failure to open changelog");
2822 return false;
2823 }
2824
2825 if (tcs->masterfd < 0)
2826 {
2827 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
2828 return false;
2829 }
2830 else
2831 {
2832 lseek(tcs->masterfd, 0, SEEK_SET);
2833 read(tcs->masterfd, &myhdr, sizeof(struct master_header));
2834 }
2835
2836 write(clfd, "## Changelog version 1\n", 23);
2837
2838 for (i = 0; i < myhdr.tch.entry_count; i++)
2839 {
2840 if (read(tcs->masterfd, &idx, sizeof(struct index_entry))
2841 != sizeof(struct index_entry))
2842 {
2843 logf("read error #9");
2844 tagcache_search_finish(tcs);
2845 close(clfd);
2846 return false;
2847 }
2848
2849 /* Skip until the entry found has been modified. */
2850 if (! (idx.flag & FLAG_DIRTYNUM) )
2851 continue;
2852
2853 /* Now retrieve all tags. */
2854 for (j = 0; j < TAG_COUNT; j++)
2855 {
2856 if (tagcache_is_numeric_tag(j))
2857 {
2858 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
2859 write_tag(clfd, tagcache_tag_to_str(j), temp);
2860 continue;
2861 }
2862
2863 tcs->type = j;
2864 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
2865 write_tag(clfd, tagcache_tag_to_str(j), buf);
2866 }
2867
2868 write(clfd, "\n", 1);
2869 yield();
2870 }
2871
2872 close(clfd);
2873
2874 tagcache_search_finish(tcs);
2875
2876 return true;
2877 }
2878
2879 static bool delete_entry(long idx_id)
2880 {
2881 int fd;
2882 int tag, i;
2883 struct index_entry idx, myidx;
2884 struct master_header myhdr;
2885 char buf[MAX_PATH];
2886 int in_use[TAG_COUNT];
2887
2888 logf("delete_entry(): %d", idx_id);
2889
2890 #ifdef HAVE_TC_RAMCACHE
2891 /* At first mark the entry removed from ram cache. */
2892 if (hdr)
2893 hdr->indices[idx_id].flag |= FLAG_DELETED;
2894 #endif
2895
2896 if ( (fd = open_master_fd(&myhdr, true) ) < 0)
2897 return false;
2898
2899 lseek(fd, idx_id * sizeof(struct index_entry), SEEK_CUR);
2900 if (read(fd, &myidx, sizeof(struct index_entry))
2901 != sizeof(struct index_entry))
2902 {
2903 logf("delete_entry(): read error");
2904 close(fd);
2905 return false;
2906 }
2907
2908 myidx.flag |= FLAG_DELETED;
2909 lseek(fd, -sizeof(struct index_entry), SEEK_CUR);
2910 if (write(fd, &myidx, sizeof(struct index_entry))
2911 != sizeof(struct index_entry))
2912 {
2913 logf("delete_entry(): write_error");
2914 close(fd);
2915 return false;
2916 }
2917
2918 /* Now check which tags are no longer in use (if any) */
2919 for (tag = 0; tag < TAG_COUNT; tag++)
2920 in_use[tag] = 0;
2921
2922 lseek(fd, sizeof(struct master_header), SEEK_SET);
2923 for (i = 0; i < myhdr.tch.entry_count; i++)
2924 {
2925 if (read(fd, &idx, sizeof(struct index_entry))
2926 != sizeof(struct index_entry))
2927 {
2928 logf("delete_entry(): read error #2");
2929 close(fd);
2930 return false;
2931 }
2932
2933 if (idx.flag & FLAG_DELETED)
2934 continue;
2935
2936 for (tag = 0; tag < TAG_COUNT; tag++)
2937 {
2938 if (tagcache_is_numeric_tag(tag))
2939 continue;
2940
2941 if (idx.tag_seek[tag] == myidx.tag_seek[tag])
2942 in_use[tag]++;
2943 }
2944 }
2945
2946 close(fd);
2947
2948 /* Now delete all tags no longer in use. */
2949 for (tag = 0; tag < TAG_COUNT; tag++)
2950 {
2951 if (tagcache_is_numeric_tag(tag))
2952 continue;
2953
2954 if (in_use[tag])
2955 {
2956 logf("in use: %d/%d", tag, in_use[tag]);
2957 continue;
2958 }
2959
2960 /* Open the index file, which contains the tag names. */
2961 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
2962 fd = open(buf, O_RDWR);
2963
2964 if (fd < 0)
2965 {
2966 logf("open failed");
2967 return false;
2968 }
2969
2970 /* Skip the header block */
2971 lseek(fd, myidx.tag_seek[tag] + sizeof(struct tagfile_entry), SEEK_SET);
2972
2973 /* Debug, print 10 first characters of the tag
2974 read(fd, buf, 10);
2975 buf[10]='\0';
2976 logf("TAG:%s", buf);
2977 lseek(fd, -10, SEEK_CUR);
2978 */
2979
2980 /* Write first data byte in tag as \0 */
2981 write(fd, "", 1);
2982
2983 /* Now tag data has been removed */
2984 close(fd);
2985 }
2986
2987 return true;
2988 }
2989
2990 /**
2991 * Returns true if there is an event waiting in the queue
2992 * that requires the current operation to be aborted.
2993 */
2994 static bool check_event_queue(void)
2995 {
2996 struct event ev;
2997
2998 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
2999 switch (ev.id)
3000 {
3001 case Q_STOP_SCAN:
3002 case SYS_POWEROFF:
3003 case SYS_USB_CONNECTED:
3004 /* Put the event back into the queue. */
3005 queue_post(&tagcache_queue, ev.id, ev.data);
3006 return true;
3007 }
3008
3009 return false;
3010 }
3011
3012 #ifdef HAVE_TC_RAMCACHE
3013 static bool allocate_tagcache(void)
3014 {
3015 struct master_header tcmh;
3016 int fd;
3017
3018 /* Load the header. */
3019 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3020 {
3021 hdr = NULL;
3022 return false;
3023 }
3024
3025 close(fd);
3026
3027 /**
3028 * Now calculate the required cache size plus
3029 * some extra space for alignment fixes.
3030 */
3031 stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3032 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3033 hdr = buffer_alloc(stat.ramcache_allocated + 128);
3034 memset(hdr, 0, sizeof(struct ramcache_header));
3035 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3036 hdr->indices = (struct index_entry *)(hdr + 1);
3037 logf("tagcache: %d bytes allocated.", stat.ramcache_allocated);
3038
3039 return true;
3040 }
3041
3042 # ifdef HAVE_EEPROM_SETTINGS
3043 static bool tagcache_dumpload(void)
3044 {
3045 struct statefile_header shdr;
3046 int fd, rc;
3047 long offpos;
3048 int i;
3049
3050 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3051 if (fd < 0)
3052 {
3053 logf("no tagcache statedump");
3054 return false;
3055 }
3056
3057 /* Check the statefile memory placement */
3058 hdr = buffer_alloc(0);
3059 rc = read(fd, &shdr, sizeof(struct statefile_header));
3060 if (rc != sizeof(struct statefile_header)
3061 /* || (long)hdr != (long)shdr.hdr */)
3062 {
3063 logf("incorrect statefile");
3064 hdr = NULL;
3065 close(fd);
3066 return false;
3067 }
3068
3069 offpos = (long)hdr - (long)shdr.hdr;
3070
3071 /* Lets allocate real memory and load it */
3072 hdr = buffer_alloc(shdr.stat.ramcache_allocated);
3073 rc = read(fd, hdr, shdr.stat.ramcache_allocated);
3074 close(fd);
3075
3076 if (rc != shdr.stat.ramcache_allocated)
3077 {
3078 logf("read failure!");
3079 hdr = NULL;
3080 return false;
3081 }
3082
3083 memcpy(&stat, &shdr.stat, sizeof(struct tagcache_stat));
3084
3085 /* Now fix the pointers */
3086 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3087 for (i = 0; i < TAG_COUNT; i++)
3088 hdr->tags[i] += offpos;
3089
3090 return true;
3091 }
3092
3093 static bool tagcache_dumpsave(void)
3094 {
3095 struct statefile_header shdr;
3096 int fd;
3097
3098 if (!stat.ramcache)
3099 return false;
3100
3101 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3102 if (fd < 0)
3103 {
3104 logf("failed to create a statedump");
3105 return false;
3106 }
3107
3108 /* Create the header */
3109 shdr.hdr = hdr;
3110 memcpy(&shdr.stat, &stat, sizeof(struct tagcache_stat));
3111 write(fd, &shdr, sizeof(struct statefile_header));
3112
3113 /* And dump the data too */
3114 write(fd, hdr, stat.ramcache_allocated);
3115 close(fd);
3116
3117 return true;
3118 }
3119 # endif
3120
3121 static bool load_tagcache(void)
3122 {
3123 struct tagcache_header *tch;
3124 long bytesleft = stat.ramcache_allocated;
3125 struct index_entry *idx;
3126 int rc, fd;
3127 char *p;
3128 int i, tag;
3129 int yield_count = 0;
3130
3131 # ifdef HAVE_DIRCACHE
3132 while (dircache_is_initializing())
3133 sleep(1);
3134 # endif
3135
3136 logf("loading tagcache to ram...");
3137
3138 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3139 if (fd < 0)
3140 {
3141 logf("tagcache open failed");
3142 return false;
3143 }
3144
3145 if (read(fd, &hdr->h, sizeof(struct master_header))
3146 != sizeof(struct master_header)
3147 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3148 {
3149 logf("incorrect header");
3150 return false;
3151 }
3152
3153 idx = hdr->indices;
3154
3155 /* Load the master index table. */
3156 for (i = 0; i < hdr->h.tch.entry_count; i++)
3157 {
3158 rc = read(fd, idx, sizeof(struct index_entry));
3159 if (rc != sizeof(struct index_entry))
3160 {
3161 logf("read error #10");
3162 close(fd);
3163 return false;
3164 }
3165
3166 bytesleft -= sizeof(struct index_entry);
3167 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= stat.ramcache_allocated)
3168 {
3169 logf("too big tagcache.");
3170 close(fd);
3171 return false;
3172 }
3173
3174 idx++;
3175 }
3176
3177 close(fd);
3178
3179 /* Load the tags. */
3180 p = (char *)idx;
3181 for (tag = 0; tag < TAG_COUNT; tag++)
3182 {
3183 struct tagfile_entry *fe;
3184 char buf[MAX_PATH];
3185
3186 if (tagcache_is_numeric_tag(tag))
3187 continue ;
3188
3189 //p = ((void *)p+1);
3190 p = (char *)((long)p & ~0x03) + 0x04;
3191 hdr->tags[tag] = p;
3192
3193 /* Check the header. */
3194 tch = (struct tagcache_header *)p;
3195 p += sizeof(struct tagcache_header);
3196
3197 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3198 return false;
3199
3200 for (hdr->entry_count[tag] = 0;
3201 hdr->entry_count[tag] < tch->entry_count;
3202 hdr->entry_count[tag]++)
3203 {
3204 long pos;
3205
3206 if (yield_count++ == 100)
3207 {
3208 yield();
3209 /* Abort if we got a critical event in queue */
3210 if (check_event_queue())
3211 return false;
3212 yield_count = 0;
3213 }
3214
3215 fe = (struct tagfile_entry *)p;
3216 pos = lseek(fd, 0, SEEK_CUR);
3217 rc = read(fd, fe, sizeof(struct tagfile_entry));
3218 if (rc != sizeof(struct tagfile_entry))
3219 {
3220 /* End of lookup table. */
3221 logf("read error #11");
3222 close(fd);
3223 return false;
3224 }
3225
3226 /* We have a special handling for the filename tags. */
3227 if (tag == tag_filename)
3228 {
3229 # ifdef HAVE_DIRCACHE
3230 const struct dircache_entry *dc;
3231 # endif
3232
3233 // FIXME: This is wrong!
3234 // idx = &hdr->indices[hdr->entry_count[i]];
3235 idx = &hdr->indices[fe->idx_id];
3236
3237 if (fe->tag_length >= (long)sizeof(buf)-1)
3238 {
3239 read(fd, buf, 10);
3240 buf[10] = '\0';
3241 logf("TAG:%s", buf);
3242 logf("too long filename");
3243 close(fd);
3244 return false;
3245 }
3246
3247 rc = read(fd, buf, fe->tag_length);
3248 if (rc != fe->tag_length)
3249 {
3250 logf("read error #12");
3251 close(fd);
3252 return false;
3253 }
3254
3255 /* Check if the entry has already been removed */
3256 if (idx->flag & FLAG_DELETED)
3257 continue;
3258
3259 /* This flag must not be used yet. */
3260 if (idx->flag & FLAG_DIRCACHE)
3261 {
3262 logf("internal error!");
3263 close(fd);
3264 return false;
3265 }
3266
3267 if (idx->tag_seek[tag] != pos)
3268 {
3269 logf("corrupt data structures!");
3270 close(fd);
3271 return false;
3272 }
3273
3274 # ifdef HAVE_DIRCACHE
3275 if (dircache_is_enabled())
3276 {
3277 dc = dircache_get_entry_ptr(buf);
3278 if (dc == NULL)
3279 {
3280 logf("Entry no longer valid.");
3281 logf("-> %s", buf);
3282 delete_entry(fe->idx_id);
3283 continue ;
3284 }
3285
3286 idx->flag |= FLAG_DIRCACHE;
3287 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3288 idx->tag_seek[tag_filename] = (long)dc;
3289 }
3290 else
3291 # endif
3292 {
3293
3294 # if 0 /* Maybe we could enable this for flash players. Too slow otherwise. */
3295 /* Check if entry has been removed. */
3296 if (global_settings.tagcache_autoupdate)
3297 {
3298 int testfd;
3299
3300 testfd = open(buf, O_RDONLY);
3301 if (testfd < 0)
3302 {
3303 logf("Entry no longer valid.");
3304 logf("-> %s", buf);
3305 delete_entry(fe->idx_id);
3306 continue;
3307 }
3308 close(testfd);
3309 }
3310 # endif
3311 }
3312
3313 continue ;
3314 }
3315
3316 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3317 if (bytesleft < 0)
3318 {
3319 logf("too big tagcache #2");
3320 logf("tl: %d", fe->tag_length);
3321 logf("bl: %d", bytesleft);
3322 close(fd);
3323 return false;
3324 }
3325
3326 p = fe->tag_data;
3327 rc = read(fd, fe->tag_data, fe->tag_length);
3328 p += rc;
3329
3330 if (rc != fe->tag_length)
3331 {
3332 logf("read error #13");
3333 logf("rc=0x%04x", rc); // 0x431
3334 logf("len=0x%04x", fe->tag_length); // 0x4000
3335 logf("pos=0x%04x", lseek(fd, 0, SEEK_CUR)); // 0x433
3336 logf("tag=0x%02x", tag); // 0x00
3337 close(fd);
3338 return false;
3339 }
3340 }
3341 close(fd);
3342 }
3343
3344 stat.ramcache_used = stat.ramcache_allocated - bytesleft;
3345 logf("tagcache loaded into ram!");
3346
3347 return true;
3348 }
3349 #endif /* HAVE_TC_RAMCACHE */
3350
3351 static bool check_deleted_files(void)
3352 {
3353 int fd, testfd;
3354 char buf[MAX_PATH];
3355 struct tagfile_entry tfe;
3356
3357 logf("reverse scan...");
3358 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
3359 fd = open(buf, O_RDONLY);
3360
3361 if (fd < 0)
3362 {
3363 logf("%s open fail", buf);
3364 return false;
3365 }
3366
3367 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
3368 while (read(fd, &tfe, sizeof(struct tagfile_entry))
3369 == sizeof(struct tagfile_entry) && !check_event_queue())
3370 {
3371 if (tfe.tag_length >= (long)sizeof(buf)-1)
3372 {
3373 logf("too long tag");
3374 close(fd);
3375 return false;
3376 }
3377
3378 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3379 {
3380 logf("read error #14");
3381 close(fd);
3382 return false;
3383 }
3384
3385 /* Now check if the file exists. */
3386 testfd = open(buf, O_RDONLY);
3387 if (testfd < 0)
3388 {
3389 logf("Entry no longer valid.");
3390 logf("-> %s", buf);
3391 delete_entry(tfe.idx_id);
3392 }
3393 close(testfd);
3394 }
3395
3396 close(fd);
3397
3398 logf("done");
3399
3400 return true;
3401 }
3402
3403 static bool check_dir(const char *dirname)
3404 {
3405 DIRCACHED *dir;
3406 int len;
3407 int success = false;
3408
3409 dir = opendir_cached(dirname);
3410 if (!dir)
3411 {
3412 logf("tagcache: opendir_cached() failed");
3413 return false;
3414 }
3415
3416 /* Recursively scan the dir. */
3417 while (!check_event_queue())
3418 {
3419 struct dircache_entry *entry;
3420
3421 entry = readdir_cached(dir);
3422
3423 if (entry == NULL)
3424 {
3425 success = true;
3426 break ;
3427 }
3428
3429 if (!strcmp(entry->d_name, ".") ||
3430 !strcmp(entry->d_name, ".."))
3431 continue;
3432
3433 yield();
3434
3435 len = strlen(curpath);
3436 snprintf(&curpath[len], curpath_size - len, "/%s",
3437 entry->d_name);
3438
3439 processed_dir_count++;
3440 if (entry->attribute & ATTR_DIRECTORY)
3441 check_dir(curpath);
3442 else
3443 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
3444 add_tagcache(curpath, dir->internal_entry);
3445 #else
3446 add_tagcache(curpath);
3447 #endif
3448
3449 curpath[len] = '\0';
3450 }
3451
3452 closedir_cached(dir);
3453
3454 return success;
3455 }
3456
3457 static void build_tagcache(void)
3458 {
3459 struct tagcache_header header;
3460 bool ret;
3461
3462 curpath[0] = '\0';
3463 data_size = 0;
3464 total_entry_count = 0;
3465 processed_dir_count = 0;
3466
3467 #ifdef HAVE_DIRCACHE
3468 while (dircache_is_initializing())
3469 sleep(1);
3470 #endif
3471
3472 logf("updating tagcache");
3473
3474 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
3475 if (cachefd >= 0)
3476 {
3477 logf("skipping, cache already waiting for commit");
3478 close(cachefd);
3479 return ;
3480 }
3481
3482 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
3483 if (cachefd < 0)
3484 {
3485 logf("master file open failed");
3486 return ;
3487 }
3488
3489 filenametag_fd = open_tag_fd(&header, tag_filename, false);
3490
3491 cpu_boost_id(true, CPUBOOSTID_TAGCACHE);
3492
3493 /* Scan for new files. */
3494 memset(&header, 0, sizeof(struct tagcache_header));
3495 write(cachefd, &header, sizeof(struct tagcache_header));
3496
3497 //strcpy(curpath, "/Best");
3498 ret = check_dir("/");
3499
3500 /* Write the header. */
3501 header.magic = TAGCACHE_MAGIC;
3502 header.datasize = data_size;
3503 header.entry_count = total_entry_count;
3504 lseek(cachefd, 0, SEEK_SET);
3505 write(cachefd, &header, sizeof(struct tagcache_header));
3506 close(cachefd);
3507
3508 if (filenametag_fd >= 0)
3509 {
3510 close(filenametag_fd);
3511 filenametag_fd = -1;
3512 }
3513
3514 if (!ret)
3515 {
3516 logf("Aborted.");
3517 cpu_boost_id(false, CPUBOOSTID_TAGCACHE);
3518 return ;
3519 }
3520
3521 /* Commit changes to the database. */
3522 if (commit())
3523 {
3524 remove(TAGCACHE_FILE_TEMP);
3525 logf("tagcache built!");
3526 }
3527
3528 #ifdef HAVE_TC_RAMCACHE
3529 if (hdr)
3530 {
3531 /* Import runtime statistics if we just initialized the db. */
3532 if (hdr->h.serial == 0)
3533 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
3534 }
3535 #endif
3536
3537 cpu_boost_id(false, CPUBOOSTID_TAGCACHE);
3538 }
3539
3540 #ifdef HAVE_TC_RAMCACHE
3541 static void load_ramcache(void)
3542 {
3543 if (!hdr)
3544 return ;
3545
3546 cpu_boost_id(true, CPUBOOSTID_TAGCACHE);
3547
3548 /* At first we should load the cache (if exists). */
3549 stat.ramcache = load_tagcache();
3550
3551 if (!stat.ramcache)
3552 {
3553 /* If loading failed, it must indicate some problem with the db
3554 * so disable it entirely to prevent further issues. */
3555 stat.ready = false;
3556 hdr = NULL;
3557 }
3558
3559 cpu_boost_id(false, CPUBOOSTID_TAGCACHE);
3560 }
3561
3562 void tagcache_unload_ramcache(void)
3563 {
3564 stat.ramcache = false;
3565 /* Just to make sure there is no statefile present. */
3566 // remove(TAGCACHE_STATEFILE);
3567 }
3568 #endif
3569
3570 static bool check_all_headers(void)
3571 {
3572 struct master_header myhdr;
3573 struct tagcache_header tch;
3574 int tag;
3575 int fd;
3576
3577 if ( (fd = open_master_fd(&myhdr, false)) < 0)
3578 return false;
3579
3580 close(fd);
3581 current_serial = myhdr.serial;
3582
3583 for (tag = 0; tag < TAG_COUNT; tag++)
3584 {
3585 if (tagcache_is_numeric_tag(tag))
3586 continue;
3587
3588 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
3589 return false;
3590
3591 close(fd);
3592 }
3593
3594 return true;
3595 }
3596
3597 static void tagcache_thread(void)
3598 {
3599 struct event ev;
3600 bool check_done = false;
3601
3602 /* If the previous cache build/update was interrupted, commit
3603 * the changes first in foreground. */
3604 cpu_boost_id(true, CPUBOOSTID_TAGCACHE);
3605 allocate_tempbuf();
3606 commit();
3607 free_tempbuf();
3608
3609 #ifdef HAVE_TC_RAMCACHE
3610 # ifdef HAVE_EEPROM_SETTINGS
3611 if (firmware_settings.initialized && firmware_settings.disk_clean)
3612 check_done = tagcache_dumpload();
3613
3614 remove(TAGCACHE_STATEFILE);
3615 # endif
3616
3617 /* Allocate space for the tagcache if found on disk. */
3618 if (global_settings.tagcache_ram && !stat.ramcache)
3619 allocate_tagcache();
3620 #endif
3621
3622 cpu_boost_id(false, CPUBOOSTID_TAGCACHE);
3623 stat.initialized = true;
3624
3625 /* Don't delay bootup with the header check but do it on background. */
3626 sleep(HZ);
3627 stat.ready = check_all_headers();
3628
3629 while (1)
3630 {
3631 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
3632
3633 switch (ev.id)
3634 {
3635 case Q_IMPORT_CHANGELOG:
3636 tagcache_import_changelog();
3637 break;
3638
3639 case Q_REBUILD:
3640 remove_files();
3641 build_tagcache();
3642 break;
3643
3644 case Q_UPDATE:
3645 build_tagcache();
3646 #ifdef HAVE_TC_RAMCACHE
3647 load_ramcache();
3648 #endif
3649 check_deleted_files();
3650 break ;
3651
3652 case Q_START_SCAN:
3653 check_done = false;
3654 case SYS_TIMEOUT:
3655 if (check_done || !stat.ready)
3656 break ;
3657
3658 #ifdef HAVE_TC_RAMCACHE
3659 if (!stat.ramcache && global_settings.tagcache_ram)
3660 {
3661 load_ramcache();
3662 if (stat.ramcache && global_settings.tagcache_autoupdate)
3663 build_tagcache();
3664 }
3665 else
3666 #endif
3667 if (global_settings.tagcache_autoupdate)
3668 {
3669 build_tagcache();
3670 /* Don't do auto removal without dircache (very slow). */
3671 #ifdef HAVE_DIRCACHE
3672 if (dircache_is_enabled())
3673 check_deleted_files();
3674 #endif
3675 }
3676
3677 logf("tagcache check done");
3678
3679 check_done = true;
3680 break ;
3681
3682 case Q_STOP_SCAN:
3683 break ;
3684
3685 case SYS_POWEROFF:
3686 break ;
3687
3688 #ifndef SIMULATOR
3689 case SYS_USB_CONNECTED:
3690 logf("USB: TagCache");
3691 usb_acknowledge(SYS_USB_CONNECTED_ACK);
3692 usb_wait_for_disconnect(&tagcache_queue);
3693 break ;
3694 #endif
3695 }
3696 }
3697 }
3698
3699 bool tagcache_prepare_shutdown(void)
3700 {
3701 if (tagcache_get_commit_step() > 0)
3702 return false;
3703
3704 tagcache_stop_scan();
3705 while (read_lock || write_lock)
3706 sleep(1);
3707
3708 return true;
3709 }
3710
3711 void tagcache_shutdown(void)
3712 {
3713 #ifdef HAVE_EEPROM_SETTINGS
3714 if (stat.ramcache)
3715 tagcache_dumpsave();
3716 #endif
3717 }
3718
3719 static int get_progress(void)
3720 {
3721 int total_count = -1;
3722
3723 #ifdef HAVE_DIRCACHE
3724 if (dircache_is_enabled())
3725 {
3726 total_count = dircache_get_entry_count();
3727 }
3728 else
3729 #endif
3730 #ifdef HAVE_TC_RAMCACHE
3731 {
3732 if (hdr && stat.ramcache)
3733 total_count = hdr->h.tch.entry_count;
3734 }
3735 #endif
3736
3737 if (total_count < 0)
3738 return -1;
3739
3740 return processed_dir_count * 100 / total_count;
3741 }
3742
3743 struct tagcache_stat* tagcache_get_stat(void)
3744 {
3745 stat.progress = get_progress();
3746 stat.processed_entries = processed_dir_count;
3747
3748 return &stat;
3749 }
3750
3751 void tagcache_start_scan(void)
3752 {
3753 queue_post(&tagcache_queue, Q_START_SCAN, 0);
3754 }
3755
3756 bool tagcache_update(void)
3757 {
3758 if (!stat.ready)
3759 return false;
3760
3761 queue_post(&tagcache_queue, Q_UPDATE, 0);
3762 gui_syncsplash(HZ*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH));
3763
3764 return false;
3765 }
3766
3767 bool tagcache_rebuild(void)
3768 {
3769 queue_post(&tagcache_queue, Q_REBUILD, 0);
3770 gui_syncsplash(HZ*2, true, str(LANG_TAGCACHE_FORCE_UPDATE_SPLASH));
3771
3772 return false;
3773 }
3774
3775 void tagcache_stop_scan(void)
3776 {
3777 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
3778 }
3779
3780 #ifdef HAVE_TC_RAMCACHE
3781 bool tagcache_is_ramcache(void)
3782 {
3783 return stat.ramcache;
3784 }
3785 #endif
3786
3787
3788 void tagcache_init(void)
3789 {
3790 memset(&stat, 0, sizeof(struct tagcache_stat));
3791 filenametag_fd = -1;
3792 current_serial = 0;
3793 write_lock = read_lock = 0;
3794
3795 queue_init(&tagcache_queue, true);
3796 create_thread(tagcache_thread, tagcache_stack,
3797 sizeof(tagcache_stack), tagcache_thread_name
3798 IF_PRIO(, PRIORITY_BACKGROUND));
3799 }
3800
3801 bool tagcache_is_initialized(void)
3802 {
3803 return stat.initialized;
3804 }
3805
3806 int tagcache_get_commit_step(void)
3807 {
3808 return stat.commit_step;
3809 }
3810
Open Source Jukebox Firmware