?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Manual: no need to keep the ONDIO_PAD mentioned in calendar.tex (see r18884).
[kugel-rb.git] / apps / tagcache.c
blob19469cde1aaf561be517fae9340373636bf782e7
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /* #define LOGF_ENABLE */
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #include "config.h"
64 #include "ata_idle_notify.h"
65 #include "thread.h"
66 #include "kernel.h"
67 #include "system.h"
68 #include "logf.h"
69 #include "string.h"
70 #include "usb.h"
71 #include "metadata.h"
72 #include "metadata.h"
73 #include "tagcache.h"
74 #include "buffer.h"
75 #include "crc32.h"
76 #include "misc.h"
77 #include "settings.h"
78 #include "dir.h"
79 #include "structec.h"
80 #include "tagcache.h"
81
82 #ifndef __PCTOOL__
83 #include "splash.h"
84 #include "lang.h"
85 #include "eeprom_settings.h"
86 #endif
87
88 #ifdef __PCTOOL__
89 #define yield() do { } while(0)
90 #define sim_sleep(timeout) do { } while(0)
91 #define do_timed_yield() do { } while(0)
92 #endif
93
94 #ifndef __PCTOOL__
95 /* Tag Cache thread. */
96 static struct event_queue tagcache_queue;
97 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
98 static const char tagcache_thread_name[] = "tagcache";
99 #endif
100
101 #define UNTAGGED "<Untagged>"
102
103 /* Previous path when scanning directory tree recursively. */
104 static char curpath[TAG_MAXLEN+32];
105
106 /* Used when removing duplicates. */
107 static char *tempbuf; /* Allocated when needed. */
108 static long tempbufidx; /* Current location in buffer. */
109 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
110 static long tempbuf_left; /* Buffer space left. */
111 static long tempbuf_pos;
112
113 /* Tags we want to get sorted (loaded to the tempbuf). */
114 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
115 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
116
117 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
118 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
119 tag_composer, tag_comment, tag_albumartist, tag_grouping };
120
121 /* Numeric tags (we can use these tags with conditional clauses). */
122 static const int numeric_tags[] = { tag_year, tag_discnumber,
123 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
124 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
125 tag_virt_length_min, tag_virt_length_sec,
126 tag_virt_playtime_min, tag_virt_playtime_sec,
127 tag_virt_entryage, tag_virt_autoscore };
128
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime" };
134
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat;
137
138 /* Queue commands. */
139 enum tagcache_queue {
140 Q_STOP_SCAN = 0,
141 Q_START_SCAN,
142 Q_IMPORT_CHANGELOG,
143 Q_UPDATE,
144 Q_REBUILD,
145
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER,
148 CMD_UPDATE_NUMERIC,
149 };
150
151 struct tagcache_command_entry {
152 int32_t command;
153 int32_t idx_id;
154 int32_t tag;
155 int32_t data;
156 };
157
158 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
159 static volatile int command_queue_widx = 0;
160 static volatile int command_queue_ridx = 0;
161 static struct mutex command_queue_mutex;
162
163 /* Tag database structures. */
164
165 /* Variable-length tag entry in tag files. */
166 struct tagfile_entry {
167 short tag_length; /* Length of the data in bytes including '\0' */
168 short idx_id; /* Corresponding entry location in index file of not unique tags */
169 char tag_data[0]; /* Begin of the tag data */
170 };
171
172 /* Fixed-size tag entry in master db index. */
173 struct index_entry {
174 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
175 int32_t flag; /* Status flags */
176 };
177
178 /* Header is the same in every file. */
179 struct tagcache_header {
180 int32_t magic; /* Header version number */
181 int32_t datasize; /* Data size in bytes */
182 int32_t entry_count; /* Number of entries in this file */
183 };
184
185 struct master_header {
186 struct tagcache_header tch;
187 int32_t serial; /* Increasing counting number */
188 int32_t commitid; /* Number of commits so far */
189 int32_t dirty;
190 };
191
192 /* For the endianess correction */
193 static const char *tagfile_entry_ec = "ss";
194 /**
195 Note: This should be (1 + TAG_COUNT) amount of l's.
196 */
197 static const char *index_entry_ec = "lllllllllllllllllllll";
198
199 static const char *tagcache_header_ec = "lll";
200 static const char *master_header_ec = "llllll";
201
202 static struct master_header current_tcmh;
203
204 #ifdef HAVE_TC_RAMCACHE
205 /* Header is created when loading database to ram. */
206 struct ramcache_header {
207 struct master_header h; /* Header from the master index */
208 struct index_entry *indices; /* Master index file content */
209 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
210 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
211 };
212
213 # ifdef HAVE_EEPROM_SETTINGS
214 struct statefile_header {
215 struct ramcache_header *hdr;
216 struct tagcache_stat tc_stat;
217 };
218 # endif
219
220 /* Pointer to allocated ramcache_header */
221 static struct ramcache_header *hdr;
222 #endif
223
224 /**
225 * Full tag entries stored in a temporary file waiting
226 * for commit to the cache. */
227 struct temp_file_entry {
228 long tag_offset[TAG_COUNT];
229 short tag_length[TAG_COUNT];
230 long flag;
231
232 long data_length;
233 };
234
235 struct tempbuf_id_list {
236 long id;
237 struct tempbuf_id_list *next;
238 };
239
240 struct tempbuf_searchidx {
241 long idx_id;
242 char *str;
243 int seek;
244 struct tempbuf_id_list idlist;
245 };
246
247 /* Lookup buffer for fixing messed up index while after sorting. */
248 static long commit_entry_count;
249 static long lookup_buffer_depth;
250 static struct tempbuf_searchidx **lookup;
251
252 /* Used when building the temporary file. */
253 static int cachefd = -1, filenametag_fd;
254 static int total_entry_count = 0;
255 static int data_size = 0;
256 static int processed_dir_count;
257
258 /* Thread safe locking */
259 static volatile int write_lock;
260 static volatile int read_lock;
261
262 static bool delete_entry(long idx_id);
263
264 const char* tagcache_tag_to_str(int tag)
265 {
266 return tags_str[tag];
267 }
268
269 bool tagcache_is_numeric_tag(int type)
270 {
271 int i;
272
273 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
274 {
275 if (type == numeric_tags[i])
276 return true;
277 }
278
279 return false;
280 }
281
282 bool tagcache_is_unique_tag(int type)
283 {
284 int i;
285
286 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
287 {
288 if (type == unique_tags[i])
289 return true;
290 }
291
292 return false;
293 }
294
295 bool tagcache_is_sorted_tag(int type)
296 {
297 int i;
298
299 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
300 {
301 if (type == sorted_tags[i])
302 return true;
303 }
304
305 return false;
306 }
307
308 #ifdef HAVE_DIRCACHE
309 /**
310 * Returns true if specified flag is still present, i.e., dircache
311 * has not been reloaded.
312 */
313 static bool is_dircache_intact(void)
314 {
315 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
316 }
317 #endif
318
319 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
320 {
321 int fd;
322 char buf[MAX_PATH];
323 int rc;
324
325 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
326 return -1;
327
328 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
329
330 fd = open(buf, write ? O_RDWR : O_RDONLY);
331 if (fd < 0)
332 {
333 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
334 tc_stat.ready = false;
335 return fd;
336 }
337
338 /* Check the header. */
339 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
340 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
341 {
342 logf("header error");
343 tc_stat.ready = false;
344 close(fd);
345 return -2;
346 }
347
348 return fd;
349 }
350
351 static int open_master_fd(struct master_header *hdr, bool write)
352 {
353 int fd;
354 int rc;
355
356 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
357 if (fd < 0)
358 {
359 logf("master file open failed for R/W");
360 tc_stat.ready = false;
361 return fd;
362 }
363
364 tc_stat.econ = false;
365
366 /* Check the header. */
367 rc = read(fd, hdr, sizeof(struct master_header));
368 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
369 {
370 /* Success. */
371 return fd;
372 }
373
374 /* Trying to read again, this time with endianess correction enabled. */
375 lseek(fd, 0, SEEK_SET);
376
377 rc = ecread(fd, hdr, 1, master_header_ec, true);
378 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
379 {
380 logf("header error");
381 tc_stat.ready = false;
382 close(fd);
383 return -2;
384 }
385
386 tc_stat.econ = true;
387
388 return fd;
389 }
390
391 #ifndef __PCTOOL__
392 static bool do_timed_yield(void)
393 {
394 /* Sorting can lock up for quite a while, so yield occasionally */
395 static long wakeup_tick = 0;
396 if (current_tick >= wakeup_tick)
397 {
398 wakeup_tick = current_tick + (HZ/4);
399 yield();
400 return true;
401 }
402 return false;
403 }
404 #endif
405
406 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
407 static long find_entry_ram(const char *filename,
408 const struct dirent *dc)
409 {
410 static long last_pos = 0;
411 int i;
412
413 /* Check if we tagcache is loaded into ram. */
414 if (!tc_stat.ramcache)
415 return -1;
416
417 if (dc == NULL)
418 dc = dircache_get_entry_ptr(filename);
419
420 if (dc == NULL)
421 {
422 logf("tagcache: file not found.");
423 return -1;
424 }
425
426 try_again:
427
428 if (last_pos > 0)
429 i = last_pos;
430 else
431 i = 0;
432
433 for (; i < hdr->h.tch.entry_count; i++)
434 {
435 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
436 {
437 last_pos = MAX(0, i - 3);
438 return i;
439 }
440
441 do_timed_yield();
442 }
443
444 if (last_pos > 0)
445 {
446 last_pos = 0;
447 goto try_again;
448 }
449
450 return -1;
451 }
452 #endif
453
454 static long find_entry_disk(const char *filename)
455 {
456 struct tagcache_header tch;
457 static long last_pos = -1;
458 long pos_history[POS_HISTORY_COUNT];
459 long pos_history_idx = 0;
460 bool found = false;
461 struct tagfile_entry tfe;
462 int fd;
463 char buf[TAG_MAXLEN+32];
464 int i;
465 int pos = -1;
466
467 if (!tc_stat.ready)
468 return -2;
469
470 fd = filenametag_fd;
471 if (fd < 0)
472 {
473 last_pos = -1;
474 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
475 return -1;
476 }
477
478 check_again:
479
480 if (last_pos > 0)
481 lseek(fd, last_pos, SEEK_SET);
482 else
483 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
484
485 while (true)
486 {
487 pos = lseek(fd, 0, SEEK_CUR);
488 for (i = pos_history_idx-1; i >= 0; i--)
489 pos_history[i+1] = pos_history[i];
490 pos_history[0] = pos;
491
492 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
493 != sizeof(struct tagfile_entry))
494 {
495 break ;
496 }
497
498 if (tfe.tag_length >= (long)sizeof(buf))
499 {
500 logf("too long tag #1");
501 close(fd);
502 last_pos = -1;
503 return -2;
504 }
505
506 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
507 {
508 logf("read error #2");
509 close(fd);
510 last_pos = -1;
511 return -3;
512 }
513
514 if (!strcasecmp(filename, buf))
515 {
516 last_pos = pos_history[pos_history_idx];
517 found = true;
518 break ;
519 }
520
521 if (pos_history_idx < POS_HISTORY_COUNT - 1)
522 pos_history_idx++;
523 }
524
525 /* Not found? */
526 if (!found)
527 {
528 if (last_pos > 0)
529 {
530 last_pos = -1;
531 logf("seek again");
532 goto check_again;
533 }
534
535 if (fd != filenametag_fd)
536 close(fd);
537 return -4;
538 }
539
540 if (fd != filenametag_fd)
541 close(fd);
542
543 return tfe.idx_id;
544 }
545
546 static int find_index(const char *filename)
547 {
548 long idx_id = -1;
549
550 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
551 if (tc_stat.ramcache && is_dircache_intact())
552 idx_id = find_entry_ram(filename, NULL);
553 #endif
554
555 if (idx_id < 0)
556 idx_id = find_entry_disk(filename);
557
558 return idx_id;
559 }
560
561 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
562 {
563 int idx_id;
564
565 if (!tc_stat.ready)
566 return false;
567
568 idx_id = find_index(filename);
569 if (idx_id < 0)
570 return false;
571
572 if (!tagcache_search(tcs, tag_filename))
573 return false;
574
575 tcs->entry_count = 0;
576 tcs->idx_id = idx_id;
577
578 return true;
579 }
580
581 static bool get_index(int masterfd, int idxid,
582 struct index_entry *idx, bool use_ram)
583 {
584 bool localfd = false;
585
586 if (idxid < 0)
587 {
588 logf("Incorrect idxid: %d", idxid);
589 return false;
590 }
591
592 #ifdef HAVE_TC_RAMCACHE
593 if (tc_stat.ramcache && use_ram)
594 {
595 if (hdr->indices[idxid].flag & FLAG_DELETED)
596 return false;
597
598 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
599 return true;
600 }
601 #else
602 (void)use_ram;
603 #endif
604
605 if (masterfd < 0)
606 {
607 struct master_header tcmh;
608
609 localfd = true;
610 masterfd = open_master_fd(&tcmh, false);
611 if (masterfd < 0)
612 return false;
613 }
614
615 lseek(masterfd, idxid * sizeof(struct index_entry)
616 + sizeof(struct master_header), SEEK_SET);
617 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
618 != sizeof(struct index_entry))
619 {
620 logf("read error #3");
621 if (localfd)
622 close(masterfd);
623
624 return false;
625 }
626
627 if (localfd)
628 close(masterfd);
629
630 if (idx->flag & FLAG_DELETED)
631 return false;
632
633 return true;
634 }
635
636 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
637 {
638 /* We need to exclude all memory only flags & tags when writing to disk. */
639 if (idx->flag & FLAG_DIRCACHE)
640 {
641 logf("memory only flags!");
642 return false;
643 }
644
645 #ifdef HAVE_TC_RAMCACHE
646 /* Only update numeric data. Writing the whole index to RAM by memcpy
647 * destroys dircache pointers!
648 */
649 if (tc_stat.ramcache)
650 {
651 int tag;
652 struct index_entry *idx_ram = &hdr->indices[idxid];
653
654 for (tag = 0; tag < TAG_COUNT; tag++)
655 {
656 if (tagcache_is_numeric_tag(tag))
657 {
658 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
659 }
660 }
661
662 /* Don't touch the dircache flag or attributes. */
663 idx_ram->flag = (idx->flag & 0x0000ffff)
664 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
665 }
666 #endif
667
668 lseek(masterfd, idxid * sizeof(struct index_entry)
669 + sizeof(struct master_header), SEEK_SET);
670 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
671 != sizeof(struct index_entry))
672 {
673 logf("write error #3");
674 logf("idxid: %d", idxid);
675 return false;
676 }
677
678 return true;
679 }
680
681 static bool open_files(struct tagcache_search *tcs, int tag)
682 {
683 if (tcs->idxfd[tag] < 0)
684 {
685 char fn[MAX_PATH];
686
687 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
688 tcs->idxfd[tag] = open(fn, O_RDONLY);
689 }
690
691 if (tcs->idxfd[tag] < 0)
692 {
693 logf("File not open!");
694 return false;
695 }
696
697 return true;
698 }
699
700 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
701 int tag, char *buf, long size)
702 {
703 struct tagfile_entry tfe;
704 long seek;
705
706 *buf = '\0';
707
708 if (tagcache_is_numeric_tag(tag))
709 return false;
710
711 seek = idx->tag_seek[tag];
712 if (seek < 0)
713 {
714 logf("Retrieve failed");
715 return false;
716 }
717
718 #ifdef HAVE_TC_RAMCACHE
719 if (tcs->ramsearch)
720 {
721 struct tagfile_entry *ep;
722
723 # ifdef HAVE_DIRCACHE
724 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
725 && is_dircache_intact())
726 {
727 dircache_copy_path((struct dirent *)seek,
728 buf, size);
729 return true;
730 }
731 else
732 # endif
733 if (tag != tag_filename)
734 {
735 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
736 strncpy(buf, ep->tag_data, size-1);
737
738 return true;
739 }
740 }
741 #endif
742
743 if (!open_files(tcs, tag))
744 return false;
745
746 lseek(tcs->idxfd[tag], seek, SEEK_SET);
747 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
748 != sizeof(struct tagfile_entry))
749 {
750 logf("read error #5");
751 return false;
752 }
753
754 if (tfe.tag_length >= size)
755 {
756 logf("too small buffer");
757 return false;
758 }
759
760 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
761 tfe.tag_length)
762 {
763 logf("read error #6");
764 return false;
765 }
766
767 buf[tfe.tag_length] = '\0';
768
769 return true;
770 }
771
772 static long check_virtual_tags(int tag, const struct index_entry *idx)
773 {
774 long data = 0;
775
776 switch (tag)
777 {
778 case tag_virt_length_sec:
779 data = (idx->tag_seek[tag_length]/1000) % 60;
780 break;
781
782 case tag_virt_length_min:
783 data = (idx->tag_seek[tag_length]/1000) / 60;
784 break;
785
786 case tag_virt_playtime_sec:
787 data = (idx->tag_seek[tag_playtime]/1000) % 60;
788 break;
789
790 case tag_virt_playtime_min:
791 data = (idx->tag_seek[tag_playtime]/1000) / 60;
792 break;
793
794 case tag_virt_autoscore:
795 if (idx->tag_seek[tag_length] == 0
796 || idx->tag_seek[tag_playcount] == 0)
797 {
798 data = 0;
799 }
800 else
801 {
802 data = 100 * idx->tag_seek[tag_playtime]
803 / idx->tag_seek[tag_length]
804 / idx->tag_seek[tag_playcount];
805 }
806 break;
807
808 /* How many commits before the file has been added to the DB. */
809 case tag_virt_entryage:
810 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
811 break;
812
813 default:
814 data = idx->tag_seek[tag];
815 }
816
817 return data;
818 }
819
820 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
821 {
822 struct index_entry idx;
823
824 if (!tc_stat.ready)
825 return false;
826
827 if (!tagcache_is_numeric_tag(tag))
828 return -1;
829
830 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
831 return -2;
832
833 return check_virtual_tags(tag, &idx);
834 }
835
836 inline static bool str_ends_with(const char *str1, const char *str2)
837 {
838 int str_len = strlen(str1);
839 int clause_len = strlen(str2);
840
841 if (clause_len > str_len)
842 return false;
843
844 return !strcasecmp(&str1[str_len - clause_len], str2);
845 }
846
847 inline static bool str_oneof(const char *str, const char *list)
848 {
849 const char *sep;
850 int l, len = strlen(str);
851
852 while (*list)
853 {
854 sep = strchr(list, '|');
855 l = sep ? (long)sep - (long)list : (int)strlen(list);
856 if ((l==len) && !strncasecmp(str, list, len))
857 return true;
858 list += sep ? l + 1 : l;
859 }
860
861 return false;
862 }
863
864 static bool check_against_clause(long numeric, const char *str,
865 const struct tagcache_search_clause *clause)
866 {
867 if (clause->numeric)
868 {
869 switch (clause->type)
870 {
871 case clause_is:
872 return numeric == clause->numeric_data;
873 case clause_is_not:
874 return numeric != clause->numeric_data;
875 case clause_gt:
876 return numeric > clause->numeric_data;
877 case clause_gteq:
878 return numeric >= clause->numeric_data;
879 case clause_lt:
880 return numeric < clause->numeric_data;
881 case clause_lteq:
882 return numeric <= clause->numeric_data;
883 default:
884 logf("Incorrect numeric tag: %d", clause->type);
885 }
886 }
887 else
888 {
889 switch (clause->type)
890 {
891 case clause_is:
892 return !strcasecmp(clause->str, str);
893 case clause_is_not:
894 return strcasecmp(clause->str, str);
895 case clause_gt:
896 return 0>strcasecmp(clause->str, str);
897 case clause_gteq:
898 return 0>=strcasecmp(clause->str, str);
899 case clause_lt:
900 return 0<strcasecmp(clause->str, str);
901 case clause_lteq:
902 return 0<=strcasecmp(clause->str, str);
903 case clause_contains:
904 return (strcasestr(str, clause->str) != NULL);
905 case clause_not_contains:
906 return (strcasestr(str, clause->str) == NULL);
907 case clause_begins_with:
908 return (strcasestr(str, clause->str) == str);
909 case clause_not_begins_with:
910 return (strcasestr(str, clause->str) != str);
911 case clause_ends_with:
912 return str_ends_with(str, clause->str);
913 case clause_not_ends_with:
914 return !str_ends_with(str, clause->str);
915 case clause_oneof:
916 return str_oneof(str, clause->str);
917
918 default:
919 logf("Incorrect tag: %d", clause->type);
920 }
921 }
922
923 return false;
924 }
925
926 static bool check_clauses(struct tagcache_search *tcs,
927 struct index_entry *idx,
928 struct tagcache_search_clause **clause, int count)
929 {
930 int i;
931
932 #ifdef HAVE_TC_RAMCACHE
933 if (tcs->ramsearch)
934 {
935 /* Go through all conditional clauses. */
936 for (i = 0; i < count; i++)
937 {
938 struct tagfile_entry *tfe;
939 int seek;
940 char buf[256];
941 char *str = NULL;
942
943 seek = check_virtual_tags(clause[i]->tag, idx);
944
945 if (!tagcache_is_numeric_tag(clause[i]->tag))
946 {
947 if (clause[i]->tag == tag_filename)
948 {
949 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
950 str = buf;
951 }
952 else
953 {
954 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
955 str = tfe->tag_data;
956 }
957 }
958
959 if (!check_against_clause(seek, str, clause[i]))
960 return false;
961 }
962 }
963 else
964 #endif
965 {
966 /* Check for conditions. */
967 for (i = 0; i < count; i++)
968 {
969 struct tagfile_entry tfe;
970 int seek;
971 char str[256];
972
973 seek = check_virtual_tags(clause[i]->tag, idx);
974
975 memset(str, 0, sizeof str);
976 if (!tagcache_is_numeric_tag(clause[i]->tag))
977 {
978 int fd = tcs->idxfd[clause[i]->tag];
979 lseek(fd, seek, SEEK_SET);
980 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
981 if (tfe.tag_length >= (int)sizeof(str))
982 {
983 logf("Too long tag read!");
984 break ;
985 }
986
987 read(fd, str, tfe.tag_length);
988
989 /* Check if entry has been deleted. */
990 if (str[0] == '\0')
991 break;
992 }
993
994 if (!check_against_clause(seek, str, clause[i]))
995 return false;
996 }
997 }
998
999 return true;
1000 }
1001
1002 bool tagcache_check_clauses(struct tagcache_search *tcs,
1003 struct tagcache_search_clause **clause, int count)
1004 {
1005 struct index_entry idx;
1006
1007 if (count == 0)
1008 return true;
1009
1010 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1011 return false;
1012
1013 return check_clauses(tcs, &idx, clause, count);
1014 }
1015
1016 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1017 {
1018 int i;
1019
1020 /* If uniq buffer is not defined we must return true for search to work. */
1021 if (tcs->unique_list == NULL
1022 || (!tagcache_is_unique_tag(tcs->type)
1023 && !tagcache_is_numeric_tag(tcs->type)))
1024 {
1025 return true;
1026 }
1027
1028 for (i = 0; i < tcs->unique_list_count; i++)
1029 {
1030 /* Return false if entry is found. */
1031 if (tcs->unique_list[i] == id)
1032 return false;
1033 }
1034
1035 if (tcs->unique_list_count < tcs->unique_list_capacity)
1036 {
1037 tcs->unique_list[i] = id;
1038 tcs->unique_list_count++;
1039 }
1040
1041 return true;
1042 }
1043
1044 static bool build_lookup_list(struct tagcache_search *tcs)
1045 {
1046 struct index_entry entry;
1047 int i;
1048
1049 tcs->seek_list_count = 0;
1050
1051 #ifdef HAVE_TC_RAMCACHE
1052 if (tcs->ramsearch)
1053 {
1054 int j;
1055
1056 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1057 {
1058 struct index_entry *idx = &hdr->indices[i];
1059 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1060 break ;
1061
1062 /* Skip deleted files. */
1063 if (idx->flag & FLAG_DELETED)
1064 continue;
1065
1066 /* Go through all filters.. */
1067 for (j = 0; j < tcs->filter_count; j++)
1068 {
1069 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1070 {
1071 break ;
1072 }
1073 }
1074
1075 if (j < tcs->filter_count)
1076 continue ;
1077
1078 /* Check for conditions. */
1079 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1080 continue;
1081
1082 /* Add to the seek list if not already in uniq buffer. */
1083 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1084 continue;
1085
1086 /* Lets add it. */
1087 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1088 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1089 tcs->seek_list_count++;
1090 }
1091
1092 tcs->seek_pos = i;
1093
1094 return tcs->seek_list_count > 0;
1095 }
1096 #endif
1097
1098 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1099 sizeof(struct master_header), SEEK_SET);
1100
1101 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1102 == sizeof(struct index_entry))
1103 {
1104 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1105 break ;
1106
1107 tcs->seek_pos++;
1108
1109 /* Check if entry has been deleted. */
1110 if (entry.flag & FLAG_DELETED)
1111 continue;
1112
1113 /* Go through all filters.. */
1114 for (i = 0; i < tcs->filter_count; i++)
1115 {
1116 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1117 break ;
1118 }
1119
1120 if (i < tcs->filter_count)
1121 continue ;
1122
1123 /* Check for conditions. */
1124 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1125 continue;
1126
1127 /* Add to the seek list if not already in uniq buffer. */
1128 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1129 continue;
1130
1131 /* Lets add it. */
1132 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1133 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1134 tcs->seek_list_count++;
1135
1136 yield();
1137 }
1138
1139 return tcs->seek_list_count > 0;
1140 }
1141
1142
1143 static void remove_files(void)
1144 {
1145 int i;
1146 char buf[MAX_PATH];
1147
1148 tc_stat.ready = false;
1149 tc_stat.ramcache = false;
1150 tc_stat.econ = false;
1151 remove(TAGCACHE_FILE_MASTER);
1152 for (i = 0; i < TAG_COUNT; i++)
1153 {
1154 if (tagcache_is_numeric_tag(i))
1155 continue;
1156
1157 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1158 remove(buf);
1159 }
1160 }
1161
1162
1163 static bool check_all_headers(void)
1164 {
1165 struct master_header myhdr;
1166 struct tagcache_header tch;
1167 int tag;
1168 int fd;
1169
1170 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1171 return false;
1172
1173 close(fd);
1174 if (myhdr.dirty)
1175 {
1176 logf("tagcache is dirty!");
1177 return false;
1178 }
1179
1180 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1181
1182 for (tag = 0; tag < TAG_COUNT; tag++)
1183 {
1184 if (tagcache_is_numeric_tag(tag))
1185 continue;
1186
1187 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1188 return false;
1189
1190 close(fd);
1191 }
1192
1193 return true;
1194 }
1195
1196 bool tagcache_search(struct tagcache_search *tcs, int tag)
1197 {
1198 struct tagcache_header tag_hdr;
1199 struct master_header master_hdr;
1200 int i;
1201
1202 if (tcs->initialized)
1203 tagcache_search_finish(tcs);
1204
1205 while (read_lock)
1206 sleep(1);
1207
1208 memset(tcs, 0, sizeof(struct tagcache_search));
1209 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1210 return false;
1211
1212 tcs->position = sizeof(struct tagcache_header);
1213 tcs->type = tag;
1214 tcs->seek_pos = 0;
1215 tcs->seek_list_count = 0;
1216 tcs->filter_count = 0;
1217 tcs->masterfd = -1;
1218
1219 for (i = 0; i < TAG_COUNT; i++)
1220 tcs->idxfd[i] = -1;
1221
1222 #ifndef HAVE_TC_RAMCACHE
1223 tcs->ramsearch = false;
1224 #else
1225 tcs->ramsearch = tc_stat.ramcache;
1226 if (tcs->ramsearch)
1227 {
1228 tcs->entry_count = hdr->entry_count[tcs->type];
1229 }
1230 else
1231 #endif
1232 {
1233 if (!tagcache_is_numeric_tag(tcs->type))
1234 {
1235 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1236 if (tcs->idxfd[tcs->type] < 0)
1237 return false;
1238 }
1239
1240 /* Always open as R/W so we can pass tcs to functions that modify data also
1241 * without failing. */
1242 tcs->masterfd = open_master_fd(&master_hdr, true);
1243
1244 if (tcs->masterfd < 0)
1245 return false;
1246 }
1247
1248 tcs->valid = true;
1249 tcs->initialized = true;
1250 write_lock++;
1251
1252 return true;
1253 }
1254
1255 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1256 void *buffer, long length)
1257 {
1258 tcs->unique_list = (unsigned long *)buffer;
1259 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1260 tcs->unique_list_count = 0;
1261 }
1262
1263 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1264 int tag, int seek)
1265 {
1266 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1267 return false;
1268
1269 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1270 return false;
1271
1272 tcs->filter_tag[tcs->filter_count] = tag;
1273 tcs->filter_seek[tcs->filter_count] = seek;
1274 tcs->filter_count++;
1275
1276 return true;
1277 }
1278
1279 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1280 struct tagcache_search_clause *clause)
1281 {
1282 int i;
1283
1284 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1285 {
1286 logf("Too many clauses");
1287 return false;
1288 }
1289
1290 /* Check if there is already a similar filter in present (filters are
1291 * much faster than clauses).
1292 */
1293 for (i = 0; i < tcs->filter_count; i++)
1294 {
1295 if (tcs->filter_tag[i] == clause->tag)
1296 return true;
1297 }
1298
1299 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1300 {
1301 char buf[MAX_PATH];
1302
1303 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1304 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1305 }
1306
1307 tcs->clause[tcs->clause_count] = clause;
1308 tcs->clause_count++;
1309
1310 return true;
1311 }
1312
1313 /* TODO: Remove this mess. */
1314 #ifdef HAVE_DIRCACHE
1315 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1316 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1317 #else
1318 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1319 #endif
1320
1321 static bool get_next(struct tagcache_search *tcs)
1322 {
1323 static char buf[TAG_MAXLEN+32];
1324 struct tagfile_entry entry;
1325 long flag = 0;
1326
1327 if (!tcs->valid || !tc_stat.ready)
1328 return false;
1329
1330 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1331 #ifdef HAVE_TC_RAMCACHE
1332 && !tcs->ramsearch
1333 #endif
1334 )
1335 return false;
1336
1337 /* Relative fetch. */
1338 if (tcs->filter_count > 0 || tcs->clause_count > 0
1339 || tagcache_is_numeric_tag(tcs->type))
1340 {
1341 /* Check for end of list. */
1342 if (tcs->seek_list_count == 0)
1343 {
1344 /* Try to fetch more. */
1345 if (!build_lookup_list(tcs))
1346 {
1347 tcs->valid = false;
1348 return false;
1349 }
1350 }
1351
1352 tcs->seek_list_count--;
1353 flag = tcs->seek_flags[tcs->seek_list_count];
1354
1355 /* Seek stream to the correct position and continue to direct fetch. */
1356 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1357 && !tagcache_is_numeric_tag(tcs->type))
1358 {
1359 if (!open_files(tcs, tcs->type))
1360 return false;
1361
1362 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1363 }
1364 else
1365 tcs->position = tcs->seek_list[tcs->seek_list_count];
1366 }
1367
1368 if (tagcache_is_numeric_tag(tcs->type))
1369 {
1370 snprintf(buf, sizeof(buf), "%d", tcs->position);
1371 tcs->result_seek = tcs->position;
1372 tcs->result = buf;
1373 tcs->result_len = strlen(buf) + 1;
1374 return true;
1375 }
1376
1377 /* Direct fetch. */
1378 #ifdef HAVE_TC_RAMCACHE
1379 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1380 {
1381 struct tagfile_entry *ep;
1382
1383 if (tcs->entry_count == 0)
1384 {
1385 tcs->valid = false;
1386 return false;
1387 }
1388 tcs->entry_count--;
1389
1390 tcs->result_seek = tcs->position;
1391
1392 # ifdef HAVE_DIRCACHE
1393 if (tcs->type == tag_filename)
1394 {
1395 dircache_copy_path((struct dirent *)tcs->position,
1396 buf, sizeof buf);
1397 tcs->result = buf;
1398 tcs->result_len = strlen(buf) + 1;
1399 tcs->idx_id = FLAG_GET_ATTR(flag);
1400 tcs->ramresult = false;
1401
1402 return true;
1403 }
1404 # endif
1405
1406 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1407 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1408 tcs->result = ep->tag_data;
1409 tcs->result_len = strlen(tcs->result) + 1;
1410 tcs->idx_id = ep->idx_id;
1411 tcs->ramresult = true;
1412
1413 return true;
1414 }
1415 else
1416 #endif
1417 {
1418 if (!open_files(tcs, tcs->type))
1419 return false;
1420
1421 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1422 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1423 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1424 {
1425 /* End of data. */
1426 tcs->valid = false;
1427 return false;
1428 }
1429 }
1430
1431 if (entry.tag_length > (long)sizeof(buf))
1432 {
1433 tcs->valid = false;
1434 logf("too long tag #2");
1435 return false;
1436 }
1437
1438 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1439 {
1440 tcs->valid = false;
1441 logf("read error #4");
1442 return false;
1443 }
1444
1445 tcs->result = buf;
1446 tcs->result_len = strlen(tcs->result) + 1;
1447 tcs->idx_id = entry.idx_id;
1448 tcs->ramresult = false;
1449
1450 return true;
1451 }
1452
1453 bool tagcache_get_next(struct tagcache_search *tcs)
1454 {
1455 while (get_next(tcs))
1456 {
1457 if (tcs->result_len > 1)
1458 return true;
1459 }
1460
1461 return false;
1462 }
1463
1464 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1465 int tag, char *buf, long size)
1466 {
1467 struct index_entry idx;
1468
1469 *buf = '\0';
1470 if (!get_index(tcs->masterfd, idxid, &idx, true))
1471 return false;
1472
1473 return retrieve(tcs, &idx, tag, buf, size);
1474 }
1475
1476 static bool update_master_header(void)
1477 {
1478 struct master_header myhdr;
1479 int fd;
1480
1481 if (!tc_stat.ready)
1482 return false;
1483
1484 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1485 return false;
1486
1487 myhdr.serial = current_tcmh.serial;
1488 myhdr.commitid = current_tcmh.commitid;
1489
1490 /* Write it back */
1491 lseek(fd, 0, SEEK_SET);
1492 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1493 close(fd);
1494
1495 #ifdef HAVE_TC_RAMCACHE
1496 if (hdr)
1497 {
1498 hdr->h.serial = current_tcmh.serial;
1499 hdr->h.commitid = current_tcmh.commitid;
1500 }
1501 #endif
1502
1503 return true;
1504 }
1505
1506 #if 0
1507
1508 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1509 {
1510 struct tagentry *entry;
1511
1512 if (tcs->type != tag_title)
1513 return ;
1514
1515 /* We will need reserve buffer for this. */
1516 if (tcs->ramcache)
1517 {
1518 struct tagfile_entry *ep;
1519
1520 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1521 tcs->seek_list[tcs->seek_list_count];
1522 }
1523
1524 entry = find_entry_ram();
1525
1526 }
1527 #endif
1528
1529 void tagcache_search_finish(struct tagcache_search *tcs)
1530 {
1531 int i;
1532
1533 if (!tcs->initialized)
1534 return;
1535
1536 if (tcs->masterfd >= 0)
1537 {
1538 close(tcs->masterfd);
1539 tcs->masterfd = -1;
1540 }
1541
1542 for (i = 0; i < TAG_COUNT; i++)
1543 {
1544 if (tcs->idxfd[i] >= 0)
1545 {
1546 close(tcs->idxfd[i]);
1547 tcs->idxfd[i] = -1;
1548 }
1549 }
1550
1551 tcs->ramsearch = false;
1552 tcs->valid = false;
1553 tcs->initialized = 0;
1554 if (write_lock > 0)
1555 write_lock--;
1556 }
1557
1558 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1559 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1560 {
1561 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1562 }
1563
1564 static long get_tag_numeric(const struct index_entry *entry, int tag)
1565 {
1566 return check_virtual_tags(tag, entry);
1567 }
1568
1569 static char* get_tag_string(const struct index_entry *entry, int tag)
1570 {
1571 char* s = get_tag(entry, tag)->tag_data;
1572 return strcmp(s, UNTAGGED) ? s : NULL;
1573 }
1574
1575 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1576 {
1577 struct index_entry *entry;
1578 int idx_id;
1579
1580 if (!tc_stat.ready)
1581 return false;
1582
1583 /* Find the corresponding entry in tagcache. */
1584 idx_id = find_entry_ram(filename, NULL);
1585 if (idx_id < 0 || !tc_stat.ramcache)
1586 return false;
1587
1588 entry = &hdr->indices[idx_id];
1589
1590 id3->title = get_tag_string(entry, tag_title);
1591 id3->artist = get_tag_string(entry, tag_artist);
1592 id3->album = get_tag_string(entry, tag_album);
1593 id3->genre_string = get_tag_string(entry, tag_genre);
1594 id3->composer = get_tag_string(entry, tag_composer);
1595 id3->comment = get_tag_string(entry, tag_comment);
1596 id3->albumartist = get_tag_string(entry, tag_albumartist);
1597 id3->grouping = get_tag_string(entry, tag_grouping);
1598
1599 id3->playcount = get_tag_numeric(entry, tag_playcount);
1600 id3->rating = get_tag_numeric(entry, tag_rating);
1601 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1602 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1603 id3->year = get_tag_numeric(entry, tag_year);
1604
1605 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1606 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1607 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1608 if (id3->bitrate == 0)
1609 id3->bitrate = 1;
1610
1611 return true;
1612 }
1613 #endif
1614
1615 static inline void write_item(const char *item)
1616 {
1617 int len = strlen(item) + 1;
1618
1619 data_size += len;
1620 write(cachefd, item, len);
1621 }
1622
1623 static int check_if_empty(char **tag)
1624 {
1625 int length;
1626
1627 if (*tag == NULL || **tag == '\0')
1628 {
1629 *tag = UNTAGGED;
1630 return sizeof(UNTAGGED); /* Tag length */
1631 }
1632
1633 length = strlen(*tag);
1634 if (length > TAG_MAXLEN)
1635 {
1636 logf("over length tag: %s", *tag);
1637 length = TAG_MAXLEN;
1638 (*tag)[length] = '\0';
1639 }
1640
1641 return length + 1;
1642 }
1643
1644 #define ADD_TAG(entry,tag,data) \
1645 /* Adding tag */ \
1646 entry.tag_offset[tag] = offset; \
1647 entry.tag_length[tag] = check_if_empty(data); \
1648 offset += entry.tag_length[tag]
1649
1650 static void add_tagcache(char *path, unsigned long mtime
1651 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1652 ,const struct dirent *dc
1653 #endif
1654 )
1655 {
1656 struct mp3entry id3;
1657 struct temp_file_entry entry;
1658 bool ret;
1659 int fd;
1660 int idx_id = -1;
1661 char tracknumfix[3];
1662 int offset = 0;
1663 int path_length = strlen(path);
1664 bool has_albumartist;
1665 bool has_grouping;
1666
1667 if (cachefd < 0)
1668 return ;
1669
1670 /* Check for overlength file path. */
1671 if (path_length > TAG_MAXLEN)
1672 {
1673 /* Path can't be shortened. */
1674 logf("Too long path: %s", path);
1675 return ;
1676 }
1677
1678 /* Check if the file is supported. */
1679 if (probe_file_format(path) == AFMT_UNKNOWN)
1680 return ;
1681
1682 /* Check if the file is already cached. */
1683 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1684 if (tc_stat.ramcache && is_dircache_intact())
1685 {
1686 idx_id = find_entry_ram(path, dc);
1687 }
1688 else
1689 #endif
1690 {
1691 if (filenametag_fd >= 0)
1692 {
1693 idx_id = find_entry_disk(path);
1694 }
1695 }
1696
1697 /* Check if file has been modified. */
1698 if (idx_id >= 0)
1699 {
1700 struct index_entry idx;
1701
1702 /* TODO: Mark that the index exists (for fast reverse scan) */
1703 //found_idx[idx_id/8] |= idx_id%8;
1704
1705 if (!get_index(-1, idx_id, &idx, true))
1706 {
1707 logf("failed to retrieve index entry");
1708 return ;
1709 }
1710
1711 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1712 {
1713 /* No changes to file. */
1714 return ;
1715 }
1716
1717 /* Metadata might have been changed. Delete the entry. */
1718 logf("Re-adding: %s", path);
1719 if (!delete_entry(idx_id))
1720 {
1721 logf("delete_entry failed: %d", idx_id);
1722 return ;
1723 }
1724 }
1725
1726 fd = open(path, O_RDONLY);
1727 if (fd < 0)
1728 {
1729 logf("open fail: %s", path);
1730 return ;
1731 }
1732
1733 memset(&id3, 0, sizeof(struct mp3entry));
1734 memset(&entry, 0, sizeof(struct temp_file_entry));
1735 memset(&tracknumfix, 0, sizeof(tracknumfix));
1736 ret = get_metadata(&id3, fd, path);
1737 close(fd);
1738
1739 if (!ret)
1740 return ;
1741
1742 logf("-> %s", path);
1743
1744 /* Generate track number if missing. */
1745 if (id3.tracknum <= 0)
1746 {
1747 const char *p = strrchr(path, '.');
1748
1749 if (p == NULL)
1750 p = &path[strlen(path)-1];
1751
1752 while (*p != '/')
1753 {
1754 if (isdigit(*p) && isdigit(*(p-1)))
1755 {
1756 tracknumfix[1] = *p--;
1757 tracknumfix[0] = *p;
1758 break;
1759 }
1760 p--;
1761 }
1762
1763 if (tracknumfix[0] != '\0')
1764 {
1765 id3.tracknum = atoi(tracknumfix);
1766 /* Set a flag to indicate track number has been generated. */
1767 entry.flag |= FLAG_TRKNUMGEN;
1768 }
1769 else
1770 {
1771 /* Unable to generate track number. */
1772 id3.tracknum = -1;
1773 }
1774 }
1775
1776 /* Numeric tags */
1777 entry.tag_offset[tag_year] = id3.year;
1778 entry.tag_offset[tag_discnumber] = id3.discnum;
1779 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1780 entry.tag_offset[tag_length] = id3.length;
1781 entry.tag_offset[tag_bitrate] = id3.bitrate;
1782 entry.tag_offset[tag_mtime] = mtime;
1783
1784 /* String tags. */
1785 has_albumartist = id3.albumartist != NULL
1786 && strlen(id3.albumartist) > 0;
1787 has_grouping = id3.grouping != NULL
1788 && strlen(id3.grouping) > 0;
1789
1790 ADD_TAG(entry, tag_filename, &path);
1791 ADD_TAG(entry, tag_title, &id3.title);
1792 ADD_TAG(entry, tag_artist, &id3.artist);
1793 ADD_TAG(entry, tag_album, &id3.album);
1794 ADD_TAG(entry, tag_genre, &id3.genre_string);
1795 ADD_TAG(entry, tag_composer, &id3.composer);
1796 ADD_TAG(entry, tag_comment, &id3.comment);
1797 if (has_albumartist)
1798 {
1799 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1800 }
1801 else
1802 {
1803 ADD_TAG(entry, tag_albumartist, &id3.artist);
1804 }
1805 if (has_grouping)
1806 {
1807 ADD_TAG(entry, tag_grouping, &id3.grouping);
1808 }
1809 else
1810 {
1811 ADD_TAG(entry, tag_grouping, &id3.title);
1812 }
1813 entry.data_length = offset;
1814
1815 /* Write the header */
1816 write(cachefd, &entry, sizeof(struct temp_file_entry));
1817
1818 /* And tags also... Correct order is critical */
1819 write_item(path);
1820 write_item(id3.title);
1821 write_item(id3.artist);
1822 write_item(id3.album);
1823 write_item(id3.genre_string);
1824 write_item(id3.composer);
1825 write_item(id3.comment);
1826 if (has_albumartist)
1827 {
1828 write_item(id3.albumartist);
1829 }
1830 else
1831 {
1832 write_item(id3.artist);
1833 }
1834 if (has_grouping)
1835 {
1836 write_item(id3.grouping);
1837 }
1838 else
1839 {
1840 write_item(id3.title);
1841 }
1842 total_entry_count++;
1843 }
1844
1845 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1846 {
1847 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1848 int len = strlen(str)+1;
1849 int i;
1850 unsigned crc32;
1851 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1852 char buf[TAG_MAXLEN+32];
1853
1854 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1855 buf[i] = tolower(str[i]);
1856 buf[i] = '\0';
1857
1858 crc32 = crc_32(buf, i, 0xffffffff);
1859
1860 if (unique)
1861 {
1862 /* Check if the crc does not exist -> entry does not exist for sure. */
1863 for (i = 0; i < tempbufidx; i++)
1864 {
1865 if (crcbuf[-i] != crc32)
1866 continue;
1867
1868 if (!strcasecmp(str, index[i].str))
1869 {
1870 if (id < 0 || id >= lookup_buffer_depth)
1871 {
1872 logf("lookup buf overf.: %d", id);
1873 return false;
1874 }
1875
1876 lookup[id] = &index[i];
1877 return true;
1878 }
1879 }
1880 }
1881
1882 /* Insert to CRC buffer. */
1883 crcbuf[-tempbufidx] = crc32;
1884 tempbuf_left -= 4;
1885
1886 /* Insert it to the buffer. */
1887 tempbuf_left -= len;
1888 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1889 return false;
1890
1891 if (id >= lookup_buffer_depth)
1892 {
1893 logf("lookup buf overf. #2: %d", id);
1894 return false;
1895 }
1896
1897 if (id >= 0)
1898 {
1899 lookup[id] = &index[tempbufidx];
1900 index[tempbufidx].idlist.id = id;
1901 }
1902 else
1903 index[tempbufidx].idlist.id = -1;
1904
1905 index[tempbufidx].idlist.next = NULL;
1906 index[tempbufidx].idx_id = idx_id;
1907 index[tempbufidx].seek = -1;
1908 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1909 memcpy(index[tempbufidx].str, str, len);
1910 tempbuf_pos += len;
1911 tempbufidx++;
1912
1913 return true;
1914 }
1915
1916 static int compare(const void *p1, const void *p2)
1917 {
1918 do_timed_yield();
1919
1920 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1921 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1922
1923 if (strcmp(e1->str, UNTAGGED) == 0)
1924 {
1925 if (strcmp(e2->str, UNTAGGED) == 0)
1926 return 0;
1927 return -1;
1928 }
1929 else if (strcmp(e2->str, UNTAGGED) == 0)
1930 return 1;
1931
1932 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1933 }
1934
1935 static int tempbuf_sort(int fd)
1936 {
1937 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1938 struct tagfile_entry fe;
1939 int i;
1940 int length;
1941
1942 /* Generate reverse lookup entries. */
1943 for (i = 0; i < lookup_buffer_depth; i++)
1944 {
1945 struct tempbuf_id_list *idlist;
1946
1947 if (!lookup[i])
1948 continue;
1949
1950 if (lookup[i]->idlist.id == i)
1951 continue;
1952
1953 idlist = &lookup[i]->idlist;
1954 while (idlist->next != NULL)
1955 idlist = idlist->next;
1956
1957 tempbuf_left -= sizeof(struct tempbuf_id_list);
1958 if (tempbuf_left - 4 < 0)
1959 return -1;
1960
1961 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1962 if (tempbuf_pos & 0x03)
1963 {
1964 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1965 tempbuf_left -= 3;
1966 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1967 }
1968 tempbuf_pos += sizeof(struct tempbuf_id_list);
1969
1970 idlist = idlist->next;
1971 idlist->id = i;
1972 idlist->next = NULL;
1973
1974 do_timed_yield();
1975 }
1976
1977 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1978 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1979
1980 for (i = 0; i < tempbufidx; i++)
1981 {
1982 struct tempbuf_id_list *idlist = &index[i].idlist;
1983
1984 /* Fix the lookup list. */
1985 while (idlist != NULL)
1986 {
1987 if (idlist->id >= 0)
1988 lookup[idlist->id] = &index[i];
1989 idlist = idlist->next;
1990 }
1991
1992 index[i].seek = lseek(fd, 0, SEEK_CUR);
1993 length = strlen(index[i].str) + 1;
1994 fe.tag_length = length;
1995 fe.idx_id = index[i].idx_id;
1996
1997 /* Check the chunk alignment. */
1998 if ((fe.tag_length + sizeof(struct tagfile_entry))
1999 % TAGFILE_ENTRY_CHUNK_LENGTH)
2000 {
2001 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2002 ((fe.tag_length + sizeof(struct tagfile_entry))
2003 % TAGFILE_ENTRY_CHUNK_LENGTH);
2004 }
2005
2006 #ifdef TAGCACHE_STRICT_ALIGN
2007 /* Make sure the entry is long aligned. */
2008 if (index[i].seek & 0x03)
2009 {
2010 logf("tempbuf_sort: alignment error!");
2011 return -3;
2012 }
2013 #endif
2014
2015 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2016 sizeof(struct tagfile_entry))
2017 {
2018 logf("tempbuf_sort: write error #1");
2019 return -1;
2020 }
2021
2022 if (write(fd, index[i].str, length) != length)
2023 {
2024 logf("tempbuf_sort: write error #2");
2025 return -2;
2026 }
2027
2028 /* Write some padding. */
2029 if (fe.tag_length - length > 0)
2030 write(fd, "XXXXXXXX", fe.tag_length - length);
2031 }
2032
2033 return i;
2034 }
2035
2036 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2037 {
2038 if (id < 0 || id >= lookup_buffer_depth)
2039 return NULL;
2040
2041 return lookup[id];
2042 }
2043
2044
2045 inline static int tempbuf_find_location(int id)
2046 {
2047 struct tempbuf_searchidx *entry;
2048
2049 entry = tempbuf_locate(id);
2050 if (entry == NULL)
2051 return -1;
2052
2053 return entry->seek;
2054 }
2055
2056 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2057 {
2058 struct master_header tcmh;
2059 struct index_entry idx;
2060 int masterfd;
2061 int masterfd_pos;
2062 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2063 int max_entries;
2064 int entries_processed = 0;
2065 int i, j;
2066 char buf[TAG_MAXLEN];
2067
2068 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2069
2070 logf("Building numeric indices...");
2071 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2072
2073 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2074 return false;
2075
2076 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2077 SEEK_CUR);
2078 if (masterfd_pos == filesize(masterfd))
2079 {
2080 logf("we can't append!");
2081 close(masterfd);
2082 return false;
2083 }
2084
2085 while (entries_processed < h->entry_count)
2086 {
2087 int count = MIN(h->entry_count - entries_processed, max_entries);
2088
2089 /* Read in as many entries as possible. */
2090 for (i = 0; i < count; i++)
2091 {
2092 struct temp_file_entry *tfe = &entrybuf[i];
2093 int datastart;
2094
2095 /* Read in numeric data. */
2096 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2097 sizeof(struct temp_file_entry))
2098 {
2099 logf("read fail #1");
2100 close(masterfd);
2101 return false;
2102 }
2103
2104 datastart = lseek(tmpfd, 0, SEEK_CUR);
2105
2106 /**
2107 * Read string data from the following tags:
2108 * - tag_filename
2109 * - tag_artist
2110 * - tag_album
2111 * - tag_title
2112 *
2113 * A crc32 hash is calculated from the read data
2114 * and stored back to the data offset field kept in memory.
2115 */
2116 #define tmpdb_read_string_tag(tag) \
2117 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2118 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2119 { \
2120 logf("read fail: buffer overflow"); \
2121 close(masterfd); \
2122 return false; \
2123 } \
2124 \
2125 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2126 tfe->tag_length[tag]) \
2127 { \
2128 logf("read fail #2"); \
2129 close(masterfd); \
2130 return false; \
2131 } \
2132 \
2133 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2134 lseek(tmpfd, datastart, SEEK_SET)
2135
2136 tmpdb_read_string_tag(tag_filename);
2137 tmpdb_read_string_tag(tag_artist);
2138 tmpdb_read_string_tag(tag_album);
2139 tmpdb_read_string_tag(tag_title);
2140
2141 /* Seek to the end of the string data. */
2142 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2143 }
2144
2145 /* Backup the master index position. */
2146 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2147 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2148
2149 /* Check if we can resurrect some deleted runtime statistics data. */
2150 for (i = 0; i < tcmh.tch.entry_count; i++)
2151 {
2152 /* Read the index entry. */
2153 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2154 != sizeof(struct index_entry))
2155 {
2156 logf("read fail #3");
2157 close(masterfd);
2158 return false;
2159 }
2160
2161 /**
2162 * Skip unless the entry is marked as being deleted
2163 * or the data has already been resurrected.
2164 */
2165 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2166 continue;
2167
2168 /* Now try to match the entry. */
2169 /**
2170 * To succesfully match a song, the following conditions
2171 * must apply:
2172 *
2173 * For numeric fields: tag_length
2174 * - Full identical match is required
2175 *
2176 * If tag_filename matches, no further checking necessary.
2177 *
2178 * For string hashes: tag_artist, tag_album, tag_title
2179 * - Two of these must match
2180 */
2181 for (j = 0; j < count; j++)
2182 {
2183 struct temp_file_entry *tfe = &entrybuf[j];
2184
2185 /* Try to match numeric fields first. */
2186 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2187 continue;
2188
2189 /* Now it's time to do the hash matching. */
2190 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2191 {
2192 int match_count = 0;
2193
2194 /* No filename match, check if we can match two other tags. */
2195 #define tmpdb_match(tag) \
2196 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2197 match_count++
2198
2199 tmpdb_match(tag_artist);
2200 tmpdb_match(tag_album);
2201 tmpdb_match(tag_title);
2202
2203 if (match_count < 2)
2204 {
2205 /* Still no match found, give up. */
2206 continue;
2207 }
2208 }
2209
2210 /* A match found, now copy & resurrect the statistical data. */
2211 #define tmpdb_copy_tag(tag) \
2212 tfe->tag_offset[tag] = idx.tag_seek[tag]
2213
2214 tmpdb_copy_tag(tag_playcount);
2215 tmpdb_copy_tag(tag_rating);
2216 tmpdb_copy_tag(tag_playtime);
2217 tmpdb_copy_tag(tag_lastplayed);
2218 tmpdb_copy_tag(tag_commitid);
2219
2220 /* Avoid processing this entry again. */
2221 idx.flag |= FLAG_RESURRECTED;
2222
2223 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2224 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2225 != sizeof(struct index_entry))
2226 {
2227 logf("masterfd writeback fail #1");
2228 close(masterfd);
2229 return false;
2230 }
2231
2232 logf("Entry resurrected");
2233 }
2234 }
2235
2236
2237 /* Restore the master index position. */
2238 lseek(masterfd, masterfd_pos, SEEK_SET);
2239
2240 /* Commit the data to the index. */
2241 for (i = 0; i < count; i++)
2242 {
2243 int loc = lseek(masterfd, 0, SEEK_CUR);
2244
2245 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2246 != sizeof(struct index_entry))
2247 {
2248 logf("read fail #3");
2249 close(masterfd);
2250 return false;
2251 }
2252
2253 for (j = 0; j < TAG_COUNT; j++)
2254 {
2255 if (!tagcache_is_numeric_tag(j))
2256 continue;
2257
2258 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2259 }
2260 idx.flag = entrybuf[i].flag;
2261
2262 if (idx.tag_seek[tag_commitid])
2263 {
2264 /* Data has been resurrected. */
2265 idx.flag |= FLAG_DIRTYNUM;
2266 }
2267 else if (tc_stat.ready && current_tcmh.commitid > 0)
2268 {
2269 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2270 idx.flag |= FLAG_DIRTYNUM;
2271 }
2272
2273 /* Write back the updated index. */
2274 lseek(masterfd, loc, SEEK_SET);
2275 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2276 != sizeof(struct index_entry))
2277 {
2278 logf("write fail");
2279 close(masterfd);
2280 return false;
2281 }
2282 }
2283
2284 entries_processed += count;
2285 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2286 }
2287
2288 close(masterfd);
2289
2290 return true;
2291 }
2292
2293 /**
2294 * Return values:
2295 * > 0 success
2296 * == 0 temporary failure
2297 * < 0 fatal error
2298 */
2299 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2300 {
2301 int i;
2302 struct tagcache_header tch;
2303 struct master_header tcmh;
2304 struct index_entry idxbuf[IDX_BUF_DEPTH];
2305 int idxbuf_pos;
2306 char buf[TAG_MAXLEN+32];
2307 int fd = -1, masterfd;
2308 bool error = false;
2309 int init;
2310 int masterfd_pos;
2311
2312 logf("Building index: %d", index_type);
2313
2314 /* Check the number of entries we need to allocate ram for. */
2315 commit_entry_count = h->entry_count + 1;
2316
2317 masterfd = open_master_fd(&tcmh, false);
2318 if (masterfd >= 0)
2319 {
2320 commit_entry_count += tcmh.tch.entry_count;
2321 close(masterfd);
2322 }
2323 else
2324 remove_files(); /* Just to be sure we are clean. */
2325
2326 /* Open the index file, which contains the tag names. */
2327 fd = open_tag_fd(&tch, index_type, true);
2328 if (fd >= 0)
2329 {
2330 logf("tch.datasize=%ld", tch.datasize);
2331 lookup_buffer_depth = 1 +
2332 /* First part */ commit_entry_count +
2333 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2334 }
2335 else
2336 {
2337 lookup_buffer_depth = 1 +
2338 /* First part */ commit_entry_count +
2339 /* Second part */ 0;
2340 }
2341
2342 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2343 logf("commit_entry_count=%ld", commit_entry_count);
2344
2345 /* Allocate buffer for all index entries from both old and new
2346 * tag files. */
2347 tempbufidx = 0;
2348 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2349
2350 /* Allocate lookup buffer. The first portion of commit_entry_count
2351 * contains the new tags in the temporary file and the second
2352 * part for locating entries already in the db.
2353 *
2354 * New tags Old tags
2355 * +---------+---------------------------+
2356 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2357 * +---------+---------------------------+
2358 *
2359 * Old tags are inserted to a temporary buffer with position:
2360 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2361 * And new tags with index:
2362 * tempbuf_insert(idx, ...);
2363 *
2364 * The buffer is sorted and written into tag file:
2365 * tempbuf_sort(...);
2366 * leaving master index locations messed up.
2367 *
2368 * That is fixed using the lookup buffer for old tags:
2369 * new_seek = tempbuf_find_location(old_seek, ...);
2370 * and for new tags:
2371 * new_seek = tempbuf_find_location(idx);
2372 */
2373 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2374 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2375 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2376
2377 /* And calculate the remaining data space used mainly for storing
2378 * tag data (strings). */
2379 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2380 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2381 {
2382 logf("Buffer way too small!");
2383 return 0;
2384 }
2385
2386 if (fd >= 0)
2387 {
2388 /**
2389 * If tag file contains unique tags (sorted index), we will load
2390 * it entirely into memory so we can resort it later for use with
2391 * chunked browsing.
2392 */
2393 if (tagcache_is_sorted_tag(index_type))
2394 {
2395 logf("loading tags...");
2396 for (i = 0; i < tch.entry_count; i++)
2397 {
2398 struct tagfile_entry entry;
2399 int loc = lseek(fd, 0, SEEK_CUR);
2400 bool ret;
2401
2402 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2403 != sizeof(struct tagfile_entry))
2404 {
2405 logf("read error #7");
2406 close(fd);
2407 return -2;
2408 }
2409
2410 if (entry.tag_length >= (int)sizeof(buf))
2411 {
2412 logf("too long tag #3");
2413 close(fd);
2414 return -2;
2415 }
2416
2417 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2418 {
2419 logf("read error #8");
2420 close(fd);
2421 return -2;
2422 }
2423
2424 /* Skip deleted entries. */
2425 if (buf[0] == '\0')
2426 continue;
2427
2428 /**
2429 * Save the tag and tag id in the memory buffer. Tag id
2430 * is saved so we can later reindex the master lookup
2431 * table when the index gets resorted.
2432 */
2433 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2434 + commit_entry_count, entry.idx_id,
2435 tagcache_is_unique_tag(index_type));
2436 if (!ret)
2437 {
2438 close(fd);
2439 return -3;
2440 }
2441 do_timed_yield();
2442 }
2443 logf("done");
2444 }
2445 else
2446 tempbufidx = tch.entry_count;
2447 }
2448 else
2449 {
2450 /**
2451 * Creating new index file to store the tags. No need to preload
2452 * anything whether the index type is sorted or not.
2453 */
2454 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2455 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2456 if (fd < 0)
2457 {
2458 logf("%s open fail", buf);
2459 return -2;
2460 }
2461
2462 tch.magic = TAGCACHE_MAGIC;
2463 tch.entry_count = 0;
2464 tch.datasize = 0;
2465
2466 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2467 != sizeof(struct tagcache_header))
2468 {
2469 logf("header write failed");
2470 close(fd);
2471 return -2;
2472 }
2473 }
2474
2475 /* Loading the tag lookup file as "master file". */
2476 logf("Loading index file");
2477 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2478
2479 if (masterfd < 0)
2480 {
2481 logf("Creating new DB");
2482 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2483
2484 if (masterfd < 0)
2485 {
2486 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2487 close(fd);
2488 return -2;
2489 }
2490
2491 /* Write the header (write real values later). */
2492 memset(&tcmh, 0, sizeof(struct master_header));
2493 tcmh.tch = *h;
2494 tcmh.tch.entry_count = 0;
2495 tcmh.tch.datasize = 0;
2496 tcmh.dirty = true;
2497 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2498 init = true;
2499 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2500 }
2501 else
2502 {
2503 /**
2504 * Master file already exists so we need to process the current
2505 * file first.
2506 */
2507 init = false;
2508
2509 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2510 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2511 {
2512 logf("header error");
2513 close(fd);
2514 close(masterfd);
2515 return -2;
2516 }
2517
2518 /**
2519 * If we reach end of the master file, we need to expand it to
2520 * hold new tags. If the current index is not sorted, we can
2521 * simply append new data to end of the file.
2522 * However, if the index is sorted, we need to update all tag
2523 * pointers in the master file for the current index.
2524 */
2525 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2526 SEEK_CUR);
2527 if (masterfd_pos == filesize(masterfd))
2528 {
2529 logf("appending...");
2530 init = true;
2531 }
2532 }
2533
2534 /**
2535 * Load new unique tags in memory to be sorted later and added
2536 * to the master lookup file.
2537 */
2538 if (tagcache_is_sorted_tag(index_type))
2539 {
2540 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2541 /* h is the header of the temporary file containing new tags. */
2542 logf("inserting new tags...");
2543 for (i = 0; i < h->entry_count; i++)
2544 {
2545 struct temp_file_entry entry;
2546
2547 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2548 sizeof(struct temp_file_entry))
2549 {
2550 logf("read fail #3");
2551 error = true;
2552 goto error_exit;
2553 }
2554
2555 /* Read data. */
2556 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2557 {
2558 logf("too long entry!");
2559 error = true;
2560 goto error_exit;
2561 }
2562
2563 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2564 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2565 entry.tag_length[index_type])
2566 {
2567 logf("read fail #4");
2568 error = true;
2569 goto error_exit;
2570 }
2571
2572 if (tagcache_is_unique_tag(index_type))
2573 error = !tempbuf_insert(buf, i, -1, true);
2574 else
2575 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2576
2577 if (error)
2578 {
2579 logf("insert error");
2580 goto error_exit;
2581 }
2582
2583 /* Skip to next. */
2584 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2585 entry.tag_length[index_type], SEEK_CUR);
2586 do_timed_yield();
2587 }
2588 logf("done");
2589
2590 /* Sort the buffer data and write it to the index file. */
2591 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2592 i = tempbuf_sort(fd);
2593 if (i < 0)
2594 goto error_exit;
2595 logf("sorted %d tags", i);
2596
2597 /**
2598 * Now update all indexes in the master lookup file.
2599 */
2600 logf("updating indices...");
2601 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2602 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2603 {
2604 int j;
2605 int loc = lseek(masterfd, 0, SEEK_CUR);
2606
2607 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2608
2609 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2610 != (int)sizeof(struct index_entry)*idxbuf_pos)
2611 {
2612 logf("read fail #5");
2613 error = true;
2614 goto error_exit ;
2615 }
2616 lseek(masterfd, loc, SEEK_SET);
2617
2618 for (j = 0; j < idxbuf_pos; j++)
2619 {
2620 if (idxbuf[j].flag & FLAG_DELETED)
2621 {
2622 /* We can just ignore deleted entries. */
2623 // idxbuf[j].tag_seek[index_type] = 0;
2624 continue;
2625 }
2626
2627 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2628 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2629 + commit_entry_count);
2630
2631 if (idxbuf[j].tag_seek[index_type] < 0)
2632 {
2633 logf("update error: %d/%d/%d",
2634 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2635 error = true;
2636 goto error_exit;
2637 }
2638
2639 do_timed_yield();
2640 }
2641
2642 /* Write back the updated index. */
2643 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2644 index_entry_ec, tc_stat.econ) !=
2645 (int)sizeof(struct index_entry)*idxbuf_pos)
2646 {
2647 logf("write fail");
2648 error = true;
2649 goto error_exit;
2650 }
2651 }
2652 logf("done");
2653 }
2654
2655 /**
2656 * Walk through the temporary file containing the new tags.
2657 */
2658 // build_normal_index(h, tmpfd, masterfd, idx);
2659 logf("updating new indices...");
2660 lseek(masterfd, masterfd_pos, SEEK_SET);
2661 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2662 lseek(fd, 0, SEEK_END);
2663 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2664 {
2665 int j;
2666
2667 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2668 if (init)
2669 {
2670 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2671 }
2672 else
2673 {
2674 int loc = lseek(masterfd, 0, SEEK_CUR);
2675
2676 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2677 != (int)sizeof(struct index_entry)*idxbuf_pos)
2678 {
2679 logf("read fail #6");
2680 error = true;
2681 break ;
2682 }
2683 lseek(masterfd, loc, SEEK_SET);
2684 }
2685
2686 /* Read entry headers. */
2687 for (j = 0; j < idxbuf_pos; j++)
2688 {
2689 if (!tagcache_is_sorted_tag(index_type))
2690 {
2691 struct temp_file_entry entry;
2692 struct tagfile_entry fe;
2693
2694 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2695 sizeof(struct temp_file_entry))
2696 {
2697 logf("read fail #7");
2698 error = true;
2699 break ;
2700 }
2701
2702 /* Read data. */
2703 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2704 {
2705 logf("too long entry!");
2706 logf("length=%d", entry.tag_length[index_type]);
2707 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2708 error = true;
2709 break ;
2710 }
2711
2712 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2713 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2714 entry.tag_length[index_type])
2715 {
2716 logf("read fail #8");
2717 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2718 logf("length=0x%02x", entry.tag_length[index_type]);
2719 error = true;
2720 break ;
2721 }
2722
2723 /* Write to index file. */
2724 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2725 fe.tag_length = entry.tag_length[index_type];
2726 fe.idx_id = tcmh.tch.entry_count + i + j;
2727 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2728 write(fd, buf, fe.tag_length);
2729 tempbufidx++;
2730
2731 /* Skip to next. */
2732 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2733 entry.tag_length[index_type], SEEK_CUR);
2734 }
2735 else
2736 {
2737 /* Locate the correct entry from the sorted array. */
2738 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2739 if (idxbuf[j].tag_seek[index_type] < 0)
2740 {
2741 logf("entry not found (%d)", j);
2742 error = true;
2743 break ;
2744 }
2745 }
2746 }
2747
2748 /* Write index. */
2749 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2750 index_entry_ec, tc_stat.econ) !=
2751 (int)sizeof(struct index_entry)*idxbuf_pos)
2752 {
2753 logf("tagcache: write fail #4");
2754 error = true;
2755 break ;
2756 }
2757
2758 do_timed_yield();
2759 }
2760 logf("done");
2761
2762 /* Finally write the header. */
2763 tch.magic = TAGCACHE_MAGIC;
2764 tch.entry_count = tempbufidx;
2765 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2766 lseek(fd, 0, SEEK_SET);
2767 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2768
2769 if (index_type != tag_filename)
2770 h->datasize += tch.datasize;
2771 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2772 error_exit:
2773
2774 close(fd);
2775 close(masterfd);
2776
2777 if (error)
2778 return -2;
2779
2780 return 1;
2781 }
2782
2783 static bool commit(void)
2784 {
2785 struct tagcache_header tch;
2786 struct master_header tcmh;
2787 int i, len, rc;
2788 int tmpfd;
2789 int masterfd;
2790 #ifdef HAVE_DIRCACHE
2791 bool dircache_buffer_stolen = false;
2792 #endif
2793 bool local_allocation = false;
2794
2795 logf("committing tagcache");
2796
2797 while (write_lock)
2798 sleep(1);
2799
2800 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2801 if (tmpfd < 0)
2802 {
2803 logf("nothing to commit");
2804 return true;
2805 }
2806
2807
2808 /* Load the header. */
2809 len = sizeof(struct tagcache_header);
2810 rc = read(tmpfd, &tch, len);
2811
2812 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2813 {
2814 logf("incorrect header");
2815 close(tmpfd);
2816 remove(TAGCACHE_FILE_TEMP);
2817 return false;
2818 }
2819
2820 if (tch.entry_count == 0)
2821 {
2822 logf("nothing to commit");
2823 close(tmpfd);
2824 remove(TAGCACHE_FILE_TEMP);
2825 return true;
2826 }
2827
2828 #ifdef HAVE_EEPROM_SETTINGS
2829 remove(TAGCACHE_STATEFILE);
2830 #endif
2831
2832 /* At first be sure to unload the ramcache! */
2833 #ifdef HAVE_TC_RAMCACHE
2834 tc_stat.ramcache = false;
2835 #endif
2836
2837 read_lock++;
2838
2839 /* Try to steal every buffer we can :) */
2840 if (tempbuf_size == 0)
2841 local_allocation = true;
2842
2843 #ifdef HAVE_DIRCACHE
2844 if (tempbuf_size == 0)
2845 {
2846 /* Try to steal the dircache buffer. */
2847 tempbuf = dircache_steal_buffer(&tempbuf_size);
2848 tempbuf_size &= ~0x03;
2849
2850 if (tempbuf_size > 0)
2851 {
2852 dircache_buffer_stolen = true;
2853 }
2854 }
2855 #endif
2856
2857 #ifdef HAVE_TC_RAMCACHE
2858 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2859 {
2860 tempbuf = (char *)(hdr + 1);
2861 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2862 tempbuf_size &= ~0x03;
2863 }
2864 #endif
2865
2866 /* And finally fail if there are no buffers available. */
2867 if (tempbuf_size == 0)
2868 {
2869 logf("delaying commit until next boot");
2870 tc_stat.commit_delayed = true;
2871 close(tmpfd);
2872 read_lock--;
2873 return false;
2874 }
2875
2876 logf("commit %ld entries...", tch.entry_count);
2877
2878 /* Mark DB dirty so it will stay disabled if commit fails. */
2879 current_tcmh.dirty = true;
2880 update_master_header();
2881
2882 /* Now create the index files. */
2883 tc_stat.commit_step = 0;
2884 tch.datasize = 0;
2885 tc_stat.commit_delayed = false;
2886
2887 for (i = 0; i < TAG_COUNT; i++)
2888 {
2889 int ret;
2890
2891 if (tagcache_is_numeric_tag(i))
2892 continue;
2893
2894 tc_stat.commit_step++;
2895 ret = build_index(i, &tch, tmpfd);
2896 if (ret <= 0)
2897 {
2898 close(tmpfd);
2899 logf("tagcache failed init");
2900 if (ret < 0)
2901 remove_files();
2902 else
2903 tc_stat.commit_delayed = true;
2904 tc_stat.commit_step = 0;
2905 read_lock--;
2906 return false;
2907 }
2908 }
2909
2910 if (!build_numeric_indices(&tch, tmpfd))
2911 {
2912 logf("Failure to commit numeric indices");
2913 close(tmpfd);
2914 remove_files();
2915 tc_stat.commit_step = 0;
2916 read_lock--;
2917 return false;
2918 }
2919
2920 close(tmpfd);
2921 tc_stat.commit_step = 0;
2922
2923 /* Update the master index headers. */
2924 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2925 {
2926 read_lock--;
2927 return false;
2928 }
2929
2930 tcmh.tch.entry_count += tch.entry_count;
2931 tcmh.tch.datasize = sizeof(struct master_header)
2932 + sizeof(struct index_entry) * tcmh.tch.entry_count
2933 + tch.datasize;
2934 tcmh.dirty = false;
2935 tcmh.commitid++;
2936
2937 lseek(masterfd, 0, SEEK_SET);
2938 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2939 close(masterfd);
2940
2941 logf("tagcache committed");
2942 remove(TAGCACHE_FILE_TEMP);
2943 tc_stat.ready = check_all_headers();
2944 tc_stat.readyvalid = true;
2945
2946 if (local_allocation)
2947 {
2948 tempbuf = NULL;
2949 tempbuf_size = 0;
2950 }
2951
2952 #ifdef HAVE_DIRCACHE
2953 /* Rebuild the dircache, if we stole the buffer. */
2954 if (dircache_buffer_stolen)
2955 dircache_build(0);
2956 #endif
2957
2958 #ifdef HAVE_TC_RAMCACHE
2959 /* Reload tagcache. */
2960 if (tc_stat.ramcache_allocated > 0)
2961 tagcache_start_scan();
2962 #endif
2963
2964 read_lock--;
2965
2966 return true;
2967 }
2968
2969 static void allocate_tempbuf(void)
2970 {
2971 /* Yeah, malloc would be really nice now :) */
2972 #ifdef __PCTOOL__
2973 tempbuf_size = 32*1024*1024;
2974 tempbuf = malloc(tempbuf_size);
2975 #else
2976 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2977 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2978 audiobuf += tempbuf_size;
2979 #endif
2980 }
2981
2982 static void free_tempbuf(void)
2983 {
2984 if (tempbuf_size == 0)
2985 return ;
2986
2987 #ifdef __PCTOOL__
2988 free(tempbuf);
2989 #else
2990 audiobuf -= tempbuf_size;
2991 #endif
2992 tempbuf = NULL;
2993 tempbuf_size = 0;
2994 }
2995
2996 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
2997 {
2998 struct index_entry idx;
2999
3000 if (!tc_stat.ready)
3001 return false;
3002
3003 if (!tagcache_is_numeric_tag(tag))
3004 return false;
3005
3006 if (!get_index(masterfd, idx_id, &idx, false))
3007 return false;
3008
3009 idx.tag_seek[tag] = data;
3010 idx.flag |= FLAG_DIRTYNUM;
3011
3012 return write_index(masterfd, idx_id, &idx);
3013 }
3014
3015 #if 0
3016 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3017 int tag, long data)
3018 {
3019 struct master_header myhdr;
3020
3021 if (tcs->masterfd < 0)
3022 {
3023 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3024 return false;
3025 }
3026
3027 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3028 }
3029 #endif
3030
3031 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3032
3033 static bool command_queue_is_full(void)
3034 {
3035 int next;
3036
3037 next = command_queue_widx + 1;
3038 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3039 next = 0;
3040
3041 return (next == command_queue_ridx);
3042 }
3043 static bool command_queue_sync_callback(void)
3044 {
3045
3046 struct master_header myhdr;
3047 int masterfd;
3048
3049 mutex_lock(&command_queue_mutex);
3050
3051 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3052 return false;
3053
3054 while (command_queue_ridx != command_queue_widx)
3055 {
3056 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3057
3058 switch (ce->command)
3059 {
3060 case CMD_UPDATE_MASTER_HEADER:
3061 {
3062 close(masterfd);
3063 update_master_header();
3064
3065 /* Re-open the masterfd. */
3066 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3067 return true;
3068
3069 break;
3070 }
3071 case CMD_UPDATE_NUMERIC:
3072 {
3073 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3074 break;
3075 }
3076 }
3077
3078 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3079 command_queue_ridx = 0;
3080 }
3081
3082 close(masterfd);
3083
3084 tc_stat.queue_length = 0;
3085 mutex_unlock(&command_queue_mutex);
3086 return true;
3087 }
3088
3089 static void run_command_queue(bool force)
3090 {
3091 if (COMMAND_QUEUE_IS_EMPTY)
3092 return;
3093
3094 if (force || command_queue_is_full())
3095 command_queue_sync_callback();
3096 else
3097 register_ata_idle_func(command_queue_sync_callback);
3098 }
3099
3100 static void queue_command(int cmd, long idx_id, int tag, long data)
3101 {
3102 while (1)
3103 {
3104 int next;
3105
3106 mutex_lock(&command_queue_mutex);
3107 next = command_queue_widx + 1;
3108 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3109 next = 0;
3110
3111 /* Make sure queue is not full. */
3112 if (next != command_queue_ridx)
3113 {
3114 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3115
3116 ce->command = cmd;
3117 ce->idx_id = idx_id;
3118 ce->tag = tag;
3119 ce->data = data;
3120
3121 command_queue_widx = next;
3122
3123 tc_stat.queue_length++;
3124
3125 mutex_unlock(&command_queue_mutex);
3126 break;
3127 }
3128
3129 /* Queue is full, try again later... */
3130 mutex_unlock(&command_queue_mutex);
3131 sleep(1);
3132 }
3133 }
3134
3135 long tagcache_increase_serial(void)
3136 {
3137 long old;
3138
3139 if (!tc_stat.ready)
3140 return -2;
3141
3142 while (read_lock)
3143 sleep(1);
3144
3145 old = current_tcmh.serial++;
3146 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3147
3148 return old;
3149 }
3150
3151 void tagcache_update_numeric(int idx_id, int tag, long data)
3152 {
3153 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3154 }
3155
3156 long tagcache_get_serial(void)
3157 {
3158 return current_tcmh.serial;
3159 }
3160
3161 long tagcache_get_commitid(void)
3162 {
3163 return current_tcmh.commitid;
3164 }
3165
3166 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3167 {
3168 char buf[512];
3169 int i;
3170
3171 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3172 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3173 {
3174 if (*datastr == '\0')
3175 break;
3176
3177 if (*datastr == '"' || *datastr == '\\')
3178 buf[i++] = '\\';
3179
3180 buf[i] = *(datastr++);
3181 }
3182
3183 strcpy(&buf[i], "\" ");
3184
3185 write(fd, buf, i + 2);
3186
3187 return true;
3188 }
3189
3190 static bool read_tag(char *dest, long size,
3191 const char *src, const char *tagstr)
3192 {
3193 int pos;
3194 char current_tag[32];
3195
3196 while (*src != '\0')
3197 {
3198 /* Skip all whitespace */
3199 while (*src == ' ')
3200 src++;
3201
3202 if (*src == '\0')
3203 break;
3204
3205 pos = 0;
3206 /* Read in tag name */
3207 while (*src != '=' && *src != ' ')
3208 {
3209 current_tag[pos] = *src;
3210 src++;
3211 pos++;
3212
3213 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3214 return false;
3215 }
3216 current_tag[pos] = '\0';
3217
3218 /* Read in tag data */
3219
3220 /* Find the start. */
3221 while (*src != '"' && *src != '\0')
3222 src++;
3223
3224 if (*src == '\0' || *(++src) == '\0')
3225 return false;
3226
3227 /* Read the data. */
3228 for (pos = 0; pos < size; pos++)
3229 {
3230 if (*src == '\0')
3231 break;
3232
3233 if (*src == '\\')
3234 {
3235 dest[pos] = *(src+1);
3236 src += 2;
3237 continue;
3238 }
3239
3240 dest[pos] = *src;
3241
3242 if (*src == '"')
3243 {
3244 src++;
3245 break;
3246 }
3247
3248 if (*src == '\0')
3249 break;
3250
3251 src++;
3252 }
3253 dest[pos] = '\0';
3254
3255 if (!strcasecmp(tagstr, current_tag))
3256 return true;
3257 }
3258
3259 return false;
3260 }
3261
3262 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3263 {
3264 struct index_entry idx;
3265 char tag_data[TAG_MAXLEN+32];
3266 int idx_id;
3267 long masterfd = (long)parameters;
3268 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3269 tag_commitid };
3270 int i;
3271 (void)line_n;
3272
3273 if (*buf == '#')
3274 return 0;
3275
3276 logf("%d/%s", line_n, buf);
3277 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3278 {
3279 logf("filename missing");
3280 logf("-> %s", buf);
3281 return 0;
3282 }
3283
3284 idx_id = find_index(tag_data);
3285 if (idx_id < 0)
3286 {
3287 logf("entry not found");
3288 return 0;
3289 }
3290
3291 if (!get_index(masterfd, idx_id, &idx, false))
3292 {
3293 logf("failed to retrieve index entry");
3294 return 0;
3295 }
3296
3297 /* Stop if tag has already been modified. */
3298 if (idx.flag & FLAG_DIRTYNUM)
3299 return 0;
3300
3301 logf("import: %s", tag_data);
3302
3303 idx.flag |= FLAG_DIRTYNUM;
3304 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3305 {
3306 int data;
3307
3308 if (!read_tag(tag_data, sizeof tag_data, buf,
3309 tagcache_tag_to_str(import_tags[i])))
3310 {
3311 continue;
3312 }
3313
3314 data = atoi(tag_data);
3315 if (data < 0)
3316 continue;
3317
3318 idx.tag_seek[import_tags[i]] = data;
3319
3320 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3321 current_tcmh.serial = data + 1;
3322 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3323 current_tcmh.commitid = data + 1;
3324 }
3325
3326 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3327 }
3328
3329 #ifndef __PCTOOL__
3330 bool tagcache_import_changelog(void)
3331 {
3332 struct master_header myhdr;
3333 struct tagcache_header tch;
3334 int clfd;
3335 long masterfd;
3336 char buf[2048];
3337
3338 if (!tc_stat.ready)
3339 return false;
3340
3341 while (read_lock)
3342 sleep(1);
3343
3344 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3345 if (clfd < 0)
3346 {
3347 logf("failure to open changelog");
3348 return false;
3349 }
3350
3351 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3352 {
3353 close(clfd);
3354 return false;
3355 }
3356
3357 write_lock++;
3358
3359 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3360
3361 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3362 parse_changelog_line);
3363
3364 close(clfd);
3365 close(masterfd);
3366
3367 if (filenametag_fd >= 0)
3368 close(filenametag_fd);
3369
3370 write_lock--;
3371
3372 update_master_header();
3373
3374 return true;
3375 }
3376 #endif
3377
3378 bool tagcache_create_changelog(struct tagcache_search *tcs)
3379 {
3380 struct master_header myhdr;
3381 struct index_entry idx;
3382 char buf[TAG_MAXLEN+32];
3383 char temp[32];
3384 int clfd;
3385 int i, j;
3386
3387 if (!tc_stat.ready)
3388 return false;
3389
3390 if (!tagcache_search(tcs, tag_filename))
3391 return false;
3392
3393 /* Initialize the changelog */
3394 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3395 if (clfd < 0)
3396 {
3397 logf("failure to open changelog");
3398 return false;
3399 }
3400
3401 if (tcs->masterfd < 0)
3402 {
3403 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3404 return false;
3405 }
3406 else
3407 {
3408 lseek(tcs->masterfd, 0, SEEK_SET);
3409 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3410 }
3411
3412 write(clfd, "## Changelog version 1\n", 23);
3413
3414 for (i = 0; i < myhdr.tch.entry_count; i++)
3415 {
3416 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3417 != sizeof(struct index_entry))
3418 {
3419 logf("read error #9");
3420 tagcache_search_finish(tcs);
3421 close(clfd);
3422 return false;
3423 }
3424
3425 /* Skip until the entry found has been modified. */
3426 if (! (idx.flag & FLAG_DIRTYNUM) )
3427 continue;
3428
3429 /* Skip deleted entries too. */
3430 if (idx.flag & FLAG_DELETED)
3431 continue;
3432
3433 /* Now retrieve all tags. */
3434 for (j = 0; j < TAG_COUNT; j++)
3435 {
3436 if (tagcache_is_numeric_tag(j))
3437 {
3438 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3439 write_tag(clfd, tagcache_tag_to_str(j), temp);
3440 continue;
3441 }
3442
3443 tcs->type = j;
3444 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3445 write_tag(clfd, tagcache_tag_to_str(j), buf);
3446 }
3447
3448 write(clfd, "\n", 1);
3449 do_timed_yield();
3450 }
3451
3452 close(clfd);
3453
3454 tagcache_search_finish(tcs);
3455
3456 return true;
3457 }
3458
3459 static bool delete_entry(long idx_id)
3460 {
3461 int fd = -1;
3462 int masterfd = -1;
3463 int tag, i;
3464 struct index_entry idx, myidx;
3465 struct master_header myhdr;
3466 char buf[TAG_MAXLEN+32];
3467 int in_use[TAG_COUNT];
3468
3469 logf("delete_entry(): %ld", idx_id);
3470
3471 #ifdef HAVE_TC_RAMCACHE
3472 /* At first mark the entry removed from ram cache. */
3473 if (tc_stat.ramcache)
3474 hdr->indices[idx_id].flag |= FLAG_DELETED;
3475 #endif
3476
3477 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3478 return false;
3479
3480 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3481 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3482 != sizeof(struct index_entry))
3483 {
3484 logf("delete_entry(): read error");
3485 goto cleanup;
3486 }
3487
3488 if (myidx.flag & FLAG_DELETED)
3489 {
3490 logf("delete_entry(): already deleted!");
3491 goto cleanup;
3492 }
3493
3494 myidx.flag |= FLAG_DELETED;
3495 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3496 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3497 != sizeof(struct index_entry))
3498 {
3499 logf("delete_entry(): write_error #1");
3500 goto cleanup;
3501 }
3502
3503 /* Now check which tags are no longer in use (if any) */
3504 for (tag = 0; tag < TAG_COUNT; tag++)
3505 in_use[tag] = 0;
3506
3507 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3508 for (i = 0; i < myhdr.tch.entry_count; i++)
3509 {
3510 struct index_entry *idxp;
3511
3512 #ifdef HAVE_TC_RAMCACHE
3513 /* Use RAM DB if available for greater speed */
3514 if (tc_stat.ramcache)
3515 idxp = &hdr->indices[i];
3516 else
3517 #endif
3518 {
3519 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3520 != sizeof(struct index_entry))
3521 {
3522 logf("delete_entry(): read error #2");
3523 goto cleanup;
3524 }
3525 idxp = &idx;
3526 }
3527
3528 if (idxp->flag & FLAG_DELETED)
3529 continue;
3530
3531 for (tag = 0; tag < TAG_COUNT; tag++)
3532 {
3533 if (tagcache_is_numeric_tag(tag))
3534 continue;
3535
3536 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3537 in_use[tag]++;
3538 }
3539 }
3540
3541 /* Now delete all tags no longer in use. */
3542 for (tag = 0; tag < TAG_COUNT; tag++)
3543 {
3544 struct tagcache_header tch;
3545 int oldseek = myidx.tag_seek[tag];
3546
3547 if (tagcache_is_numeric_tag(tag))
3548 continue;
3549
3550 /**
3551 * Replace tag seek with a hash value of the field string data.
3552 * That way runtime statistics of moved or altered files can be
3553 * resurrected.
3554 */
3555 #ifdef HAVE_TC_RAMCACHE
3556 if (tc_stat.ramcache && tag != tag_filename)
3557 {
3558 struct tagfile_entry *tfe;
3559 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3560
3561 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3562 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3563 myidx.tag_seek[tag] = *seek;
3564 }
3565 else
3566 #endif
3567 {
3568 struct tagfile_entry tfe;
3569
3570 /* Open the index file, which contains the tag names. */
3571 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3572 goto cleanup;
3573
3574 /* Skip the header block */
3575 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3576 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3577 != sizeof(struct tagfile_entry))
3578 {
3579 logf("delete_entry(): read error #3");
3580 goto cleanup;
3581 }
3582
3583 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3584 {
3585 logf("delete_entry(): read error #4");
3586 goto cleanup;
3587 }
3588
3589 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3590 }
3591
3592 if (in_use[tag])
3593 {
3594 logf("in use: %d/%d", tag, in_use[tag]);
3595 if (fd >= 0)
3596 {
3597 close(fd);
3598 fd = -1;
3599 }
3600 continue;
3601 }
3602
3603 #ifdef HAVE_TC_RAMCACHE
3604 /* Delete from ram. */
3605 if (tc_stat.ramcache && tag != tag_filename)
3606 {
3607 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3608 tagentry->tag_data[0] = '\0';
3609 }
3610 #endif
3611
3612 /* Open the index file, which contains the tag names. */
3613 if (fd < 0)
3614 {
3615 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3616 goto cleanup;
3617 }
3618
3619 /* Skip the header block */
3620 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3621
3622 /* Debug, print 10 first characters of the tag
3623 read(fd, buf, 10);
3624 buf[10]='\0';
3625 logf("TAG:%s", buf);
3626 lseek(fd, -10, SEEK_CUR);
3627 */
3628
3629 /* Write first data byte in tag as \0 */
3630 write(fd, "", 1);
3631
3632 /* Now tag data has been removed */
3633 close(fd);
3634 fd = -1;
3635 }
3636
3637 /* Write index entry back into master index. */
3638 lseek(masterfd, sizeof(struct master_header) +
3639 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3640 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3641 != sizeof(struct index_entry))
3642 {
3643 logf("delete_entry(): write_error #2");
3644 goto cleanup;
3645 }
3646
3647 close(masterfd);
3648
3649 return true;
3650
3651 cleanup:
3652 if (fd >= 0)
3653 close(fd);
3654 if (masterfd >= 0)
3655 close(masterfd);
3656
3657 return false;
3658 }
3659
3660 #ifndef __PCTOOL__
3661 /**
3662 * Returns true if there is an event waiting in the queue
3663 * that requires the current operation to be aborted.
3664 */
3665 static bool check_event_queue(void)
3666 {
3667 struct queue_event ev;
3668
3669 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3670 switch (ev.id)
3671 {
3672 case Q_STOP_SCAN:
3673 case SYS_POWEROFF:
3674 case SYS_USB_CONNECTED:
3675 /* Put the event back into the queue. */
3676 queue_post(&tagcache_queue, ev.id, ev.data);
3677 return true;
3678 }
3679
3680 return false;
3681 }
3682 #endif
3683
3684 #ifdef HAVE_TC_RAMCACHE
3685 static bool allocate_tagcache(void)
3686 {
3687 struct master_header tcmh;
3688 int fd;
3689
3690 /* Load the header. */
3691 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3692 {
3693 hdr = NULL;
3694 return false;
3695 }
3696
3697 close(fd);
3698
3699 /**
3700 * Now calculate the required cache size plus
3701 * some extra space for alignment fixes.
3702 */
3703 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3704 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3705 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3706 memset(hdr, 0, sizeof(struct ramcache_header));
3707 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3708 hdr->indices = (struct index_entry *)(hdr + 1);
3709 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3710
3711 return true;
3712 }
3713
3714 # ifdef HAVE_EEPROM_SETTINGS
3715 static bool tagcache_dumpload(void)
3716 {
3717 struct statefile_header shdr;
3718 int fd, rc;
3719 long offpos;
3720 int i;
3721
3722 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3723 if (fd < 0)
3724 {
3725 logf("no tagcache statedump");
3726 return false;
3727 }
3728
3729 /* Check the statefile memory placement */
3730 hdr = buffer_alloc(0);
3731 rc = read(fd, &shdr, sizeof(struct statefile_header));
3732 if (rc != sizeof(struct statefile_header)
3733 /* || (long)hdr != (long)shdr.hdr */)
3734 {
3735 logf("incorrect statefile");
3736 hdr = NULL;
3737 close(fd);
3738 return false;
3739 }
3740
3741 offpos = (long)hdr - (long)shdr.hdr;
3742
3743 /* Lets allocate real memory and load it */
3744 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3745 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3746 close(fd);
3747
3748 if (rc != shdr.tc_stat.ramcache_allocated)
3749 {
3750 logf("read failure!");
3751 hdr = NULL;
3752 return false;
3753 }
3754
3755 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3756
3757 /* Now fix the pointers */
3758 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3759 for (i = 0; i < TAG_COUNT; i++)
3760 hdr->tags[i] += offpos;
3761
3762 return true;
3763 }
3764
3765 static bool tagcache_dumpsave(void)
3766 {
3767 struct statefile_header shdr;
3768 int fd;
3769
3770 if (!tc_stat.ramcache)
3771 return false;
3772
3773 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3774 if (fd < 0)
3775 {
3776 logf("failed to create a statedump");
3777 return false;
3778 }
3779
3780 /* Create the header */
3781 shdr.hdr = hdr;
3782 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3783 write(fd, &shdr, sizeof(struct statefile_header));
3784
3785 /* And dump the data too */
3786 write(fd, hdr, tc_stat.ramcache_allocated);
3787 close(fd);
3788
3789 return true;
3790 }
3791 # endif
3792
3793 static bool load_tagcache(void)
3794 {
3795 struct tagcache_header *tch;
3796 long bytesleft = tc_stat.ramcache_allocated;
3797 struct index_entry *idx;
3798 int rc, fd;
3799 char *p;
3800 int i, tag;
3801
3802 # ifdef HAVE_DIRCACHE
3803 while (dircache_is_initializing())
3804 sleep(1);
3805
3806 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3807 # endif
3808
3809 logf("loading tagcache to ram...");
3810
3811 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3812 if (fd < 0)
3813 {
3814 logf("tagcache open failed");
3815 return false;
3816 }
3817
3818 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3819 != sizeof(struct master_header)
3820 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3821 {
3822 logf("incorrect header");
3823 return false;
3824 }
3825
3826 idx = hdr->indices;
3827
3828 /* Load the master index table. */
3829 for (i = 0; i < hdr->h.tch.entry_count; i++)
3830 {
3831 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3832 if (rc != sizeof(struct index_entry))
3833 {
3834 logf("read error #10");
3835 close(fd);
3836 return false;
3837 }
3838
3839 bytesleft -= sizeof(struct index_entry);
3840 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3841 {
3842 logf("too big tagcache.");
3843 close(fd);
3844 return false;
3845 }
3846
3847 idx++;
3848 }
3849
3850 close(fd);
3851
3852 /* Load the tags. */
3853 p = (char *)idx;
3854 for (tag = 0; tag < TAG_COUNT; tag++)
3855 {
3856 struct tagfile_entry *fe;
3857 char buf[TAG_MAXLEN+32];
3858
3859 if (tagcache_is_numeric_tag(tag))
3860 continue ;
3861
3862 //p = ((void *)p+1);
3863 p = (char *)((long)p & ~0x03) + 0x04;
3864 hdr->tags[tag] = p;
3865
3866 /* Check the header. */
3867 tch = (struct tagcache_header *)p;
3868 p += sizeof(struct tagcache_header);
3869
3870 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3871 return false;
3872
3873 for (hdr->entry_count[tag] = 0;
3874 hdr->entry_count[tag] < tch->entry_count;
3875 hdr->entry_count[tag]++)
3876 {
3877 long pos;
3878
3879 if (do_timed_yield())
3880 {
3881 /* Abort if we got a critical event in queue */
3882 if (check_event_queue())
3883 return false;
3884 }
3885
3886 fe = (struct tagfile_entry *)p;
3887 pos = lseek(fd, 0, SEEK_CUR);
3888 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3889 if (rc != sizeof(struct tagfile_entry))
3890 {
3891 /* End of lookup table. */
3892 logf("read error #11");
3893 close(fd);
3894 return false;
3895 }
3896
3897 /* We have a special handling for the filename tags. */
3898 if (tag == tag_filename)
3899 {
3900 # ifdef HAVE_DIRCACHE
3901 const struct dirent *dc;
3902 # endif
3903
3904 // FIXME: This is wrong!
3905 // idx = &hdr->indices[hdr->entry_count[i]];
3906 idx = &hdr->indices[fe->idx_id];
3907
3908 if (fe->tag_length >= (long)sizeof(buf)-1)
3909 {
3910 read(fd, buf, 10);
3911 buf[10] = '\0';
3912 logf("TAG:%s", buf);
3913 logf("too long filename");
3914 close(fd);
3915 return false;
3916 }
3917
3918 rc = read(fd, buf, fe->tag_length);
3919 if (rc != fe->tag_length)
3920 {
3921 logf("read error #12");
3922 close(fd);
3923 return false;
3924 }
3925
3926 /* Check if the entry has already been removed */
3927 if (idx->flag & FLAG_DELETED)
3928 continue;
3929
3930 /* This flag must not be used yet. */
3931 if (idx->flag & FLAG_DIRCACHE)
3932 {
3933 logf("internal error!");
3934 close(fd);
3935 return false;
3936 }
3937
3938 if (idx->tag_seek[tag] != pos)
3939 {
3940 logf("corrupt data structures!");
3941 close(fd);
3942 return false;
3943 }
3944
3945 # ifdef HAVE_DIRCACHE
3946 if (dircache_is_enabled())
3947 {
3948 dc = dircache_get_entry_ptr(buf);
3949 if (dc == NULL)
3950 {
3951 logf("Entry no longer valid.");
3952 logf("-> %s", buf);
3953 delete_entry(fe->idx_id);
3954 continue ;
3955 }
3956
3957 idx->flag |= FLAG_DIRCACHE;
3958 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3959 idx->tag_seek[tag_filename] = (long)dc;
3960 }
3961 else
3962 # endif
3963 {
3964 /* This will be very slow unless dircache is enabled
3965 or target is flash based, but do it anyway for
3966 consistency. */
3967 /* Check if entry has been removed. */
3968 if (global_settings.tagcache_autoupdate)
3969 {
3970 if (!file_exists(buf))
3971 {
3972 logf("Entry no longer valid.");
3973 logf("-> %s", buf);
3974 delete_entry(fe->idx_id);
3975 continue;
3976 }
3977 }
3978 }
3979
3980 continue ;
3981 }
3982
3983 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3984 if (bytesleft < 0)
3985 {
3986 logf("too big tagcache #2");
3987 logf("tl: %d", fe->tag_length);
3988 logf("bl: %ld", bytesleft);
3989 close(fd);
3990 return false;
3991 }
3992
3993 p = fe->tag_data;
3994 rc = read(fd, fe->tag_data, fe->tag_length);
3995 p += rc;
3996
3997 if (rc != fe->tag_length)
3998 {
3999 logf("read error #13");
4000 logf("rc=0x%04x", rc); // 0x431
4001 logf("len=0x%04x", fe->tag_length); // 0x4000
4002 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4003 logf("tag=0x%02x", tag); // 0x00
4004 close(fd);
4005 return false;
4006 }
4007 }
4008 close(fd);
4009 }
4010
4011 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4012 logf("tagcache loaded into ram!");
4013
4014 return true;
4015 }
4016 #endif /* HAVE_TC_RAMCACHE */
4017
4018 static bool check_deleted_files(void)
4019 {
4020 int fd;
4021 char buf[TAG_MAXLEN+32];
4022 struct tagfile_entry tfe;
4023
4024 logf("reverse scan...");
4025 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4026 fd = open(buf, O_RDONLY);
4027
4028 if (fd < 0)
4029 {
4030 logf("%s open fail", buf);
4031 return false;
4032 }
4033
4034 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4035 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4036 == sizeof(struct tagfile_entry)
4037 #ifndef __PCTOOL__
4038 && !check_event_queue()
4039 #endif
4040 )
4041 {
4042 if (tfe.tag_length >= (long)sizeof(buf)-1)
4043 {
4044 logf("too long tag");
4045 close(fd);
4046 return false;
4047 }
4048
4049 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4050 {
4051 logf("read error #14");
4052 close(fd);
4053 return false;
4054 }
4055
4056 /* Check if the file has already deleted from the db. */
4057 if (*buf == '\0')
4058 continue;
4059
4060 /* Now check if the file exists. */
4061 if (!file_exists(buf))
4062 {
4063 logf("Entry no longer valid.");
4064 logf("-> %s / %d", buf, tfe.tag_length);
4065 delete_entry(tfe.idx_id);
4066 }
4067 }
4068
4069 close(fd);
4070
4071 logf("done");
4072
4073 return true;
4074 }
4075
4076 static bool check_dir(const char *dirname, int add_files)
4077 {
4078 DIR *dir;
4079 int len;
4080 int success = false;
4081 int ignore, unignore;
4082 char newpath[MAX_PATH];
4083
4084 dir = opendir(dirname);
4085 if (!dir)
4086 {
4087 logf("tagcache: opendir() failed");
4088 return false;
4089 }
4090
4091 /* check for a database.ignore file */
4092 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4093 ignore = file_exists(newpath);
4094 /* check for a database.unignore file */
4095 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4096 unignore = file_exists(newpath);
4097
4098 /* don't do anything if both ignore and unignore are there */
4099 if (ignore != unignore)
4100 add_files = unignore;
4101
4102 /* Recursively scan the dir. */
4103 #ifdef __PCTOOL__
4104 while (1)
4105 #else
4106 while (!check_event_queue())
4107 #endif
4108 {
4109 struct dirent *entry;
4110
4111 entry = readdir(dir);
4112
4113 if (entry == NULL)
4114 {
4115 success = true;
4116 break ;
4117 }
4118
4119 if (!strcmp((char *)entry->d_name, ".") ||
4120 !strcmp((char *)entry->d_name, ".."))
4121 continue;
4122
4123 yield();
4124
4125 len = strlen(curpath);
4126 snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4127 entry->d_name);
4128
4129 processed_dir_count++;
4130 if (entry->attribute & ATTR_DIRECTORY)
4131 check_dir(curpath, add_files);
4132 else if (add_files)
4133 {
4134 tc_stat.curentry = curpath;
4135
4136 /* Add a new entry to the temporary db file. */
4137 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4138 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4139 , dir->internal_entry
4140 #endif
4141 );
4142
4143 /* Wait until current path for debug screen is read and unset. */
4144 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4145 yield();
4146
4147 tc_stat.curentry = NULL;
4148 }
4149
4150 curpath[len] = '\0';
4151 }
4152
4153 closedir(dir);
4154
4155 return success;
4156 }
4157
4158 void tagcache_screensync_event(void)
4159 {
4160 tc_stat.curentry = NULL;
4161 }
4162
4163 void tagcache_screensync_enable(bool state)
4164 {
4165 tc_stat.syncscreen = state;
4166 }
4167
4168 void tagcache_build(const char *path)
4169 {
4170 struct tagcache_header header;
4171 bool ret;
4172
4173 curpath[0] = '\0';
4174 data_size = 0;
4175 total_entry_count = 0;
4176 processed_dir_count = 0;
4177
4178 #ifdef HAVE_DIRCACHE
4179 while (dircache_is_initializing())
4180 sleep(1);
4181 #endif
4182
4183 logf("updating tagcache");
4184
4185 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4186 if (cachefd >= 0)
4187 {
4188 logf("skipping, cache already waiting for commit");
4189 close(cachefd);
4190 return ;
4191 }
4192
4193 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4194 if (cachefd < 0)
4195 {
4196 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4197 return ;
4198 }
4199
4200 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4201
4202 cpu_boost(true);
4203
4204 logf("Scanning files...");
4205 /* Scan for new files. */
4206 memset(&header, 0, sizeof(struct tagcache_header));
4207 write(cachefd, &header, sizeof(struct tagcache_header));
4208
4209 if (strcmp("/", path) != 0)
4210 strcpy(curpath, path);
4211 ret = check_dir(path, true);
4212
4213 /* Write the header. */
4214 header.magic = TAGCACHE_MAGIC;
4215 header.datasize = data_size;
4216 header.entry_count = total_entry_count;
4217 lseek(cachefd, 0, SEEK_SET);
4218 write(cachefd, &header, sizeof(struct tagcache_header));
4219 close(cachefd);
4220
4221 if (filenametag_fd >= 0)
4222 {
4223 close(filenametag_fd);
4224 filenametag_fd = -1;
4225 }
4226
4227 if (!ret)
4228 {
4229 logf("Aborted.");
4230 cpu_boost(false);
4231 return ;
4232 }
4233
4234 /* Commit changes to the database. */
4235 #ifdef __PCTOOL__
4236 allocate_tempbuf();
4237 #endif
4238 if (commit())
4239 {
4240 remove(TAGCACHE_FILE_TEMP);
4241 logf("tagcache built!");
4242 }
4243 #ifdef __PCTOOL__
4244 free_tempbuf();
4245 #endif
4246
4247 #ifdef HAVE_TC_RAMCACHE
4248 if (hdr)
4249 {
4250 /* Import runtime statistics if we just initialized the db. */
4251 if (hdr->h.serial == 0)
4252 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4253 }
4254 #endif
4255
4256 cpu_boost(false);
4257 }
4258
4259 #ifdef HAVE_TC_RAMCACHE
4260 static void load_ramcache(void)
4261 {
4262 if (!hdr)
4263 return ;
4264
4265 cpu_boost(true);
4266
4267 /* At first we should load the cache (if exists). */
4268 tc_stat.ramcache = load_tagcache();
4269
4270 if (!tc_stat.ramcache)
4271 {
4272 /* If loading failed, it must indicate some problem with the db
4273 * so disable it entirely to prevent further issues. */
4274 tc_stat.ready = false;
4275 hdr = NULL;
4276 }
4277
4278 cpu_boost(false);
4279 }
4280
4281 void tagcache_unload_ramcache(void)
4282 {
4283 tc_stat.ramcache = false;
4284 /* Just to make sure there is no statefile present. */
4285 // remove(TAGCACHE_STATEFILE);
4286 }
4287 #endif
4288
4289 #ifndef __PCTOOL__
4290 static void tagcache_thread(void)
4291 {
4292 struct queue_event ev;
4293 bool check_done = false;
4294
4295 /* If the previous cache build/update was interrupted, commit
4296 * the changes first in foreground. */
4297 cpu_boost(true);
4298 allocate_tempbuf();
4299 commit();
4300 free_tempbuf();
4301
4302 #ifdef HAVE_TC_RAMCACHE
4303 # ifdef HAVE_EEPROM_SETTINGS
4304 if (firmware_settings.initialized && firmware_settings.disk_clean)
4305 check_done = tagcache_dumpload();
4306
4307 remove(TAGCACHE_STATEFILE);
4308 # endif
4309
4310 /* Allocate space for the tagcache if found on disk. */
4311 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4312 allocate_tagcache();
4313 #endif
4314
4315 cpu_boost(false);
4316 tc_stat.initialized = true;
4317
4318 /* Don't delay bootup with the header check but do it on background. */
4319 sleep(HZ);
4320 tc_stat.ready = check_all_headers();
4321 tc_stat.readyvalid = true;
4322
4323 while (1)
4324 {
4325 run_command_queue(false);
4326
4327 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4328
4329 switch (ev.id)
4330 {
4331 case Q_IMPORT_CHANGELOG:
4332 tagcache_import_changelog();
4333 break;
4334
4335 case Q_REBUILD:
4336 remove_files();
4337 remove(TAGCACHE_FILE_TEMP);
4338 tagcache_build("/");
4339 break;
4340
4341 case Q_UPDATE:
4342 tagcache_build("/");
4343 #ifdef HAVE_TC_RAMCACHE
4344 load_ramcache();
4345 #endif
4346 check_deleted_files();
4347 break ;
4348
4349 case Q_START_SCAN:
4350 check_done = false;
4351 case SYS_TIMEOUT:
4352 if (check_done || !tc_stat.ready)
4353 break ;
4354
4355 #ifdef HAVE_TC_RAMCACHE
4356 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4357 {
4358 load_ramcache();
4359 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4360 tagcache_build("/");
4361 }
4362 else
4363 #endif
4364 if (global_settings.tagcache_autoupdate)
4365 {
4366 tagcache_build("/");
4367
4368 /* This will be very slow unless dircache is enabled
4369 or target is flash based, but do it anyway for
4370 consistency. */
4371 check_deleted_files();
4372 }
4373
4374 logf("tagcache check done");
4375
4376 check_done = true;
4377 break ;
4378
4379 case Q_STOP_SCAN:
4380 break ;
4381
4382 case SYS_POWEROFF:
4383 break ;
4384
4385 #ifndef SIMULATOR
4386 case SYS_USB_CONNECTED:
4387 logf("USB: TagCache");
4388 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4389 usb_wait_for_disconnect(&tagcache_queue);
4390 break ;
4391 #endif
4392 }
4393 }
4394 }
4395
4396 bool tagcache_prepare_shutdown(void)
4397 {
4398 if (tagcache_get_commit_step() > 0)
4399 return false;
4400
4401 tagcache_stop_scan();
4402 while (read_lock || write_lock)
4403 sleep(1);
4404
4405 return true;
4406 }
4407
4408 void tagcache_shutdown(void)
4409 {
4410 /* Flush the command queue. */
4411 run_command_queue(true);
4412
4413 #ifdef HAVE_EEPROM_SETTINGS
4414 if (tc_stat.ramcache)
4415 tagcache_dumpsave();
4416 #endif
4417 }
4418
4419 static int get_progress(void)
4420 {
4421 int total_count = -1;
4422
4423 #ifdef HAVE_DIRCACHE
4424 if (dircache_is_enabled())
4425 {
4426 total_count = dircache_get_entry_count();
4427 }
4428 else
4429 #endif
4430 #ifdef HAVE_TC_RAMCACHE
4431 {
4432 if (hdr && tc_stat.ramcache)
4433 total_count = hdr->h.tch.entry_count;
4434 }
4435 #endif
4436
4437 if (total_count < 0)
4438 return -1;
4439
4440 return processed_dir_count * 100 / total_count;
4441 }
4442
4443 struct tagcache_stat* tagcache_get_stat(void)
4444 {
4445 tc_stat.progress = get_progress();
4446 tc_stat.processed_entries = processed_dir_count;
4447
4448 return &tc_stat;
4449 }
4450
4451 void tagcache_start_scan(void)
4452 {
4453 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4454 }
4455
4456 bool tagcache_update(void)
4457 {
4458 if (!tc_stat.ready)
4459 return false;
4460
4461 queue_post(&tagcache_queue, Q_UPDATE, 0);
4462 return false;
4463 }
4464
4465 bool tagcache_rebuild()
4466 {
4467 queue_post(&tagcache_queue, Q_REBUILD, 0);
4468 return false;
4469 }
4470
4471 void tagcache_stop_scan(void)
4472 {
4473 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4474 }
4475
4476 #ifdef HAVE_TC_RAMCACHE
4477 bool tagcache_is_ramcache(void)
4478 {
4479 return tc_stat.ramcache;
4480 }
4481 #endif
4482
4483 #endif /* !__PCTOOL__ */
4484
4485
4486 void tagcache_init(void)
4487 {
4488 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4489 memset(&current_tcmh, 0, sizeof(struct master_header));
4490 filenametag_fd = -1;
4491 write_lock = read_lock = 0;
4492
4493 #ifndef __PCTOOL__
4494 mutex_init(&command_queue_mutex);
4495 queue_init(&tagcache_queue, true);
4496 create_thread(tagcache_thread, tagcache_stack,
4497 sizeof(tagcache_stack), 0, tagcache_thread_name
4498 IF_PRIO(, PRIORITY_BACKGROUND)
4499 IF_COP(, CPU));
4500 #else
4501 tc_stat.initialized = true;
4502 allocate_tempbuf();
4503 commit();
4504 free_tempbuf();
4505 tc_stat.ready = check_all_headers();
4506 #endif
4507 }
4508
4509 #ifdef __PCTOOL__
4510 void tagcache_reverse_scan(void)
4511 {
4512 logf("Checking for deleted files");
4513 check_deleted_files();
4514 }
4515 #endif
4516
4517 bool tagcache_is_initialized(void)
4518 {
4519 return tc_stat.initialized;
4520 }
4521 bool tagcache_is_usable(void)
4522 {
4523 return tc_stat.initialized && tc_stat.ready;
4524 }
4525 int tagcache_get_commit_step(void)
4526 {
4527 return tc_stat.commit_step;
4528 }
4529 int tagcache_get_max_commit_step(void)
4530 {
4531 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4532 }
4533
My public Rockbox git repo