?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
rbutil: update french translation.
[kugel-rb.git] / apps / tagcache.c
blob0d06d8f56383215b8c918e4cdce4ce36776a4e6c
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /*#define LOGF_ENABLE*/
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #include "config.h"
64 #include "ata_idle_notify.h"
65 #include "thread.h"
66 #include "kernel.h"
67 #include "system.h"
68 #include "logf.h"
69 #include "string.h"
70 #include "usb.h"
71 #include "metadata.h"
72 #include "tagcache.h"
73 #include "buffer.h"
74 #include "crc32.h"
75 #include "misc.h"
76 #include "settings.h"
77 #include "dir.h"
78 #include "structec.h"
79
80 #ifndef __PCTOOL__
81 #include "lang.h"
82 #include "eeprom_settings.h"
83 #endif
84
85 #ifdef __PCTOOL__
86 #define yield() do { } while(0)
87 #define sim_sleep(timeout) do { } while(0)
88 #define do_timed_yield() do { } while(0)
89 #endif
90
91 #ifndef __PCTOOL__
92 /* Tag Cache thread. */
93 static struct event_queue tagcache_queue;
94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
95 static const char tagcache_thread_name[] = "tagcache";
96 #endif
97
98 #define UNTAGGED "<Untagged>"
99
100 /* Previous path when scanning directory tree recursively. */
101 static char curpath[TAG_MAXLEN+32];
102
103 /* Used when removing duplicates. */
104 static char *tempbuf; /* Allocated when needed. */
105 static long tempbufidx; /* Current location in buffer. */
106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
107 static long tempbuf_left; /* Buffer space left. */
108 static long tempbuf_pos;
109
110 #define SORTED_TAGS_COUNT 8
111 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
112 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
113 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
114 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
115 /* Tags we want to get sorted (loaded to the tempbuf). */
116 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
117 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
118 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
119
120 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
121 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
122 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
123 (1LU << tag_albumartist) | (1LU << tag_grouping))
124
125 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
126 static const char *tags_str[] = { "artist", "album", "genre", "title",
127 "filename", "composer", "comment", "albumartist", "grouping", "year",
128 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
129 "playtime", "lastplayed", "commitid", "mtime" };
130
131 /* Status information of the tagcache. */
132 static struct tagcache_stat tc_stat;
133
134 /* Queue commands. */
135 enum tagcache_queue {
136 Q_STOP_SCAN = 0,
137 Q_START_SCAN,
138 Q_IMPORT_CHANGELOG,
139 Q_UPDATE,
140 Q_REBUILD,
141
142 /* Internal tagcache command queue. */
143 CMD_UPDATE_MASTER_HEADER,
144 CMD_UPDATE_NUMERIC,
145 };
146
147 struct tagcache_command_entry {
148 int32_t command;
149 int32_t idx_id;
150 int32_t tag;
151 int32_t data;
152 };
153
154 #ifndef __PCTOOL__
155 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
156 static volatile int command_queue_widx = 0;
157 static volatile int command_queue_ridx = 0;
158 static struct mutex command_queue_mutex;
159 #endif
160
161 /* Tag database structures. */
162
163 /* Variable-length tag entry in tag files. */
164 struct tagfile_entry {
165 int32_t tag_length; /* Length of the data in bytes including '\0' */
166 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
167 char tag_data[0]; /* Begin of the tag data */
168 };
169
170 /* Fixed-size tag entry in master db index. */
171 struct index_entry {
172 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
173 int32_t flag; /* Status flags */
174 };
175
176 /* Header is the same in every file. */
177 struct tagcache_header {
178 int32_t magic; /* Header version number */
179 int32_t datasize; /* Data size in bytes */
180 int32_t entry_count; /* Number of entries in this file */
181 };
182
183 struct master_header {
184 struct tagcache_header tch;
185 int32_t serial; /* Increasing counting number */
186 int32_t commitid; /* Number of commits so far */
187 int32_t dirty;
188 };
189
190 /* For the endianess correction */
191 static const char *tagfile_entry_ec = "ll";
192 /**
193 Note: This should be (1 + TAG_COUNT) amount of l's.
194 */
195 static const char *index_entry_ec = "lllllllllllllllllllll";
196
197 static const char *tagcache_header_ec = "lll";
198 static const char *master_header_ec = "llllll";
199
200 static struct master_header current_tcmh;
201
202 #ifdef HAVE_TC_RAMCACHE
203 /* Header is created when loading database to ram. */
204 struct ramcache_header {
205 struct master_header h; /* Header from the master index */
206 struct index_entry *indices; /* Master index file content */
207 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
208 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
209 };
210
211 # ifdef HAVE_EEPROM_SETTINGS
212 struct statefile_header {
213 struct ramcache_header *hdr;
214 struct tagcache_stat tc_stat;
215 };
216 # endif
217
218 /* Pointer to allocated ramcache_header */
219 static struct ramcache_header *hdr;
220 #endif
221
222 /**
223 * Full tag entries stored in a temporary file waiting
224 * for commit to the cache. */
225 struct temp_file_entry {
226 long tag_offset[TAG_COUNT];
227 short tag_length[TAG_COUNT];
228 long flag;
229
230 long data_length;
231 };
232
233 struct tempbuf_id_list {
234 long id;
235 struct tempbuf_id_list *next;
236 };
237
238 struct tempbuf_searchidx {
239 long idx_id;
240 char *str;
241 int seek;
242 struct tempbuf_id_list idlist;
243 };
244
245 /* Lookup buffer for fixing messed up index while after sorting. */
246 static long commit_entry_count;
247 static long lookup_buffer_depth;
248 static struct tempbuf_searchidx **lookup;
249
250 /* Used when building the temporary file. */
251 static int cachefd = -1, filenametag_fd;
252 static int total_entry_count = 0;
253 static int data_size = 0;
254 static int processed_dir_count;
255
256 /* Thread safe locking */
257 static volatile int write_lock;
258 static volatile int read_lock;
259
260 static bool delete_entry(long idx_id);
261
262 const char* tagcache_tag_to_str(int tag)
263 {
264 return tags_str[tag];
265 }
266
267 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
268 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
269 {
270 return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
271 }
272
273 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
274 {
275 return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
276 }
277
278 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
279 {
280 return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
281 }
282
283 #ifdef HAVE_DIRCACHE
284 /**
285 * Returns true if specified flag is still present, i.e., dircache
286 * has not been reloaded.
287 */
288 static bool is_dircache_intact(void)
289 {
290 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
291 }
292 #endif
293
294 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
295 {
296 int fd;
297 char buf[MAX_PATH];
298 int rc;
299
300 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
301 return -1;
302
303 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
304
305 fd = open(buf, write ? O_RDWR : O_RDONLY);
306 if (fd < 0)
307 {
308 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
309 tc_stat.ready = false;
310 return fd;
311 }
312
313 /* Check the header. */
314 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
315 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
316 {
317 logf("header error");
318 tc_stat.ready = false;
319 close(fd);
320 return -2;
321 }
322
323 return fd;
324 }
325
326 static int open_master_fd(struct master_header *hdr, bool write)
327 {
328 int fd;
329 int rc;
330
331 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
332 if (fd < 0)
333 {
334 logf("master file open failed for R/W");
335 tc_stat.ready = false;
336 return fd;
337 }
338
339 tc_stat.econ = false;
340
341 /* Check the header. */
342 rc = read(fd, hdr, sizeof(struct master_header));
343 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
344 {
345 /* Success. */
346 return fd;
347 }
348
349 /* Trying to read again, this time with endianess correction enabled. */
350 lseek(fd, 0, SEEK_SET);
351
352 rc = ecread(fd, hdr, 1, master_header_ec, true);
353 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
354 {
355 logf("header error");
356 tc_stat.ready = false;
357 close(fd);
358 return -2;
359 }
360
361 tc_stat.econ = true;
362
363 return fd;
364 }
365
366 #ifndef __PCTOOL__
367 static bool do_timed_yield(void)
368 {
369 /* Sorting can lock up for quite a while, so yield occasionally */
370 static long wakeup_tick = 0;
371 if (TIME_AFTER(current_tick, wakeup_tick))
372 {
373 wakeup_tick = current_tick + (HZ/4);
374 yield();
375 return true;
376 }
377 return false;
378 }
379 #endif
380
381 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
382 static long find_entry_ram(const char *filename,
383 const struct dircache_entry *dc)
384 {
385 static long last_pos = 0;
386 int i;
387
388 /* Check if we tagcache is loaded into ram. */
389 if (!tc_stat.ramcache)
390 return -1;
391
392 if (dc == NULL)
393 dc = dircache_get_entry_ptr(filename);
394
395 if (dc == NULL)
396 {
397 logf("tagcache: file not found.");
398 return -1;
399 }
400
401 try_again:
402
403 if (last_pos > 0)
404 i = last_pos;
405 else
406 i = 0;
407
408 for (; i < hdr->h.tch.entry_count; i++)
409 {
410 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
411 {
412 last_pos = MAX(0, i - 3);
413 return i;
414 }
415
416 do_timed_yield();
417 }
418
419 if (last_pos > 0)
420 {
421 last_pos = 0;
422 goto try_again;
423 }
424
425 return -1;
426 }
427 #endif
428
429 static long find_entry_disk(const char *filename, bool localfd)
430 {
431 struct tagcache_header tch;
432 static long last_pos = -1;
433 long pos_history[POS_HISTORY_COUNT];
434 long pos_history_idx = 0;
435 bool found = false;
436 struct tagfile_entry tfe;
437 int fd;
438 char buf[TAG_MAXLEN+32];
439 int i;
440 int pos = -1;
441
442 if (!tc_stat.ready)
443 return -2;
444
445 fd = filenametag_fd;
446 if (fd < 0 || localfd)
447 {
448 last_pos = -1;
449 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
450 return -1;
451 }
452
453 check_again:
454
455 if (last_pos > 0)
456 lseek(fd, last_pos, SEEK_SET);
457 else
458 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
459
460 while (true)
461 {
462 pos = lseek(fd, 0, SEEK_CUR);
463 for (i = pos_history_idx-1; i >= 0; i--)
464 pos_history[i+1] = pos_history[i];
465 pos_history[0] = pos;
466
467 if (ecread_tagfile_entry(fd, &tfe)
468 != sizeof(struct tagfile_entry))
469 {
470 break ;
471 }
472
473 if (tfe.tag_length >= (long)sizeof(buf))
474 {
475 logf("too long tag #1");
476 close(fd);
477 if (!localfd)
478 filenametag_fd = -1;
479 last_pos = -1;
480 return -2;
481 }
482
483 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
484 {
485 logf("read error #2");
486 close(fd);
487 if (!localfd)
488 filenametag_fd = -1;
489 last_pos = -1;
490 return -3;
491 }
492
493 if (!strcasecmp(filename, buf))
494 {
495 last_pos = pos_history[pos_history_idx];
496 found = true;
497 break ;
498 }
499
500 if (pos_history_idx < POS_HISTORY_COUNT - 1)
501 pos_history_idx++;
502 }
503
504 /* Not found? */
505 if (!found)
506 {
507 if (last_pos > 0)
508 {
509 last_pos = -1;
510 logf("seek again");
511 goto check_again;
512 }
513
514 if (fd != filenametag_fd || localfd)
515 close(fd);
516 return -4;
517 }
518
519 if (fd != filenametag_fd || localfd)
520 close(fd);
521
522 return tfe.idx_id;
523 }
524
525 static int find_index(const char *filename)
526 {
527 long idx_id = -1;
528
529 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
530 if (tc_stat.ramcache && is_dircache_intact())
531 idx_id = find_entry_ram(filename, NULL);
532 #endif
533
534 if (idx_id < 0)
535 idx_id = find_entry_disk(filename, true);
536
537 return idx_id;
538 }
539
540 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
541 {
542 int idx_id;
543
544 if (!tc_stat.ready)
545 return false;
546
547 idx_id = find_index(filename);
548 if (idx_id < 0)
549 return false;
550
551 if (!tagcache_search(tcs, tag_filename))
552 return false;
553
554 tcs->entry_count = 0;
555 tcs->idx_id = idx_id;
556
557 return true;
558 }
559
560 static bool get_index(int masterfd, int idxid,
561 struct index_entry *idx, bool use_ram)
562 {
563 bool localfd = false;
564
565 if (idxid < 0)
566 {
567 logf("Incorrect idxid: %d", idxid);
568 return false;
569 }
570
571 #ifdef HAVE_TC_RAMCACHE
572 if (tc_stat.ramcache && use_ram)
573 {
574 if (hdr->indices[idxid].flag & FLAG_DELETED)
575 return false;
576
577 # ifdef HAVE_DIRCACHE
578 if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
579 || is_dircache_intact())
580 #endif
581 {
582 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
583 return true;
584 }
585 }
586 #else
587 (void)use_ram;
588 #endif
589
590 if (masterfd < 0)
591 {
592 struct master_header tcmh;
593
594 localfd = true;
595 masterfd = open_master_fd(&tcmh, false);
596 if (masterfd < 0)
597 return false;
598 }
599
600 lseek(masterfd, idxid * sizeof(struct index_entry)
601 + sizeof(struct master_header), SEEK_SET);
602 if (ecread_index_entry(masterfd, idx)
603 != sizeof(struct index_entry))
604 {
605 logf("read error #3");
606 if (localfd)
607 close(masterfd);
608
609 return false;
610 }
611
612 if (localfd)
613 close(masterfd);
614
615 if (idx->flag & FLAG_DELETED)
616 return false;
617
618 return true;
619 }
620
621 #ifndef __PCTOOL__
622
623 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
624 {
625 /* We need to exclude all memory only flags & tags when writing to disk. */
626 if (idx->flag & FLAG_DIRCACHE)
627 {
628 logf("memory only flags!");
629 return false;
630 }
631
632 #ifdef HAVE_TC_RAMCACHE
633 /* Only update numeric data. Writing the whole index to RAM by memcpy
634 * destroys dircache pointers!
635 */
636 if (tc_stat.ramcache)
637 {
638 int tag;
639 struct index_entry *idx_ram = &hdr->indices[idxid];
640
641 for (tag = 0; tag < TAG_COUNT; tag++)
642 {
643 if (TAGCACHE_IS_NUMERIC(tag))
644 {
645 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
646 }
647 }
648
649 /* Don't touch the dircache flag or attributes. */
650 idx_ram->flag = (idx->flag & 0x0000ffff)
651 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
652 }
653 #endif
654
655 lseek(masterfd, idxid * sizeof(struct index_entry)
656 + sizeof(struct master_header), SEEK_SET);
657 if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
658 {
659 logf("write error #3");
660 logf("idxid: %d", idxid);
661 return false;
662 }
663
664 return true;
665 }
666
667 #endif /* !__PCTOOL__ */
668
669 static bool open_files(struct tagcache_search *tcs, int tag)
670 {
671 if (tcs->idxfd[tag] < 0)
672 {
673 char fn[MAX_PATH];
674
675 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
676 tcs->idxfd[tag] = open(fn, O_RDONLY);
677 }
678
679 if (tcs->idxfd[tag] < 0)
680 {
681 logf("File not open!");
682 return false;
683 }
684
685 return true;
686 }
687
688 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
689 int tag, char *buf, long size)
690 {
691 struct tagfile_entry tfe;
692 long seek;
693
694 *buf = '\0';
695
696 if (TAGCACHE_IS_NUMERIC(tag))
697 return false;
698
699 seek = idx->tag_seek[tag];
700 if (seek < 0)
701 {
702 logf("Retrieve failed");
703 return false;
704 }
705
706 #ifdef HAVE_TC_RAMCACHE
707 if (tcs->ramsearch)
708 {
709 struct tagfile_entry *ep;
710
711 # ifdef HAVE_DIRCACHE
712 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
713 && is_dircache_intact())
714 {
715 dircache_copy_path((struct dircache_entry *)seek,
716 buf, size);
717 return true;
718 }
719 else
720 # endif
721 if (tag != tag_filename)
722 {
723 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
724 strlcpy(buf, ep->tag_data, size);
725
726 return true;
727 }
728 }
729 #endif
730
731 if (!open_files(tcs, tag))
732 return false;
733
734 lseek(tcs->idxfd[tag], seek, SEEK_SET);
735 if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
736 != sizeof(struct tagfile_entry))
737 {
738 logf("read error #5");
739 return false;
740 }
741
742 if (tfe.tag_length >= size)
743 {
744 logf("too small buffer");
745 return false;
746 }
747
748 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
749 tfe.tag_length)
750 {
751 logf("read error #6");
752 return false;
753 }
754
755 buf[tfe.tag_length] = '\0';
756
757 return true;
758 }
759
760 static long check_virtual_tags(int tag, const struct index_entry *idx)
761 {
762 long data = 0;
763
764 switch (tag)
765 {
766 case tag_virt_length_sec:
767 data = (idx->tag_seek[tag_length]/1000) % 60;
768 break;
769
770 case tag_virt_length_min:
771 data = (idx->tag_seek[tag_length]/1000) / 60;
772 break;
773
774 case tag_virt_playtime_sec:
775 data = (idx->tag_seek[tag_playtime]/1000) % 60;
776 break;
777
778 case tag_virt_playtime_min:
779 data = (idx->tag_seek[tag_playtime]/1000) / 60;
780 break;
781
782 case tag_virt_autoscore:
783 if (idx->tag_seek[tag_length] == 0
784 || idx->tag_seek[tag_playcount] == 0)
785 {
786 data = 0;
787 }
788 else
789 {
790 /* A straight calculus gives:
791 autoscore = 100 * playtime / length / playcout (1)
792 Now, consider the euclidian division of playtime by length:
793 playtime = alpha * length + beta
794 With:
795 0 <= beta < length
796 Now, (1) becomes:
797 autoscore = 100 * (alpha / playcout + beta / length / playcount)
798 Both terms should be small enough to avoid any overflow
799 */
800 data = 100 * (idx->tag_seek[tag_playtime] / idx->tag_seek[tag_length])
801 + (100 * (idx->tag_seek[tag_playtime] % idx->tag_seek[tag_length])) / idx->tag_seek[tag_length];
802 data /= idx->tag_seek[tag_playcount];
803 }
804 break;
805
806 /* How many commits before the file has been added to the DB. */
807 case tag_virt_entryage:
808 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
809 break;
810
811 default:
812 data = idx->tag_seek[tag];
813 }
814
815 return data;
816 }
817
818 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
819 {
820 struct index_entry idx;
821
822 if (!tc_stat.ready)
823 return false;
824
825 if (!TAGCACHE_IS_NUMERIC(tag))
826 return -1;
827
828 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
829 return -2;
830
831 return check_virtual_tags(tag, &idx);
832 }
833
834 inline static bool str_ends_with(const char *str1, const char *str2)
835 {
836 int str_len = strlen(str1);
837 int clause_len = strlen(str2);
838
839 if (clause_len > str_len)
840 return false;
841
842 return !strcasecmp(&str1[str_len - clause_len], str2);
843 }
844
845 inline static bool str_oneof(const char *str, const char *list)
846 {
847 const char *sep;
848 int l, len = strlen(str);
849
850 while (*list)
851 {
852 sep = strchr(list, '|');
853 l = sep ? (long)sep - (long)list : (int)strlen(list);
854 if ((l==len) && !strncasecmp(str, list, len))
855 return true;
856 list += sep ? l + 1 : l;
857 }
858
859 return false;
860 }
861
862 static bool check_against_clause(long numeric, const char *str,
863 const struct tagcache_search_clause *clause)
864 {
865 if (clause->numeric)
866 {
867 switch (clause->type)
868 {
869 case clause_is:
870 return numeric == clause->numeric_data;
871 case clause_is_not:
872 return numeric != clause->numeric_data;
873 case clause_gt:
874 return numeric > clause->numeric_data;
875 case clause_gteq:
876 return numeric >= clause->numeric_data;
877 case clause_lt:
878 return numeric < clause->numeric_data;
879 case clause_lteq:
880 return numeric <= clause->numeric_data;
881 default:
882 logf("Incorrect numeric tag: %d", clause->type);
883 }
884 }
885 else
886 {
887 switch (clause->type)
888 {
889 case clause_is:
890 return !strcasecmp(clause->str, str);
891 case clause_is_not:
892 return strcasecmp(clause->str, str);
893 case clause_gt:
894 return 0>strcasecmp(clause->str, str);
895 case clause_gteq:
896 return 0>=strcasecmp(clause->str, str);
897 case clause_lt:
898 return 0<strcasecmp(clause->str, str);
899 case clause_lteq:
900 return 0<=strcasecmp(clause->str, str);
901 case clause_contains:
902 return (strcasestr(str, clause->str) != NULL);
903 case clause_not_contains:
904 return (strcasestr(str, clause->str) == NULL);
905 case clause_begins_with:
906 return (strcasestr(str, clause->str) == str);
907 case clause_not_begins_with:
908 return (strcasestr(str, clause->str) != str);
909 case clause_ends_with:
910 return str_ends_with(str, clause->str);
911 case clause_not_ends_with:
912 return !str_ends_with(str, clause->str);
913 case clause_oneof:
914 return str_oneof(str, clause->str);
915
916 default:
917 logf("Incorrect tag: %d", clause->type);
918 }
919 }
920
921 return false;
922 }
923
924 static bool check_clauses(struct tagcache_search *tcs,
925 struct index_entry *idx,
926 struct tagcache_search_clause **clause, int count)
927 {
928 int i;
929
930 #ifdef HAVE_TC_RAMCACHE
931 if (tcs->ramsearch)
932 {
933 /* Go through all conditional clauses. */
934 for (i = 0; i < count; i++)
935 {
936 struct tagfile_entry *tfe;
937 int seek;
938 char buf[256];
939 char *str = NULL;
940
941 seek = check_virtual_tags(clause[i]->tag, idx);
942
943 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
944 {
945 if (clause[i]->tag == tag_filename)
946 {
947 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
948 str = buf;
949 }
950 else
951 {
952 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
953 str = tfe->tag_data;
954 }
955 }
956
957 if (!check_against_clause(seek, str, clause[i]))
958 return false;
959 }
960 }
961 else
962 #endif
963 {
964 /* Check for conditions. */
965 for (i = 0; i < count; i++)
966 {
967 struct tagfile_entry tfe;
968 int seek;
969 char str[256];
970
971 seek = check_virtual_tags(clause[i]->tag, idx);
972
973 memset(str, 0, sizeof str);
974 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
975 {
976 int fd = tcs->idxfd[clause[i]->tag];
977 lseek(fd, seek, SEEK_SET);
978 ecread_tagfile_entry(fd, &tfe);
979 if (tfe.tag_length >= (int)sizeof(str))
980 {
981 logf("Too long tag read!");
982 break ;
983 }
984
985 read(fd, str, tfe.tag_length);
986
987 /* Check if entry has been deleted. */
988 if (str[0] == '\0')
989 break;
990 }
991
992 if (!check_against_clause(seek, str, clause[i]))
993 return false;
994 }
995 }
996
997 return true;
998 }
999
1000 bool tagcache_check_clauses(struct tagcache_search *tcs,
1001 struct tagcache_search_clause **clause, int count)
1002 {
1003 struct index_entry idx;
1004
1005 if (count == 0)
1006 return true;
1007
1008 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1009 return false;
1010
1011 return check_clauses(tcs, &idx, clause, count);
1012 }
1013
1014 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1015 {
1016 int i;
1017
1018 /* If uniq buffer is not defined we must return true for search to work. */
1019 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1020 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1021 {
1022 return true;
1023 }
1024
1025 for (i = 0; i < tcs->unique_list_count; i++)
1026 {
1027 /* Return false if entry is found. */
1028 if (tcs->unique_list[i] == id)
1029 return false;
1030 }
1031
1032 if (tcs->unique_list_count < tcs->unique_list_capacity)
1033 {
1034 tcs->unique_list[i] = id;
1035 tcs->unique_list_count++;
1036 }
1037
1038 return true;
1039 }
1040
1041 static bool build_lookup_list(struct tagcache_search *tcs)
1042 {
1043 struct index_entry entry;
1044 int i, j;
1045
1046 tcs->seek_list_count = 0;
1047
1048 #ifdef HAVE_TC_RAMCACHE
1049 if (tcs->ramsearch
1050 # ifdef HAVE_DIRCACHE
1051 && (tcs->type != tag_filename || is_dircache_intact())
1052 # endif
1053 )
1054 {
1055 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1056 {
1057 struct tagcache_seeklist_entry *seeklist;
1058 struct index_entry *idx = &hdr->indices[i];
1059 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1060 break ;
1061
1062 /* Skip deleted files. */
1063 if (idx->flag & FLAG_DELETED)
1064 continue;
1065
1066 /* Go through all filters.. */
1067 for (j = 0; j < tcs->filter_count; j++)
1068 {
1069 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1070 {
1071 break ;
1072 }
1073 }
1074
1075 if (j < tcs->filter_count)
1076 continue ;
1077
1078 /* Check for conditions. */
1079 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1080 continue;
1081
1082 /* Add to the seek list if not already in uniq buffer. */
1083 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1084 continue;
1085
1086 /* Lets add it. */
1087 seeklist = &tcs->seeklist[tcs->seek_list_count];
1088 seeklist->seek = idx->tag_seek[tcs->type];
1089 seeklist->flag = idx->flag;
1090 seeklist->idx_id = i;
1091 tcs->seek_list_count++;
1092 }
1093
1094 tcs->seek_pos = i;
1095
1096 return tcs->seek_list_count > 0;
1097 }
1098 #endif
1099
1100 if (tcs->masterfd < 0)
1101 {
1102 struct master_header tcmh;
1103 tcs->masterfd = open_master_fd(&tcmh, false);
1104 }
1105
1106 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1107 sizeof(struct master_header), SEEK_SET);
1108
1109 while (ecread_index_entry(tcs->masterfd, &entry)
1110 == sizeof(struct index_entry))
1111 {
1112 struct tagcache_seeklist_entry *seeklist;
1113
1114 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1115 break ;
1116
1117 i = tcs->seek_pos;
1118 tcs->seek_pos++;
1119
1120 /* Check if entry has been deleted. */
1121 if (entry.flag & FLAG_DELETED)
1122 continue;
1123
1124 /* Go through all filters.. */
1125 for (j = 0; j < tcs->filter_count; j++)
1126 {
1127 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1128 break ;
1129 }
1130
1131 if (j < tcs->filter_count)
1132 continue ;
1133
1134 /* Check for conditions. */
1135 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1136 continue;
1137
1138 /* Add to the seek list if not already in uniq buffer. */
1139 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1140 continue;
1141
1142 /* Lets add it. */
1143 seeklist = &tcs->seeklist[tcs->seek_list_count];
1144 seeklist->seek = entry.tag_seek[tcs->type];
1145 seeklist->flag = entry.flag;
1146 seeklist->idx_id = i;
1147 tcs->seek_list_count++;
1148
1149 yield();
1150 }
1151
1152 return tcs->seek_list_count > 0;
1153 }
1154
1155
1156 static void remove_files(void)
1157 {
1158 int i;
1159 char buf[MAX_PATH];
1160
1161 tc_stat.ready = false;
1162 tc_stat.ramcache = false;
1163 tc_stat.econ = false;
1164 remove(TAGCACHE_FILE_MASTER);
1165 for (i = 0; i < TAG_COUNT; i++)
1166 {
1167 if (TAGCACHE_IS_NUMERIC(i))
1168 continue;
1169
1170 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1171 remove(buf);
1172 }
1173 }
1174
1175
1176 static bool check_all_headers(void)
1177 {
1178 struct master_header myhdr;
1179 struct tagcache_header tch;
1180 int tag;
1181 int fd;
1182
1183 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1184 return false;
1185
1186 close(fd);
1187 if (myhdr.dirty)
1188 {
1189 logf("tagcache is dirty!");
1190 return false;
1191 }
1192
1193 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1194
1195 for (tag = 0; tag < TAG_COUNT; tag++)
1196 {
1197 if (TAGCACHE_IS_NUMERIC(tag))
1198 continue;
1199
1200 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1201 return false;
1202
1203 close(fd);
1204 }
1205
1206 return true;
1207 }
1208
1209 bool tagcache_is_busy(void)
1210 {
1211 return read_lock || write_lock;
1212 }
1213
1214 bool tagcache_search(struct tagcache_search *tcs, int tag)
1215 {
1216 struct tagcache_header tag_hdr;
1217 struct master_header master_hdr;
1218 int i;
1219
1220 while (read_lock)
1221 sleep(1);
1222
1223 memset(tcs, 0, sizeof(struct tagcache_search));
1224 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1225 return false;
1226
1227 tcs->position = sizeof(struct tagcache_header);
1228 tcs->type = tag;
1229 tcs->seek_pos = 0;
1230 tcs->list_position = 0;
1231 tcs->seek_list_count = 0;
1232 tcs->filter_count = 0;
1233 tcs->masterfd = -1;
1234
1235 for (i = 0; i < TAG_COUNT; i++)
1236 tcs->idxfd[i] = -1;
1237
1238 #ifndef HAVE_TC_RAMCACHE
1239 tcs->ramsearch = false;
1240 #else
1241 tcs->ramsearch = tc_stat.ramcache;
1242 if (tcs->ramsearch)
1243 {
1244 tcs->entry_count = hdr->entry_count[tcs->type];
1245 }
1246 else
1247 #endif
1248 {
1249 /* Always open as R/W so we can pass tcs to functions that modify data also
1250 * without failing. */
1251 tcs->masterfd = open_master_fd(&master_hdr, true);
1252 if (tcs->masterfd < 0)
1253 return false;
1254
1255 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1256 {
1257 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1258 if (tcs->idxfd[tcs->type] < 0)
1259 return false;
1260
1261 tcs->entry_count = tag_hdr.entry_count;
1262 }
1263 else
1264 {
1265 tcs->entry_count = master_hdr.tch.entry_count;
1266 }
1267 }
1268
1269 tcs->valid = true;
1270 tcs->initialized = true;
1271 write_lock++;
1272
1273 return true;
1274 }
1275
1276 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1277 void *buffer, long length)
1278 {
1279 tcs->unique_list = (unsigned long *)buffer;
1280 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1281 tcs->unique_list_count = 0;
1282 }
1283
1284 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1285 int tag, int seek)
1286 {
1287 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1288 return false;
1289
1290 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1291 return false;
1292
1293 tcs->filter_tag[tcs->filter_count] = tag;
1294 tcs->filter_seek[tcs->filter_count] = seek;
1295 tcs->filter_count++;
1296
1297 return true;
1298 }
1299
1300 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1301 struct tagcache_search_clause *clause)
1302 {
1303 int i;
1304
1305 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1306 {
1307 logf("Too many clauses");
1308 return false;
1309 }
1310
1311 /* Check if there is already a similar filter in present (filters are
1312 * much faster than clauses).
1313 */
1314 for (i = 0; i < tcs->filter_count; i++)
1315 {
1316 if (tcs->filter_tag[i] == clause->tag)
1317 return true;
1318 }
1319
1320 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1321 {
1322 char buf[MAX_PATH];
1323
1324 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1325 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1326 }
1327
1328 tcs->clause[tcs->clause_count] = clause;
1329 tcs->clause_count++;
1330
1331 return true;
1332 }
1333
1334 static bool get_next(struct tagcache_search *tcs)
1335 {
1336 static char buf[TAG_MAXLEN+32];
1337 struct tagfile_entry entry;
1338 long flag = 0;
1339
1340 if (!tcs->valid || !tc_stat.ready)
1341 return false;
1342
1343 if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1344 #ifdef HAVE_TC_RAMCACHE
1345 && !tcs->ramsearch
1346 #endif
1347 )
1348 return false;
1349
1350 /* Relative fetch. */
1351 if (tcs->filter_count > 0 || tcs->clause_count > 0
1352 || TAGCACHE_IS_NUMERIC(tcs->type)
1353 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1354 /* We need to retrieve flag status for dircache. */
1355 || (tcs->ramsearch && tcs->type == tag_filename)
1356 #endif
1357 )
1358 {
1359 struct tagcache_seeklist_entry *seeklist;
1360
1361 /* Check for end of list. */
1362 if (tcs->list_position == tcs->seek_list_count)
1363 {
1364 tcs->list_position = 0;
1365
1366 /* Try to fetch more. */
1367 if (!build_lookup_list(tcs))
1368 {
1369 tcs->valid = false;
1370 return false;
1371 }
1372 }
1373
1374 seeklist = &tcs->seeklist[tcs->list_position];
1375 flag = seeklist->flag;
1376 tcs->position = seeklist->seek;
1377 tcs->idx_id = seeklist->idx_id;
1378 tcs->list_position++;
1379 }
1380 else
1381 {
1382 if (tcs->entry_count == 0)
1383 {
1384 tcs->valid = false;
1385 return false;
1386 }
1387
1388 tcs->entry_count--;
1389 }
1390
1391 tcs->result_seek = tcs->position;
1392
1393 if (TAGCACHE_IS_NUMERIC(tcs->type))
1394 {
1395 snprintf(buf, sizeof(buf), "%d", tcs->position);
1396 tcs->result = buf;
1397 tcs->result_len = strlen(buf) + 1;
1398 return true;
1399 }
1400
1401 /* Direct fetch. */
1402 #ifdef HAVE_TC_RAMCACHE
1403 if (tcs->ramsearch)
1404 {
1405
1406 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1407 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1408 && is_dircache_intact())
1409 {
1410 dircache_copy_path((struct dircache_entry *)tcs->position,
1411 buf, sizeof buf);
1412 tcs->result = buf;
1413 tcs->result_len = strlen(buf) + 1;
1414 tcs->ramresult = false;
1415
1416 return true;
1417 }
1418 else
1419 #endif
1420 if (tcs->type != tag_filename)
1421 {
1422 struct tagfile_entry *ep;
1423
1424 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1425 tcs->result = ep->tag_data;
1426 tcs->result_len = strlen(tcs->result) + 1;
1427 tcs->idx_id = ep->idx_id;
1428 tcs->ramresult = true;
1429
1430 /* Increase position for the next run. This may get overwritten. */
1431 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1432
1433 return true;
1434 }
1435 }
1436 #endif
1437
1438 if (!open_files(tcs, tcs->type))
1439 {
1440 tcs->valid = false;
1441 return false;
1442 }
1443
1444 /* Seek stream to the correct position and continue to direct fetch. */
1445 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1446
1447 if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1448 {
1449 logf("read error #5");
1450 tcs->valid = false;
1451 return false;
1452 }
1453
1454 if (entry.tag_length > (long)sizeof(buf))
1455 {
1456 tcs->valid = false;
1457 logf("too long tag #2");
1458 logf("P:%lX/%lX", tcs->position, entry.tag_length);
1459 return false;
1460 }
1461
1462 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1463 {
1464 tcs->valid = false;
1465 logf("read error #4");
1466 return false;
1467 }
1468
1469 /**
1470 Update the position for the next read (this may be overridden
1471 if filters or clauses are being used).
1472 */
1473 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1474 tcs->result = buf;
1475 tcs->result_len = strlen(tcs->result) + 1;
1476 tcs->idx_id = entry.idx_id;
1477 tcs->ramresult = false;
1478
1479 return true;
1480 }
1481
1482 bool tagcache_get_next(struct tagcache_search *tcs)
1483 {
1484 while (get_next(tcs))
1485 {
1486 if (tcs->result_len > 1)
1487 return true;
1488 }
1489
1490 return false;
1491 }
1492
1493 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1494 int tag, char *buf, long size)
1495 {
1496 struct index_entry idx;
1497
1498 *buf = '\0';
1499 if (!get_index(tcs->masterfd, idxid, &idx, true))
1500 return false;
1501
1502 return retrieve(tcs, &idx, tag, buf, size);
1503 }
1504
1505 static bool update_master_header(void)
1506 {
1507 struct master_header myhdr;
1508 int fd;
1509
1510 if (!tc_stat.ready)
1511 return false;
1512
1513 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1514 return false;
1515
1516 myhdr.serial = current_tcmh.serial;
1517 myhdr.commitid = current_tcmh.commitid;
1518 myhdr.dirty = current_tcmh.dirty;
1519
1520 /* Write it back */
1521 lseek(fd, 0, SEEK_SET);
1522 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1523 close(fd);
1524
1525 #ifdef HAVE_TC_RAMCACHE
1526 if (hdr)
1527 {
1528 hdr->h.serial = current_tcmh.serial;
1529 hdr->h.commitid = current_tcmh.commitid;
1530 hdr->h.dirty = current_tcmh.dirty;
1531 }
1532 #endif
1533
1534 return true;
1535 }
1536
1537 #if 0
1538
1539 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1540 {
1541 struct tagentry *entry;
1542
1543 if (tcs->type != tag_title)
1544 return ;
1545
1546 /* We will need reserve buffer for this. */
1547 if (tcs->ramcache)
1548 {
1549 struct tagfile_entry *ep;
1550
1551 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1552 tcs->seek_list[tcs->seek_list_count];
1553 }
1554
1555 entry = find_entry_ram();
1556
1557 }
1558 #endif
1559
1560 void tagcache_search_finish(struct tagcache_search *tcs)
1561 {
1562 int i;
1563
1564 if (!tcs->initialized)
1565 return;
1566
1567 if (tcs->masterfd >= 0)
1568 {
1569 close(tcs->masterfd);
1570 tcs->masterfd = -1;
1571 }
1572
1573 for (i = 0; i < TAG_COUNT; i++)
1574 {
1575 if (tcs->idxfd[i] >= 0)
1576 {
1577 close(tcs->idxfd[i]);
1578 tcs->idxfd[i] = -1;
1579 }
1580 }
1581
1582 tcs->ramsearch = false;
1583 tcs->valid = false;
1584 tcs->initialized = 0;
1585 if (write_lock > 0)
1586 write_lock--;
1587 }
1588
1589 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1590 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1591 {
1592 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1593 }
1594
1595 static long get_tag_numeric(const struct index_entry *entry, int tag)
1596 {
1597 return check_virtual_tags(tag, entry);
1598 }
1599
1600 static char* get_tag_string(const struct index_entry *entry, int tag)
1601 {
1602 char* s = get_tag(entry, tag)->tag_data;
1603 return strcmp(s, UNTAGGED) ? s : NULL;
1604 }
1605
1606 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1607 {
1608 struct index_entry *entry;
1609 int idx_id;
1610
1611 if (!tc_stat.ready || !tc_stat.ramcache)
1612 return false;
1613
1614 /* Find the corresponding entry in tagcache. */
1615 idx_id = find_entry_ram(filename, NULL);
1616 if (idx_id < 0)
1617 return false;
1618
1619 entry = &hdr->indices[idx_id];
1620
1621 id3->title = get_tag_string(entry, tag_title);
1622 id3->artist = get_tag_string(entry, tag_artist);
1623 id3->album = get_tag_string(entry, tag_album);
1624 id3->genre_string = get_tag_string(entry, tag_genre);
1625 id3->composer = get_tag_string(entry, tag_composer);
1626 id3->comment = get_tag_string(entry, tag_comment);
1627 id3->albumartist = get_tag_string(entry, tag_albumartist);
1628 id3->grouping = get_tag_string(entry, tag_grouping);
1629
1630 id3->length = get_tag_numeric(entry, tag_length);
1631 id3->playcount = get_tag_numeric(entry, tag_playcount);
1632 id3->rating = get_tag_numeric(entry, tag_rating);
1633 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1634 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1635 id3->year = get_tag_numeric(entry, tag_year);
1636
1637 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1638 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1639 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1640 if (id3->bitrate == 0)
1641 id3->bitrate = 1;
1642
1643 return true;
1644 }
1645 #endif
1646
1647 static inline void write_item(const char *item)
1648 {
1649 int len = strlen(item) + 1;
1650
1651 data_size += len;
1652 write(cachefd, item, len);
1653 }
1654
1655 static int check_if_empty(char **tag)
1656 {
1657 int length;
1658
1659 if (*tag == NULL || **tag == '\0')
1660 {
1661 *tag = UNTAGGED;
1662 return sizeof(UNTAGGED); /* Tag length */
1663 }
1664
1665 length = strlen(*tag);
1666 if (length > TAG_MAXLEN)
1667 {
1668 logf("over length tag: %s", *tag);
1669 length = TAG_MAXLEN;
1670 (*tag)[length] = '\0';
1671 }
1672
1673 return length + 1;
1674 }
1675
1676 #define ADD_TAG(entry,tag,data) \
1677 /* Adding tag */ \
1678 entry.tag_offset[tag] = offset; \
1679 entry.tag_length[tag] = check_if_empty(data); \
1680 offset += entry.tag_length[tag]
1681 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1682 * idea, as it uses lots of stack and is called from a recursive function
1683 * (check_dir).
1684 */
1685 static void __attribute__ ((noinline)) add_tagcache(char *path,
1686 unsigned long mtime
1687 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1688 ,const struct dircache_entry *dc
1689 #endif
1690 )
1691 {
1692 struct mp3entry id3;
1693 struct temp_file_entry entry;
1694 bool ret;
1695 int fd;
1696 int idx_id = -1;
1697 char tracknumfix[3];
1698 int offset = 0;
1699 int path_length = strlen(path);
1700 bool has_albumartist;
1701 bool has_grouping;
1702
1703 #ifdef SIMULATOR
1704 /* Crude logging for the sim - to aid in debugging */
1705 int logfd = open(ROCKBOX_DIR "/database.log",
1706 O_WRONLY | O_APPEND | O_CREAT);
1707 if (logfd >= 0) {
1708 write(logfd, path, strlen(path));
1709 write(logfd, "\n", 1);
1710 close(logfd);
1711 }
1712 #endif
1713
1714 if (cachefd < 0)
1715 return ;
1716
1717 /* Check for overlength file path. */
1718 if (path_length > TAG_MAXLEN)
1719 {
1720 /* Path can't be shortened. */
1721 logf("Too long path: %s", path);
1722 return ;
1723 }
1724
1725 /* Check if the file is supported. */
1726 if (probe_file_format(path) == AFMT_UNKNOWN)
1727 return ;
1728
1729 /* Check if the file is already cached. */
1730 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1731 if (tc_stat.ramcache && is_dircache_intact())
1732 {
1733 idx_id = find_entry_ram(path, dc);
1734 }
1735 #endif
1736
1737 /* Be sure the entry doesn't exist. */
1738 if (filenametag_fd >= 0 && idx_id < 0)
1739 idx_id = find_entry_disk(path, false);
1740
1741 /* Check if file has been modified. */
1742 if (idx_id >= 0)
1743 {
1744 struct index_entry idx;
1745
1746 /* TODO: Mark that the index exists (for fast reverse scan) */
1747 //found_idx[idx_id/8] |= idx_id%8;
1748
1749 if (!get_index(-1, idx_id, &idx, true))
1750 {
1751 logf("failed to retrieve index entry");
1752 return ;
1753 }
1754
1755 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1756 {
1757 /* No changes to file. */
1758 return ;
1759 }
1760
1761 /* Metadata might have been changed. Delete the entry. */
1762 logf("Re-adding: %s", path);
1763 if (!delete_entry(idx_id))
1764 {
1765 logf("delete_entry failed: %d", idx_id);
1766 return ;
1767 }
1768 }
1769
1770 fd = open(path, O_RDONLY);
1771 if (fd < 0)
1772 {
1773 logf("open fail: %s", path);
1774 return ;
1775 }
1776
1777 memset(&id3, 0, sizeof(struct mp3entry));
1778 memset(&entry, 0, sizeof(struct temp_file_entry));
1779 memset(&tracknumfix, 0, sizeof(tracknumfix));
1780 ret = get_metadata(&id3, fd, path);
1781 close(fd);
1782
1783 if (!ret)
1784 return ;
1785
1786 logf("-> %s", path);
1787
1788 /* Generate track number if missing. */
1789 if (id3.tracknum <= 0)
1790 {
1791 const char *p = strrchr(path, '.');
1792
1793 if (p == NULL)
1794 p = &path[strlen(path)-1];
1795
1796 while (*p != '/')
1797 {
1798 if (isdigit(*p) && isdigit(*(p-1)))
1799 {
1800 tracknumfix[1] = *p--;
1801 tracknumfix[0] = *p;
1802 break;
1803 }
1804 p--;
1805 }
1806
1807 if (tracknumfix[0] != '\0')
1808 {
1809 id3.tracknum = atoi(tracknumfix);
1810 /* Set a flag to indicate track number has been generated. */
1811 entry.flag |= FLAG_TRKNUMGEN;
1812 }
1813 else
1814 {
1815 /* Unable to generate track number. */
1816 id3.tracknum = -1;
1817 }
1818 }
1819
1820 /* Numeric tags */
1821 entry.tag_offset[tag_year] = id3.year;
1822 entry.tag_offset[tag_discnumber] = id3.discnum;
1823 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1824 entry.tag_offset[tag_length] = id3.length;
1825 entry.tag_offset[tag_bitrate] = id3.bitrate;
1826 entry.tag_offset[tag_mtime] = mtime;
1827
1828 /* String tags. */
1829 has_albumartist = id3.albumartist != NULL
1830 && strlen(id3.albumartist) > 0;
1831 has_grouping = id3.grouping != NULL
1832 && strlen(id3.grouping) > 0;
1833
1834 ADD_TAG(entry, tag_filename, &path);
1835 ADD_TAG(entry, tag_title, &id3.title);
1836 ADD_TAG(entry, tag_artist, &id3.artist);
1837 ADD_TAG(entry, tag_album, &id3.album);
1838 ADD_TAG(entry, tag_genre, &id3.genre_string);
1839 ADD_TAG(entry, tag_composer, &id3.composer);
1840 ADD_TAG(entry, tag_comment, &id3.comment);
1841 if (has_albumartist)
1842 {
1843 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1844 }
1845 else
1846 {
1847 ADD_TAG(entry, tag_albumartist, &id3.artist);
1848 }
1849 if (has_grouping)
1850 {
1851 ADD_TAG(entry, tag_grouping, &id3.grouping);
1852 }
1853 else
1854 {
1855 ADD_TAG(entry, tag_grouping, &id3.title);
1856 }
1857 entry.data_length = offset;
1858
1859 /* Write the header */
1860 write(cachefd, &entry, sizeof(struct temp_file_entry));
1861
1862 /* And tags also... Correct order is critical */
1863 write_item(path);
1864 write_item(id3.title);
1865 write_item(id3.artist);
1866 write_item(id3.album);
1867 write_item(id3.genre_string);
1868 write_item(id3.composer);
1869 write_item(id3.comment);
1870 if (has_albumartist)
1871 {
1872 write_item(id3.albumartist);
1873 }
1874 else
1875 {
1876 write_item(id3.artist);
1877 }
1878 if (has_grouping)
1879 {
1880 write_item(id3.grouping);
1881 }
1882 else
1883 {
1884 write_item(id3.title);
1885 }
1886 total_entry_count++;
1887 }
1888
1889 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1890 {
1891 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1892 int len = strlen(str)+1;
1893 int i;
1894 unsigned crc32;
1895 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1896 char buf[TAG_MAXLEN+32];
1897
1898 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1899 buf[i] = tolower(str[i]);
1900 buf[i] = '\0';
1901
1902 crc32 = crc_32(buf, i, 0xffffffff);
1903
1904 if (unique)
1905 {
1906 /* Check if the crc does not exist -> entry does not exist for sure. */
1907 for (i = 0; i < tempbufidx; i++)
1908 {
1909 if (crcbuf[-i] != crc32)
1910 continue;
1911
1912 if (!strcasecmp(str, index[i].str))
1913 {
1914 if (id < 0 || id >= lookup_buffer_depth)
1915 {
1916 logf("lookup buf overf.: %d", id);
1917 return false;
1918 }
1919
1920 lookup[id] = &index[i];
1921 return true;
1922 }
1923 }
1924 }
1925
1926 /* Insert to CRC buffer. */
1927 crcbuf[-tempbufidx] = crc32;
1928 tempbuf_left -= 4;
1929
1930 /* Insert it to the buffer. */
1931 tempbuf_left -= len;
1932 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1933 return false;
1934
1935 if (id >= lookup_buffer_depth)
1936 {
1937 logf("lookup buf overf. #2: %d", id);
1938 return false;
1939 }
1940
1941 if (id >= 0)
1942 {
1943 lookup[id] = &index[tempbufidx];
1944 index[tempbufidx].idlist.id = id;
1945 }
1946 else
1947 index[tempbufidx].idlist.id = -1;
1948
1949 index[tempbufidx].idlist.next = NULL;
1950 index[tempbufidx].idx_id = idx_id;
1951 index[tempbufidx].seek = -1;
1952 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1953 memcpy(index[tempbufidx].str, str, len);
1954 tempbuf_pos += len;
1955 tempbufidx++;
1956
1957 return true;
1958 }
1959
1960 static int compare(const void *p1, const void *p2)
1961 {
1962 do_timed_yield();
1963
1964 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1965 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1966
1967 if (strcmp(e1->str, UNTAGGED) == 0)
1968 {
1969 if (strcmp(e2->str, UNTAGGED) == 0)
1970 return 0;
1971 return -1;
1972 }
1973 else if (strcmp(e2->str, UNTAGGED) == 0)
1974 return 1;
1975
1976 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1977 }
1978
1979 static int tempbuf_sort(int fd)
1980 {
1981 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1982 struct tagfile_entry fe;
1983 int i;
1984 int length;
1985
1986 /* Generate reverse lookup entries. */
1987 for (i = 0; i < lookup_buffer_depth; i++)
1988 {
1989 struct tempbuf_id_list *idlist;
1990
1991 if (!lookup[i])
1992 continue;
1993
1994 if (lookup[i]->idlist.id == i)
1995 continue;
1996
1997 idlist = &lookup[i]->idlist;
1998 while (idlist->next != NULL)
1999 idlist = idlist->next;
2000
2001 tempbuf_left -= sizeof(struct tempbuf_id_list);
2002 if (tempbuf_left - 4 < 0)
2003 return -1;
2004
2005 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2006 if (tempbuf_pos & 0x03)
2007 {
2008 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2009 tempbuf_left -= 3;
2010 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2011 }
2012 tempbuf_pos += sizeof(struct tempbuf_id_list);
2013
2014 idlist = idlist->next;
2015 idlist->id = i;
2016 idlist->next = NULL;
2017
2018 do_timed_yield();
2019 }
2020
2021 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2022 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2023
2024 for (i = 0; i < tempbufidx; i++)
2025 {
2026 struct tempbuf_id_list *idlist = &index[i].idlist;
2027
2028 /* Fix the lookup list. */
2029 while (idlist != NULL)
2030 {
2031 if (idlist->id >= 0)
2032 lookup[idlist->id] = &index[i];
2033 idlist = idlist->next;
2034 }
2035
2036 index[i].seek = lseek(fd, 0, SEEK_CUR);
2037 length = strlen(index[i].str) + 1;
2038 fe.tag_length = length;
2039 fe.idx_id = index[i].idx_id;
2040
2041 /* Check the chunk alignment. */
2042 if ((fe.tag_length + sizeof(struct tagfile_entry))
2043 % TAGFILE_ENTRY_CHUNK_LENGTH)
2044 {
2045 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2046 ((fe.tag_length + sizeof(struct tagfile_entry))
2047 % TAGFILE_ENTRY_CHUNK_LENGTH);
2048 }
2049
2050 #ifdef TAGCACHE_STRICT_ALIGN
2051 /* Make sure the entry is long aligned. */
2052 if (index[i].seek & 0x03)
2053 {
2054 logf("tempbuf_sort: alignment error!");
2055 return -3;
2056 }
2057 #endif
2058
2059 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2060 sizeof(struct tagfile_entry))
2061 {
2062 logf("tempbuf_sort: write error #1");
2063 return -1;
2064 }
2065
2066 if (write(fd, index[i].str, length) != length)
2067 {
2068 logf("tempbuf_sort: write error #2");
2069 return -2;
2070 }
2071
2072 /* Write some padding. */
2073 if (fe.tag_length - length > 0)
2074 write(fd, "XXXXXXXX", fe.tag_length - length);
2075 }
2076
2077 return i;
2078 }
2079
2080 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2081 {
2082 if (id < 0 || id >= lookup_buffer_depth)
2083 return NULL;
2084
2085 return lookup[id];
2086 }
2087
2088
2089 inline static int tempbuf_find_location(int id)
2090 {
2091 struct tempbuf_searchidx *entry;
2092
2093 entry = tempbuf_locate(id);
2094 if (entry == NULL)
2095 return -1;
2096
2097 return entry->seek;
2098 }
2099
2100 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2101 {
2102 struct master_header tcmh;
2103 struct index_entry idx;
2104 int masterfd;
2105 int masterfd_pos;
2106 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2107 int max_entries;
2108 int entries_processed = 0;
2109 int i, j;
2110 char buf[TAG_MAXLEN];
2111
2112 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2113
2114 logf("Building numeric indices...");
2115 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2116
2117 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2118 return false;
2119
2120 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2121 SEEK_CUR);
2122 if (masterfd_pos == filesize(masterfd))
2123 {
2124 logf("we can't append!");
2125 close(masterfd);
2126 return false;
2127 }
2128
2129 while (entries_processed < h->entry_count)
2130 {
2131 int count = MIN(h->entry_count - entries_processed, max_entries);
2132
2133 /* Read in as many entries as possible. */
2134 for (i = 0; i < count; i++)
2135 {
2136 struct temp_file_entry *tfe = &entrybuf[i];
2137 int datastart;
2138
2139 /* Read in numeric data. */
2140 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2141 sizeof(struct temp_file_entry))
2142 {
2143 logf("read fail #1");
2144 close(masterfd);
2145 return false;
2146 }
2147
2148 datastart = lseek(tmpfd, 0, SEEK_CUR);
2149
2150 /**
2151 * Read string data from the following tags:
2152 * - tag_filename
2153 * - tag_artist
2154 * - tag_album
2155 * - tag_title
2156 *
2157 * A crc32 hash is calculated from the read data
2158 * and stored back to the data offset field kept in memory.
2159 */
2160 #define tmpdb_read_string_tag(tag) \
2161 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2162 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2163 { \
2164 logf("read fail: buffer overflow"); \
2165 close(masterfd); \
2166 return false; \
2167 } \
2168 \
2169 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2170 tfe->tag_length[tag]) \
2171 { \
2172 logf("read fail #2"); \
2173 close(masterfd); \
2174 return false; \
2175 } \
2176 \
2177 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2178 lseek(tmpfd, datastart, SEEK_SET)
2179
2180 tmpdb_read_string_tag(tag_filename);
2181 tmpdb_read_string_tag(tag_artist);
2182 tmpdb_read_string_tag(tag_album);
2183 tmpdb_read_string_tag(tag_title);
2184
2185 /* Seek to the end of the string data. */
2186 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2187 }
2188
2189 /* Backup the master index position. */
2190 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2191 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2192
2193 /* Check if we can resurrect some deleted runtime statistics data. */
2194 for (i = 0; i < tcmh.tch.entry_count; i++)
2195 {
2196 /* Read the index entry. */
2197 if (ecread_index_entry(masterfd, &idx)
2198 != sizeof(struct index_entry))
2199 {
2200 logf("read fail #3");
2201 close(masterfd);
2202 return false;
2203 }
2204
2205 /**
2206 * Skip unless the entry is marked as being deleted
2207 * or the data has already been resurrected.
2208 */
2209 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2210 continue;
2211
2212 /* Now try to match the entry. */
2213 /**
2214 * To succesfully match a song, the following conditions
2215 * must apply:
2216 *
2217 * For numeric fields: tag_length
2218 * - Full identical match is required
2219 *
2220 * If tag_filename matches, no further checking necessary.
2221 *
2222 * For string hashes: tag_artist, tag_album, tag_title
2223 * - Two of these must match
2224 */
2225 for (j = 0; j < count; j++)
2226 {
2227 struct temp_file_entry *tfe = &entrybuf[j];
2228
2229 /* Try to match numeric fields first. */
2230 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2231 continue;
2232
2233 /* Now it's time to do the hash matching. */
2234 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2235 {
2236 int match_count = 0;
2237
2238 /* No filename match, check if we can match two other tags. */
2239 #define tmpdb_match(tag) \
2240 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2241 match_count++
2242
2243 tmpdb_match(tag_artist);
2244 tmpdb_match(tag_album);
2245 tmpdb_match(tag_title);
2246
2247 if (match_count < 2)
2248 {
2249 /* Still no match found, give up. */
2250 continue;
2251 }
2252 }
2253
2254 /* A match found, now copy & resurrect the statistical data. */
2255 #define tmpdb_copy_tag(tag) \
2256 tfe->tag_offset[tag] = idx.tag_seek[tag]
2257
2258 tmpdb_copy_tag(tag_playcount);
2259 tmpdb_copy_tag(tag_rating);
2260 tmpdb_copy_tag(tag_playtime);
2261 tmpdb_copy_tag(tag_lastplayed);
2262 tmpdb_copy_tag(tag_commitid);
2263
2264 /* Avoid processing this entry again. */
2265 idx.flag |= FLAG_RESURRECTED;
2266
2267 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2268 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2269 {
2270 logf("masterfd writeback fail #1");
2271 close(masterfd);
2272 return false;
2273 }
2274
2275 logf("Entry resurrected");
2276 }
2277 }
2278
2279
2280 /* Restore the master index position. */
2281 lseek(masterfd, masterfd_pos, SEEK_SET);
2282
2283 /* Commit the data to the index. */
2284 for (i = 0; i < count; i++)
2285 {
2286 int loc = lseek(masterfd, 0, SEEK_CUR);
2287
2288 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2289 {
2290 logf("read fail #3");
2291 close(masterfd);
2292 return false;
2293 }
2294
2295 for (j = 0; j < TAG_COUNT; j++)
2296 {
2297 if (!TAGCACHE_IS_NUMERIC(j))
2298 continue;
2299
2300 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2301 }
2302 idx.flag = entrybuf[i].flag;
2303
2304 if (idx.tag_seek[tag_commitid])
2305 {
2306 /* Data has been resurrected. */
2307 idx.flag |= FLAG_DIRTYNUM;
2308 }
2309 else if (tc_stat.ready && current_tcmh.commitid > 0)
2310 {
2311 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2312 idx.flag |= FLAG_DIRTYNUM;
2313 }
2314
2315 /* Write back the updated index. */
2316 lseek(masterfd, loc, SEEK_SET);
2317 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2318 {
2319 logf("write fail");
2320 close(masterfd);
2321 return false;
2322 }
2323 }
2324
2325 entries_processed += count;
2326 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2327 }
2328
2329 close(masterfd);
2330
2331 return true;
2332 }
2333
2334 /**
2335 * Return values:
2336 * > 0 success
2337 * == 0 temporary failure
2338 * < 0 fatal error
2339 */
2340 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2341 {
2342 int i;
2343 struct tagcache_header tch;
2344 struct master_header tcmh;
2345 struct index_entry idxbuf[IDX_BUF_DEPTH];
2346 int idxbuf_pos;
2347 char buf[TAG_MAXLEN+32];
2348 int fd = -1, masterfd;
2349 bool error = false;
2350 int init;
2351 int masterfd_pos;
2352
2353 logf("Building index: %d", index_type);
2354
2355 /* Check the number of entries we need to allocate ram for. */
2356 commit_entry_count = h->entry_count + 1;
2357
2358 masterfd = open_master_fd(&tcmh, false);
2359 if (masterfd >= 0)
2360 {
2361 commit_entry_count += tcmh.tch.entry_count;
2362 close(masterfd);
2363 }
2364 else
2365 remove_files(); /* Just to be sure we are clean. */
2366
2367 /* Open the index file, which contains the tag names. */
2368 fd = open_tag_fd(&tch, index_type, true);
2369 if (fd >= 0)
2370 {
2371 logf("tch.datasize=%ld", tch.datasize);
2372 lookup_buffer_depth = 1 +
2373 /* First part */ commit_entry_count +
2374 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2375 }
2376 else
2377 {
2378 lookup_buffer_depth = 1 +
2379 /* First part */ commit_entry_count +
2380 /* Second part */ 0;
2381 }
2382
2383 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2384 logf("commit_entry_count=%ld", commit_entry_count);
2385
2386 /* Allocate buffer for all index entries from both old and new
2387 * tag files. */
2388 tempbufidx = 0;
2389 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2390
2391 /* Allocate lookup buffer. The first portion of commit_entry_count
2392 * contains the new tags in the temporary file and the second
2393 * part for locating entries already in the db.
2394 *
2395 * New tags Old tags
2396 * +---------+---------------------------+
2397 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2398 * +---------+---------------------------+
2399 *
2400 * Old tags are inserted to a temporary buffer with position:
2401 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2402 * And new tags with index:
2403 * tempbuf_insert(idx, ...);
2404 *
2405 * The buffer is sorted and written into tag file:
2406 * tempbuf_sort(...);
2407 * leaving master index locations messed up.
2408 *
2409 * That is fixed using the lookup buffer for old tags:
2410 * new_seek = tempbuf_find_location(old_seek, ...);
2411 * and for new tags:
2412 * new_seek = tempbuf_find_location(idx);
2413 */
2414 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2415 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2416 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2417
2418 /* And calculate the remaining data space used mainly for storing
2419 * tag data (strings). */
2420 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2421 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2422 {
2423 logf("Buffer way too small!");
2424 return 0;
2425 }
2426
2427 if (fd >= 0)
2428 {
2429 /**
2430 * If tag file contains unique tags (sorted index), we will load
2431 * it entirely into memory so we can resort it later for use with
2432 * chunked browsing.
2433 */
2434 if (TAGCACHE_IS_SORTED(index_type))
2435 {
2436 logf("loading tags...");
2437 for (i = 0; i < tch.entry_count; i++)
2438 {
2439 struct tagfile_entry entry;
2440 int loc = lseek(fd, 0, SEEK_CUR);
2441 bool ret;
2442
2443 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2444 {
2445 logf("read error #7");
2446 close(fd);
2447 return -2;
2448 }
2449
2450 if (entry.tag_length >= (int)sizeof(buf))
2451 {
2452 logf("too long tag #3");
2453 close(fd);
2454 return -2;
2455 }
2456
2457 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2458 {
2459 logf("read error #8");
2460 close(fd);
2461 return -2;
2462 }
2463
2464 /* Skip deleted entries. */
2465 if (buf[0] == '\0')
2466 continue;
2467
2468 /**
2469 * Save the tag and tag id in the memory buffer. Tag id
2470 * is saved so we can later reindex the master lookup
2471 * table when the index gets resorted.
2472 */
2473 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2474 + commit_entry_count, entry.idx_id,
2475 TAGCACHE_IS_UNIQUE(index_type));
2476 if (!ret)
2477 {
2478 close(fd);
2479 return -3;
2480 }
2481 do_timed_yield();
2482 }
2483 logf("done");
2484 }
2485 else
2486 tempbufidx = tch.entry_count;
2487 }
2488 else
2489 {
2490 /**
2491 * Creating new index file to store the tags. No need to preload
2492 * anything whether the index type is sorted or not.
2493 */
2494 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2495 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2496 if (fd < 0)
2497 {
2498 logf("%s open fail", buf);
2499 return -2;
2500 }
2501
2502 tch.magic = TAGCACHE_MAGIC;
2503 tch.entry_count = 0;
2504 tch.datasize = 0;
2505
2506 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2507 != sizeof(struct tagcache_header))
2508 {
2509 logf("header write failed");
2510 close(fd);
2511 return -2;
2512 }
2513 }
2514
2515 /* Loading the tag lookup file as "master file". */
2516 logf("Loading index file");
2517 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2518
2519 if (masterfd < 0)
2520 {
2521 logf("Creating new DB");
2522 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2523
2524 if (masterfd < 0)
2525 {
2526 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2527 close(fd);
2528 return -2;
2529 }
2530
2531 /* Write the header (write real values later). */
2532 memset(&tcmh, 0, sizeof(struct master_header));
2533 tcmh.tch = *h;
2534 tcmh.tch.entry_count = 0;
2535 tcmh.tch.datasize = 0;
2536 tcmh.dirty = true;
2537 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2538 init = true;
2539 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2540 }
2541 else
2542 {
2543 /**
2544 * Master file already exists so we need to process the current
2545 * file first.
2546 */
2547 init = false;
2548
2549 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2550 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2551 {
2552 logf("header error");
2553 close(fd);
2554 close(masterfd);
2555 return -2;
2556 }
2557
2558 /**
2559 * If we reach end of the master file, we need to expand it to
2560 * hold new tags. If the current index is not sorted, we can
2561 * simply append new data to end of the file.
2562 * However, if the index is sorted, we need to update all tag
2563 * pointers in the master file for the current index.
2564 */
2565 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2566 SEEK_CUR);
2567 if (masterfd_pos == filesize(masterfd))
2568 {
2569 logf("appending...");
2570 init = true;
2571 }
2572 }
2573
2574 /**
2575 * Load new unique tags in memory to be sorted later and added
2576 * to the master lookup file.
2577 */
2578 if (TAGCACHE_IS_SORTED(index_type))
2579 {
2580 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2581 /* h is the header of the temporary file containing new tags. */
2582 logf("inserting new tags...");
2583 for (i = 0; i < h->entry_count; i++)
2584 {
2585 struct temp_file_entry entry;
2586
2587 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2588 sizeof(struct temp_file_entry))
2589 {
2590 logf("read fail #3");
2591 error = true;
2592 goto error_exit;
2593 }
2594
2595 /* Read data. */
2596 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2597 {
2598 logf("too long entry!");
2599 error = true;
2600 goto error_exit;
2601 }
2602
2603 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2604 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2605 entry.tag_length[index_type])
2606 {
2607 logf("read fail #4");
2608 error = true;
2609 goto error_exit;
2610 }
2611
2612 if (TAGCACHE_IS_UNIQUE(index_type))
2613 error = !tempbuf_insert(buf, i, -1, true);
2614 else
2615 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2616
2617 if (error)
2618 {
2619 logf("insert error");
2620 goto error_exit;
2621 }
2622
2623 /* Skip to next. */
2624 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2625 entry.tag_length[index_type], SEEK_CUR);
2626 do_timed_yield();
2627 }
2628 logf("done");
2629
2630 /* Sort the buffer data and write it to the index file. */
2631 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2632 /**
2633 * We need to truncate the index file now. There can be junk left
2634 * at the end of file (however, we _should_ always follow the
2635 * entry_count and don't crash with that).
2636 */
2637 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2638
2639 i = tempbuf_sort(fd);
2640 if (i < 0)
2641 goto error_exit;
2642 logf("sorted %d tags", i);
2643
2644 /**
2645 * Now update all indexes in the master lookup file.
2646 */
2647 logf("updating indices...");
2648 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2649 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2650 {
2651 int j;
2652 int loc = lseek(masterfd, 0, SEEK_CUR);
2653
2654 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2655
2656 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2657 != (int)sizeof(struct index_entry)*idxbuf_pos)
2658 {
2659 logf("read fail #5");
2660 error = true;
2661 goto error_exit ;
2662 }
2663 lseek(masterfd, loc, SEEK_SET);
2664
2665 for (j = 0; j < idxbuf_pos; j++)
2666 {
2667 if (idxbuf[j].flag & FLAG_DELETED)
2668 {
2669 /* We can just ignore deleted entries. */
2670 // idxbuf[j].tag_seek[index_type] = 0;
2671 continue;
2672 }
2673
2674 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2675 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2676 + commit_entry_count);
2677
2678 if (idxbuf[j].tag_seek[index_type] < 0)
2679 {
2680 logf("update error: %ld/%d/%ld",
2681 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2682 error = true;
2683 goto error_exit;
2684 }
2685
2686 do_timed_yield();
2687 }
2688
2689 /* Write back the updated index. */
2690 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2691 index_entry_ec, tc_stat.econ) !=
2692 (int)sizeof(struct index_entry)*idxbuf_pos)
2693 {
2694 logf("write fail");
2695 error = true;
2696 goto error_exit;
2697 }
2698 }
2699 logf("done");
2700 }
2701
2702 /**
2703 * Walk through the temporary file containing the new tags.
2704 */
2705 // build_normal_index(h, tmpfd, masterfd, idx);
2706 logf("updating new indices...");
2707 lseek(masterfd, masterfd_pos, SEEK_SET);
2708 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2709 lseek(fd, 0, SEEK_END);
2710 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2711 {
2712 int j;
2713
2714 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2715 if (init)
2716 {
2717 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2718 }
2719 else
2720 {
2721 int loc = lseek(masterfd, 0, SEEK_CUR);
2722
2723 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2724 != (int)sizeof(struct index_entry)*idxbuf_pos)
2725 {
2726 logf("read fail #6");
2727 error = true;
2728 break ;
2729 }
2730 lseek(masterfd, loc, SEEK_SET);
2731 }
2732
2733 /* Read entry headers. */
2734 for (j = 0; j < idxbuf_pos; j++)
2735 {
2736 if (!TAGCACHE_IS_SORTED(index_type))
2737 {
2738 struct temp_file_entry entry;
2739 struct tagfile_entry fe;
2740
2741 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2742 sizeof(struct temp_file_entry))
2743 {
2744 logf("read fail #7");
2745 error = true;
2746 break ;
2747 }
2748
2749 /* Read data. */
2750 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2751 {
2752 logf("too long entry!");
2753 logf("length=%d", entry.tag_length[index_type]);
2754 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2755 error = true;
2756 break ;
2757 }
2758
2759 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2760 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2761 entry.tag_length[index_type])
2762 {
2763 logf("read fail #8");
2764 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2765 logf("length=0x%02x", entry.tag_length[index_type]);
2766 error = true;
2767 break ;
2768 }
2769
2770 /* Write to index file. */
2771 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2772 fe.tag_length = entry.tag_length[index_type];
2773 fe.idx_id = tcmh.tch.entry_count + i + j;
2774 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2775 write(fd, buf, fe.tag_length);
2776 tempbufidx++;
2777
2778 /* Skip to next. */
2779 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2780 entry.tag_length[index_type], SEEK_CUR);
2781 }
2782 else
2783 {
2784 /* Locate the correct entry from the sorted array. */
2785 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2786 if (idxbuf[j].tag_seek[index_type] < 0)
2787 {
2788 logf("entry not found (%d)", j);
2789 error = true;
2790 break ;
2791 }
2792 }
2793 }
2794
2795 /* Write index. */
2796 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2797 index_entry_ec, tc_stat.econ) !=
2798 (int)sizeof(struct index_entry)*idxbuf_pos)
2799 {
2800 logf("tagcache: write fail #4");
2801 error = true;
2802 break ;
2803 }
2804
2805 do_timed_yield();
2806 }
2807 logf("done");
2808
2809 /* Finally write the header. */
2810 tch.magic = TAGCACHE_MAGIC;
2811 tch.entry_count = tempbufidx;
2812 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2813 lseek(fd, 0, SEEK_SET);
2814 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2815
2816 if (index_type != tag_filename)
2817 h->datasize += tch.datasize;
2818 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2819 error_exit:
2820
2821 close(fd);
2822 close(masterfd);
2823
2824 if (error)
2825 return -2;
2826
2827 return 1;
2828 }
2829
2830 static bool commit(void)
2831 {
2832 struct tagcache_header tch;
2833 struct master_header tcmh;
2834 int i, len, rc;
2835 int tmpfd;
2836 int masterfd;
2837 #ifdef HAVE_DIRCACHE
2838 bool dircache_buffer_stolen = false;
2839 #endif
2840 bool local_allocation = false;
2841
2842 logf("committing tagcache");
2843
2844 while (write_lock)
2845 sleep(1);
2846
2847 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2848 if (tmpfd < 0)
2849 {
2850 logf("nothing to commit");
2851 return true;
2852 }
2853
2854
2855 /* Load the header. */
2856 len = sizeof(struct tagcache_header);
2857 rc = read(tmpfd, &tch, len);
2858
2859 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2860 {
2861 logf("incorrect header");
2862 close(tmpfd);
2863 remove(TAGCACHE_FILE_TEMP);
2864 return false;
2865 }
2866
2867 if (tch.entry_count == 0)
2868 {
2869 logf("nothing to commit");
2870 close(tmpfd);
2871 remove(TAGCACHE_FILE_TEMP);
2872 return true;
2873 }
2874
2875 #ifdef HAVE_EEPROM_SETTINGS
2876 remove(TAGCACHE_STATEFILE);
2877 #endif
2878
2879 /* At first be sure to unload the ramcache! */
2880 #ifdef HAVE_TC_RAMCACHE
2881 tc_stat.ramcache = false;
2882 #endif
2883
2884 read_lock++;
2885
2886 /* Try to steal every buffer we can :) */
2887 if (tempbuf_size == 0)
2888 local_allocation = true;
2889
2890 #ifdef HAVE_DIRCACHE
2891 if (tempbuf_size == 0)
2892 {
2893 /* Try to steal the dircache buffer. */
2894 tempbuf = dircache_steal_buffer(&tempbuf_size);
2895 tempbuf_size &= ~0x03;
2896
2897 if (tempbuf_size > 0)
2898 {
2899 dircache_buffer_stolen = true;
2900 }
2901 }
2902 #endif
2903
2904 #ifdef HAVE_TC_RAMCACHE
2905 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2906 {
2907 tempbuf = (char *)(hdr + 1);
2908 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2909 tempbuf_size &= ~0x03;
2910 }
2911 #endif
2912
2913 /* And finally fail if there are no buffers available. */
2914 if (tempbuf_size == 0)
2915 {
2916 logf("delaying commit until next boot");
2917 tc_stat.commit_delayed = true;
2918 close(tmpfd);
2919 read_lock--;
2920 return false;
2921 }
2922
2923 logf("commit %ld entries...", tch.entry_count);
2924
2925 /* Mark DB dirty so it will stay disabled if commit fails. */
2926 current_tcmh.dirty = true;
2927 update_master_header();
2928
2929 /* Now create the index files. */
2930 tc_stat.commit_step = 0;
2931 tch.datasize = 0;
2932 tc_stat.commit_delayed = false;
2933
2934 for (i = 0; i < TAG_COUNT; i++)
2935 {
2936 int ret;
2937
2938 if (TAGCACHE_IS_NUMERIC(i))
2939 continue;
2940
2941 tc_stat.commit_step++;
2942 ret = build_index(i, &tch, tmpfd);
2943 if (ret <= 0)
2944 {
2945 close(tmpfd);
2946 logf("tagcache failed init");
2947 if (ret == 0)
2948 tc_stat.commit_delayed = true;
2949
2950 tc_stat.commit_step = 0;
2951 read_lock--;
2952 return false;
2953 }
2954 }
2955
2956 if (!build_numeric_indices(&tch, tmpfd))
2957 {
2958 logf("Failure to commit numeric indices");
2959 close(tmpfd);
2960 tc_stat.commit_step = 0;
2961 read_lock--;
2962 return false;
2963 }
2964
2965 close(tmpfd);
2966 remove(TAGCACHE_FILE_TEMP);
2967
2968 tc_stat.commit_step = 0;
2969
2970 /* Update the master index headers. */
2971 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2972 {
2973 read_lock--;
2974 return false;
2975 }
2976
2977 tcmh.tch.entry_count += tch.entry_count;
2978 tcmh.tch.datasize = sizeof(struct master_header)
2979 + sizeof(struct index_entry) * tcmh.tch.entry_count
2980 + tch.datasize;
2981 tcmh.dirty = false;
2982 tcmh.commitid++;
2983
2984 lseek(masterfd, 0, SEEK_SET);
2985 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2986 close(masterfd);
2987
2988 logf("tagcache committed");
2989 tc_stat.ready = check_all_headers();
2990 tc_stat.readyvalid = true;
2991
2992 if (local_allocation)
2993 {
2994 tempbuf = NULL;
2995 tempbuf_size = 0;
2996 }
2997
2998 #ifdef HAVE_DIRCACHE
2999 /* Rebuild the dircache, if we stole the buffer. */
3000 if (dircache_buffer_stolen)
3001 dircache_build(0);
3002 #endif
3003
3004 #ifdef HAVE_TC_RAMCACHE
3005 /* Reload tagcache. */
3006 if (tc_stat.ramcache_allocated > 0)
3007 tagcache_start_scan();
3008 #endif
3009
3010 read_lock--;
3011
3012 return true;
3013 }
3014
3015 static void allocate_tempbuf(void)
3016 {
3017 /* Yeah, malloc would be really nice now :) */
3018 #ifdef __PCTOOL__
3019 tempbuf_size = 32*1024*1024;
3020 tempbuf = malloc(tempbuf_size);
3021 #else
3022 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3023 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3024 audiobuf += tempbuf_size;
3025 #endif
3026 }
3027
3028 static void free_tempbuf(void)
3029 {
3030 if (tempbuf_size == 0)
3031 return ;
3032
3033 #ifdef __PCTOOL__
3034 free(tempbuf);
3035 #else
3036 audiobuf -= tempbuf_size;
3037 #endif
3038 tempbuf = NULL;
3039 tempbuf_size = 0;
3040 }
3041
3042 #ifndef __PCTOOL__
3043
3044 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3045 {
3046 struct index_entry idx;
3047
3048 if (!tc_stat.ready)
3049 return false;
3050
3051 if (!TAGCACHE_IS_NUMERIC(tag))
3052 return false;
3053
3054 if (!get_index(masterfd, idx_id, &idx, false))
3055 return false;
3056
3057 idx.tag_seek[tag] = data;
3058 idx.flag |= FLAG_DIRTYNUM;
3059
3060 return write_index(masterfd, idx_id, &idx);
3061 }
3062
3063 #if 0
3064 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3065 int tag, long data)
3066 {
3067 struct master_header myhdr;
3068
3069 if (tcs->masterfd < 0)
3070 {
3071 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3072 return false;
3073 }
3074
3075 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3076 }
3077 #endif
3078
3079 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3080
3081 static bool command_queue_is_full(void)
3082 {
3083 int next;
3084
3085 next = command_queue_widx + 1;
3086 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3087 next = 0;
3088
3089 return (next == command_queue_ridx);
3090 }
3091
3092 static void command_queue_sync_callback(void *data)
3093 {
3094 (void)data;
3095 struct master_header myhdr;
3096 int masterfd;
3097
3098 mutex_lock(&command_queue_mutex);
3099
3100 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3101 return;
3102
3103 while (command_queue_ridx != command_queue_widx)
3104 {
3105 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3106
3107 switch (ce->command)
3108 {
3109 case CMD_UPDATE_MASTER_HEADER:
3110 {
3111 close(masterfd);
3112 update_master_header();
3113
3114 /* Re-open the masterfd. */
3115 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3116 return;
3117
3118 break;
3119 }
3120 case CMD_UPDATE_NUMERIC:
3121 {
3122 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3123 break;
3124 }
3125 }
3126
3127 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3128 command_queue_ridx = 0;
3129 }
3130
3131 close(masterfd);
3132
3133 tc_stat.queue_length = 0;
3134 mutex_unlock(&command_queue_mutex);
3135 }
3136
3137 static void run_command_queue(bool force)
3138 {
3139 if (COMMAND_QUEUE_IS_EMPTY)
3140 return;
3141
3142 if (force || command_queue_is_full())
3143 command_queue_sync_callback(NULL);
3144 else
3145 register_storage_idle_func(command_queue_sync_callback);
3146 }
3147
3148 static void queue_command(int cmd, long idx_id, int tag, long data)
3149 {
3150 while (1)
3151 {
3152 int next;
3153
3154 mutex_lock(&command_queue_mutex);
3155 next = command_queue_widx + 1;
3156 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3157 next = 0;
3158
3159 /* Make sure queue is not full. */
3160 if (next != command_queue_ridx)
3161 {
3162 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3163
3164 ce->command = cmd;
3165 ce->idx_id = idx_id;
3166 ce->tag = tag;
3167 ce->data = data;
3168
3169 command_queue_widx = next;
3170
3171 tc_stat.queue_length++;
3172
3173 mutex_unlock(&command_queue_mutex);
3174 break;
3175 }
3176
3177 /* Queue is full, try again later... */
3178 mutex_unlock(&command_queue_mutex);
3179 sleep(1);
3180 }
3181 }
3182
3183 long tagcache_increase_serial(void)
3184 {
3185 long old;
3186
3187 if (!tc_stat.ready)
3188 return -2;
3189
3190 while (read_lock)
3191 sleep(1);
3192
3193 old = current_tcmh.serial++;
3194 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3195
3196 return old;
3197 }
3198
3199 void tagcache_update_numeric(int idx_id, int tag, long data)
3200 {
3201 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3202 }
3203 #endif /* !__PCTOOL__ */
3204
3205 long tagcache_get_serial(void)
3206 {
3207 return current_tcmh.serial;
3208 }
3209
3210 long tagcache_get_commitid(void)
3211 {
3212 return current_tcmh.commitid;
3213 }
3214
3215 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3216 {
3217 char buf[512];
3218 int i;
3219
3220 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3221 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3222 {
3223 if (*datastr == '\0')
3224 break;
3225
3226 if (*datastr == '"' || *datastr == '\\')
3227 buf[i++] = '\\';
3228
3229 buf[i] = *(datastr++);
3230 }
3231
3232 strcpy(&buf[i], "\" ");
3233
3234 write(fd, buf, i + 2);
3235
3236 return true;
3237 }
3238
3239 #ifndef __PCTOOL__
3240
3241 static bool read_tag(char *dest, long size,
3242 const char *src, const char *tagstr)
3243 {
3244 int pos;
3245 char current_tag[32];
3246
3247 while (*src != '\0')
3248 {
3249 /* Skip all whitespace */
3250 while (*src == ' ')
3251 src++;
3252
3253 if (*src == '\0')
3254 break;
3255
3256 pos = 0;
3257 /* Read in tag name */
3258 while (*src != '=' && *src != ' ')
3259 {
3260 current_tag[pos] = *src;
3261 src++;
3262 pos++;
3263
3264 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3265 return false;
3266 }
3267 current_tag[pos] = '\0';
3268
3269 /* Read in tag data */
3270
3271 /* Find the start. */
3272 while (*src != '"' && *src != '\0')
3273 src++;
3274
3275 if (*src == '\0' || *(++src) == '\0')
3276 return false;
3277
3278 /* Read the data. */
3279 for (pos = 0; pos < size; pos++)
3280 {
3281 if (*src == '\0')
3282 break;
3283
3284 if (*src == '\\')
3285 {
3286 dest[pos] = *(src+1);
3287 src += 2;
3288 continue;
3289 }
3290
3291 dest[pos] = *src;
3292
3293 if (*src == '"')
3294 {
3295 src++;
3296 break;
3297 }
3298
3299 if (*src == '\0')
3300 break;
3301
3302 src++;
3303 }
3304 dest[pos] = '\0';
3305
3306 if (!strcasecmp(tagstr, current_tag))
3307 return true;
3308 }
3309
3310 return false;
3311 }
3312
3313 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3314 {
3315 struct index_entry idx;
3316 char tag_data[TAG_MAXLEN+32];
3317 int idx_id;
3318 long masterfd = (long)parameters;
3319 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3320 tag_commitid };
3321 int i;
3322 (void)line_n;
3323
3324 if (*buf == '#')
3325 return 0;
3326
3327 logf("%d/%s", line_n, buf);
3328 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3329 {
3330 logf("filename missing");
3331 logf("-> %s", buf);
3332 return 0;
3333 }
3334
3335 idx_id = find_index(tag_data);
3336 if (idx_id < 0)
3337 {
3338 logf("entry not found");
3339 return 0;
3340 }
3341
3342 if (!get_index(masterfd, idx_id, &idx, false))
3343 {
3344 logf("failed to retrieve index entry");
3345 return 0;
3346 }
3347
3348 /* Stop if tag has already been modified. */
3349 if (idx.flag & FLAG_DIRTYNUM)
3350 return 0;
3351
3352 logf("import: %s", tag_data);
3353
3354 idx.flag |= FLAG_DIRTYNUM;
3355 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3356 {
3357 int data;
3358
3359 if (!read_tag(tag_data, sizeof tag_data, buf,
3360 tagcache_tag_to_str(import_tags[i])))
3361 {
3362 continue;
3363 }
3364
3365 data = atoi(tag_data);
3366 if (data < 0)
3367 continue;
3368
3369 idx.tag_seek[import_tags[i]] = data;
3370
3371 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3372 current_tcmh.serial = data + 1;
3373 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3374 current_tcmh.commitid = data + 1;
3375 }
3376
3377 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3378 }
3379
3380 bool tagcache_import_changelog(void)
3381 {
3382 struct master_header myhdr;
3383 struct tagcache_header tch;
3384 int clfd;
3385 long masterfd;
3386 char buf[2048];
3387
3388 if (!tc_stat.ready)
3389 return false;
3390
3391 while (read_lock)
3392 sleep(1);
3393
3394 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3395 if (clfd < 0)
3396 {
3397 logf("failure to open changelog");
3398 return false;
3399 }
3400
3401 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3402 {
3403 close(clfd);
3404 return false;
3405 }
3406
3407 write_lock++;
3408
3409 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3410
3411 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3412 parse_changelog_line);
3413
3414 close(clfd);
3415 close(masterfd);
3416
3417 if (filenametag_fd >= 0)
3418 {
3419 close(filenametag_fd);
3420 filenametag_fd = -1;
3421 }
3422
3423 write_lock--;
3424
3425 update_master_header();
3426
3427 return true;
3428 }
3429
3430 #endif /* !__PCTOOL__ */
3431
3432 bool tagcache_create_changelog(struct tagcache_search *tcs)
3433 {
3434 struct master_header myhdr;
3435 struct index_entry idx;
3436 char buf[TAG_MAXLEN+32];
3437 char temp[32];
3438 int clfd;
3439 int i, j;
3440
3441 if (!tc_stat.ready)
3442 return false;
3443
3444 if (!tagcache_search(tcs, tag_filename))
3445 return false;
3446
3447 /* Initialize the changelog */
3448 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3449 if (clfd < 0)
3450 {
3451 logf("failure to open changelog");
3452 return false;
3453 }
3454
3455 if (tcs->masterfd < 0)
3456 {
3457 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3458 return false;
3459 }
3460 else
3461 {
3462 lseek(tcs->masterfd, 0, SEEK_SET);
3463 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3464 }
3465
3466 write(clfd, "## Changelog version 1\n", 23);
3467
3468 for (i = 0; i < myhdr.tch.entry_count; i++)
3469 {
3470 if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3471 {
3472 logf("read error #9");
3473 tagcache_search_finish(tcs);
3474 close(clfd);
3475 return false;
3476 }
3477
3478 /* Skip until the entry found has been modified. */
3479 if (! (idx.flag & FLAG_DIRTYNUM) )
3480 continue;
3481
3482 /* Skip deleted entries too. */
3483 if (idx.flag & FLAG_DELETED)
3484 continue;
3485
3486 /* Now retrieve all tags. */
3487 for (j = 0; j < TAG_COUNT; j++)
3488 {
3489 if (TAGCACHE_IS_NUMERIC(j))
3490 {
3491 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3492 write_tag(clfd, tagcache_tag_to_str(j), temp);
3493 continue;
3494 }
3495
3496 tcs->type = j;
3497 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3498 write_tag(clfd, tagcache_tag_to_str(j), buf);
3499 }
3500
3501 write(clfd, "\n", 1);
3502 do_timed_yield();
3503 }
3504
3505 close(clfd);
3506
3507 tagcache_search_finish(tcs);
3508
3509 return true;
3510 }
3511
3512 static bool delete_entry(long idx_id)
3513 {
3514 int fd = -1;
3515 int masterfd = -1;
3516 int tag, i;
3517 struct index_entry idx, myidx;
3518 struct master_header myhdr;
3519 char buf[TAG_MAXLEN+32];
3520 int in_use[TAG_COUNT];
3521
3522 logf("delete_entry(): %ld", idx_id);
3523
3524 #ifdef HAVE_TC_RAMCACHE
3525 /* At first mark the entry removed from ram cache. */
3526 if (tc_stat.ramcache)
3527 hdr->indices[idx_id].flag |= FLAG_DELETED;
3528 #endif
3529
3530 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3531 return false;
3532
3533 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3534 if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3535 {
3536 logf("delete_entry(): read error");
3537 goto cleanup;
3538 }
3539
3540 if (myidx.flag & FLAG_DELETED)
3541 {
3542 logf("delete_entry(): already deleted!");
3543 goto cleanup;
3544 }
3545
3546 myidx.flag |= FLAG_DELETED;
3547 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3548 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3549 {
3550 logf("delete_entry(): write_error #1");
3551 goto cleanup;
3552 }
3553
3554 /* Now check which tags are no longer in use (if any) */
3555 for (tag = 0; tag < TAG_COUNT; tag++)
3556 in_use[tag] = 0;
3557
3558 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3559 for (i = 0; i < myhdr.tch.entry_count; i++)
3560 {
3561 struct index_entry *idxp;
3562
3563 #ifdef HAVE_TC_RAMCACHE
3564 /* Use RAM DB if available for greater speed */
3565 if (tc_stat.ramcache)
3566 idxp = &hdr->indices[i];
3567 else
3568 #endif
3569 {
3570 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3571 {
3572 logf("delete_entry(): read error #2");
3573 goto cleanup;
3574 }
3575 idxp = &idx;
3576 }
3577
3578 if (idxp->flag & FLAG_DELETED)
3579 continue;
3580
3581 for (tag = 0; tag < TAG_COUNT; tag++)
3582 {
3583 if (TAGCACHE_IS_NUMERIC(tag))
3584 continue;
3585
3586 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3587 in_use[tag]++;
3588 }
3589 }
3590
3591 /* Now delete all tags no longer in use. */
3592 for (tag = 0; tag < TAG_COUNT; tag++)
3593 {
3594 struct tagcache_header tch;
3595 int oldseek = myidx.tag_seek[tag];
3596
3597 if (TAGCACHE_IS_NUMERIC(tag))
3598 continue;
3599
3600 /**
3601 * Replace tag seek with a hash value of the field string data.
3602 * That way runtime statistics of moved or altered files can be
3603 * resurrected.
3604 */
3605 #ifdef HAVE_TC_RAMCACHE
3606 if (tc_stat.ramcache && tag != tag_filename)
3607 {
3608 struct tagfile_entry *tfe;
3609 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3610
3611 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3612 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3613 myidx.tag_seek[tag] = *seek;
3614 }
3615 else
3616 #endif
3617 {
3618 struct tagfile_entry tfe;
3619
3620 /* Open the index file, which contains the tag names. */
3621 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3622 goto cleanup;
3623
3624 /* Skip the header block */
3625 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3626 if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3627 {
3628 logf("delete_entry(): read error #3");
3629 goto cleanup;
3630 }
3631
3632 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3633 {
3634 logf("delete_entry(): read error #4");
3635 goto cleanup;
3636 }
3637
3638 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3639 }
3640
3641 if (in_use[tag])
3642 {
3643 logf("in use: %d/%d", tag, in_use[tag]);
3644 if (fd >= 0)
3645 {
3646 close(fd);
3647 fd = -1;
3648 }
3649 continue;
3650 }
3651
3652 #ifdef HAVE_TC_RAMCACHE
3653 /* Delete from ram. */
3654 if (tc_stat.ramcache && tag != tag_filename)
3655 {
3656 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3657 tagentry->tag_data[0] = '\0';
3658 }
3659 #endif
3660
3661 /* Open the index file, which contains the tag names. */
3662 if (fd < 0)
3663 {
3664 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3665 goto cleanup;
3666 }
3667
3668 /* Skip the header block */
3669 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3670
3671 /* Debug, print 10 first characters of the tag
3672 read(fd, buf, 10);
3673 buf[10]='\0';
3674 logf("TAG:%s", buf);
3675 lseek(fd, -10, SEEK_CUR);
3676 */
3677
3678 /* Write first data byte in tag as \0 */
3679 write(fd, "", 1);
3680
3681 /* Now tag data has been removed */
3682 close(fd);
3683 fd = -1;
3684 }
3685
3686 /* Write index entry back into master index. */
3687 lseek(masterfd, sizeof(struct master_header) +
3688 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3689 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3690 {
3691 logf("delete_entry(): write_error #2");
3692 goto cleanup;
3693 }
3694
3695 close(masterfd);
3696
3697 return true;
3698
3699 cleanup:
3700 if (fd >= 0)
3701 close(fd);
3702 if (masterfd >= 0)
3703 close(masterfd);
3704
3705 return false;
3706 }
3707
3708 #ifndef __PCTOOL__
3709 /**
3710 * Returns true if there is an event waiting in the queue
3711 * that requires the current operation to be aborted.
3712 */
3713 static bool check_event_queue(void)
3714 {
3715 struct queue_event ev;
3716
3717 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3718 switch (ev.id)
3719 {
3720 case Q_STOP_SCAN:
3721 case SYS_POWEROFF:
3722 case SYS_USB_CONNECTED:
3723 /* Put the event back into the queue. */
3724 queue_post(&tagcache_queue, ev.id, ev.data);
3725 return true;
3726 }
3727
3728 return false;
3729 }
3730 #endif
3731
3732 #ifdef HAVE_TC_RAMCACHE
3733 static bool allocate_tagcache(void)
3734 {
3735 struct master_header tcmh;
3736 int fd;
3737
3738 /* Load the header. */
3739 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3740 {
3741 hdr = NULL;
3742 return false;
3743 }
3744
3745 close(fd);
3746
3747 /**
3748 * Now calculate the required cache size plus
3749 * some extra space for alignment fixes.
3750 */
3751 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3752 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3753 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3754 memset(hdr, 0, sizeof(struct ramcache_header));
3755 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3756 hdr->indices = (struct index_entry *)(hdr + 1);
3757 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3758
3759 return true;
3760 }
3761
3762 # ifdef HAVE_EEPROM_SETTINGS
3763 static bool tagcache_dumpload(void)
3764 {
3765 struct statefile_header shdr;
3766 int fd, rc;
3767 long offpos;
3768 int i;
3769
3770 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3771 if (fd < 0)
3772 {
3773 logf("no tagcache statedump");
3774 return false;
3775 }
3776
3777 /* Check the statefile memory placement */
3778 hdr = buffer_alloc(0);
3779 rc = read(fd, &shdr, sizeof(struct statefile_header));
3780 if (rc != sizeof(struct statefile_header)
3781 /* || (long)hdr != (long)shdr.hdr */)
3782 {
3783 logf("incorrect statefile");
3784 hdr = NULL;
3785 close(fd);
3786 return false;
3787 }
3788
3789 offpos = (long)hdr - (long)shdr.hdr;
3790
3791 /* Lets allocate real memory and load it */
3792 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3793 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3794 close(fd);
3795
3796 if (rc != shdr.tc_stat.ramcache_allocated)
3797 {
3798 logf("read failure!");
3799 hdr = NULL;
3800 return false;
3801 }
3802
3803 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3804
3805 /* Now fix the pointers */
3806 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3807 for (i = 0; i < TAG_COUNT; i++)
3808 hdr->tags[i] += offpos;
3809
3810 return true;
3811 }
3812
3813 static bool tagcache_dumpsave(void)
3814 {
3815 struct statefile_header shdr;
3816 int fd;
3817
3818 if (!tc_stat.ramcache)
3819 return false;
3820
3821 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3822 if (fd < 0)
3823 {
3824 logf("failed to create a statedump");
3825 return false;
3826 }
3827
3828 /* Create the header */
3829 shdr.hdr = hdr;
3830 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3831 write(fd, &shdr, sizeof(struct statefile_header));
3832
3833 /* And dump the data too */
3834 write(fd, hdr, tc_stat.ramcache_allocated);
3835 close(fd);
3836
3837 return true;
3838 }
3839 # endif
3840
3841 static bool load_tagcache(void)
3842 {
3843 struct tagcache_header *tch;
3844 long bytesleft = tc_stat.ramcache_allocated;
3845 struct index_entry *idx;
3846 int rc, fd;
3847 char *p;
3848 int i, tag;
3849
3850 # ifdef HAVE_DIRCACHE
3851 while (dircache_is_initializing())
3852 sleep(1);
3853
3854 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3855 # endif
3856
3857 logf("loading tagcache to ram...");
3858
3859 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3860 if (fd < 0)
3861 {
3862 logf("tagcache open failed");
3863 return false;
3864 }
3865
3866 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3867 != sizeof(struct master_header)
3868 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3869 {
3870 logf("incorrect header");
3871 return false;
3872 }
3873
3874 idx = hdr->indices;
3875
3876 /* Load the master index table. */
3877 for (i = 0; i < hdr->h.tch.entry_count; i++)
3878 {
3879 rc = ecread_index_entry(fd, idx);
3880 if (rc != sizeof(struct index_entry))
3881 {
3882 logf("read error #10");
3883 close(fd);
3884 return false;
3885 }
3886
3887 bytesleft -= sizeof(struct index_entry);
3888 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3889 {
3890 logf("too big tagcache.");
3891 close(fd);
3892 return false;
3893 }
3894
3895 idx++;
3896 }
3897
3898 close(fd);
3899
3900 /* Load the tags. */
3901 p = (char *)idx;
3902 for (tag = 0; tag < TAG_COUNT; tag++)
3903 {
3904 struct tagfile_entry *fe;
3905 char buf[TAG_MAXLEN+32];
3906
3907 if (TAGCACHE_IS_NUMERIC(tag))
3908 continue ;
3909
3910 //p = ((void *)p+1);
3911 p = (char *)((long)p & ~0x03) + 0x04;
3912 hdr->tags[tag] = p;
3913
3914 /* Check the header. */
3915 tch = (struct tagcache_header *)p;
3916 p += sizeof(struct tagcache_header);
3917
3918 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3919 return false;
3920
3921 for (hdr->entry_count[tag] = 0;
3922 hdr->entry_count[tag] < tch->entry_count;
3923 hdr->entry_count[tag]++)
3924 {
3925 long pos;
3926
3927 if (do_timed_yield())
3928 {
3929 /* Abort if we got a critical event in queue */
3930 if (check_event_queue())
3931 return false;
3932 }
3933
3934 fe = (struct tagfile_entry *)p;
3935 pos = lseek(fd, 0, SEEK_CUR);
3936 rc = ecread_tagfile_entry(fd, fe);
3937 if (rc != sizeof(struct tagfile_entry))
3938 {
3939 /* End of lookup table. */
3940 logf("read error #11");
3941 close(fd);
3942 return false;
3943 }
3944
3945 /* We have a special handling for the filename tags. */
3946 if (tag == tag_filename)
3947 {
3948 # ifdef HAVE_DIRCACHE
3949 const struct dircache_entry *dc;
3950 # endif
3951
3952 // FIXME: This is wrong!
3953 // idx = &hdr->indices[hdr->entry_count[i]];
3954 idx = &hdr->indices[fe->idx_id];
3955
3956 if (fe->tag_length >= (long)sizeof(buf)-1)
3957 {
3958 read(fd, buf, 10);
3959 buf[10] = '\0';
3960 logf("TAG:%s", buf);
3961 logf("too long filename");
3962 close(fd);
3963 return false;
3964 }
3965
3966 rc = read(fd, buf, fe->tag_length);
3967 if (rc != fe->tag_length)
3968 {
3969 logf("read error #12");
3970 close(fd);
3971 return false;
3972 }
3973
3974 /* Check if the entry has already been removed */
3975 if (idx->flag & FLAG_DELETED)
3976 continue;
3977
3978 /* This flag must not be used yet. */
3979 if (idx->flag & FLAG_DIRCACHE)
3980 {
3981 logf("internal error!");
3982 close(fd);
3983 return false;
3984 }
3985
3986 if (idx->tag_seek[tag] != pos)
3987 {
3988 logf("corrupt data structures!");
3989 close(fd);
3990 return false;
3991 }
3992
3993 # ifdef HAVE_DIRCACHE
3994 if (dircache_is_enabled())
3995 {
3996 dc = dircache_get_entry_ptr(buf);
3997 if (dc == NULL)
3998 {
3999 logf("Entry no longer valid.");
4000 logf("-> %s", buf);
4001 if (global_settings.tagcache_autoupdate)
4002 delete_entry(fe->idx_id);
4003 continue ;
4004 }
4005
4006 idx->flag |= FLAG_DIRCACHE;
4007 idx->tag_seek[tag_filename] = (long)dc;
4008 }
4009 else
4010 # endif
4011 {
4012 /* This will be very slow unless dircache is enabled
4013 or target is flash based, but do it anyway for
4014 consistency. */
4015 /* Check if entry has been removed. */
4016 if (global_settings.tagcache_autoupdate)
4017 {
4018 if (!file_exists(buf))
4019 {
4020 logf("Entry no longer valid.");
4021 logf("-> %s", buf);
4022 delete_entry(fe->idx_id);
4023 continue;
4024 }
4025 }
4026 }
4027
4028 continue ;
4029 }
4030
4031 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4032 if (bytesleft < 0)
4033 {
4034 logf("too big tagcache #2");
4035 logf("tl: %ld", fe->tag_length);
4036 logf("bl: %ld", bytesleft);
4037 close(fd);
4038 return false;
4039 }
4040
4041 p = fe->tag_data;
4042 rc = read(fd, fe->tag_data, fe->tag_length);
4043 p += rc;
4044
4045 if (rc != fe->tag_length)
4046 {
4047 logf("read error #13");
4048 logf("rc=0x%04x", rc); // 0x431
4049 logf("len=0x%04lx", fe->tag_length); // 0x4000
4050 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4051 logf("tag=0x%02x", tag); // 0x00
4052 close(fd);
4053 return false;
4054 }
4055 }
4056 close(fd);
4057 }
4058
4059 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4060 logf("tagcache loaded into ram!");
4061
4062 return true;
4063 }
4064 #endif /* HAVE_TC_RAMCACHE */
4065
4066 static bool check_deleted_files(void)
4067 {
4068 int fd;
4069 char buf[TAG_MAXLEN+32];
4070 struct tagfile_entry tfe;
4071
4072 logf("reverse scan...");
4073 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4074 fd = open(buf, O_RDONLY);
4075
4076 if (fd < 0)
4077 {
4078 logf("%s open fail", buf);
4079 return false;
4080 }
4081
4082 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4083 while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4084 #ifndef __PCTOOL__
4085 && !check_event_queue()
4086 #endif
4087 )
4088 {
4089 if (tfe.tag_length >= (long)sizeof(buf)-1)
4090 {
4091 logf("too long tag");
4092 close(fd);
4093 return false;
4094 }
4095
4096 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4097 {
4098 logf("read error #14");
4099 close(fd);
4100 return false;
4101 }
4102
4103 /* Check if the file has already deleted from the db. */
4104 if (*buf == '\0')
4105 continue;
4106
4107 /* Now check if the file exists. */
4108 if (!file_exists(buf))
4109 {
4110 logf("Entry no longer valid.");
4111 logf("-> %s / %ld", buf, tfe.tag_length);
4112 delete_entry(tfe.idx_id);
4113 }
4114 }
4115
4116 close(fd);
4117
4118 logf("done");
4119
4120 return true;
4121 }
4122
4123
4124 /* Note that this function must not be inlined, otherwise the whole point
4125 * of having the code in a separate function is lost.
4126 */
4127 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4128 int *ignore, int *unignore)
4129 {
4130 char newpath[MAX_PATH];
4131
4132 /* check for a database.ignore file */
4133 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4134 *ignore = file_exists(newpath);
4135 /* check for a database.unignore file */
4136 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4137 *unignore = file_exists(newpath);
4138 }
4139
4140
4141 static bool check_dir(const char *dirname, int add_files)
4142 {
4143 DIR *dir;
4144 int len;
4145 int success = false;
4146 int ignore, unignore;
4147
4148 dir = opendir(dirname);
4149 if (!dir)
4150 {
4151 logf("tagcache: opendir(%s) failed", dirname);
4152 return false;
4153 }
4154
4155 /* check for a database.ignore and database.unignore */
4156 check_ignore(dirname, &ignore, &unignore);
4157
4158 /* don't do anything if both ignore and unignore are there */
4159 if (ignore != unignore)
4160 add_files = unignore;
4161
4162 /* Recursively scan the dir. */
4163 #ifdef __PCTOOL__
4164 while (1)
4165 #else
4166 while (!check_event_queue())
4167 #endif
4168 {
4169 struct dirent *entry;
4170
4171 entry = readdir(dir);
4172
4173 if (entry == NULL)
4174 {
4175 success = true;
4176 break ;
4177 }
4178
4179 if (!strcmp((char *)entry->d_name, ".") ||
4180 !strcmp((char *)entry->d_name, ".."))
4181 continue;
4182
4183 yield();
4184
4185 len = strlen(curpath);
4186 snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4187 entry->d_name);
4188
4189 processed_dir_count++;
4190 if (entry->attribute & ATTR_DIRECTORY)
4191 check_dir(curpath, add_files);
4192 else if (add_files)
4193 {
4194 tc_stat.curentry = curpath;
4195
4196 /* Add a new entry to the temporary db file. */
4197 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4198 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4199 , dir->internal_entry
4200 #endif
4201 );
4202
4203 /* Wait until current path for debug screen is read and unset. */
4204 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4205 yield();
4206
4207 tc_stat.curentry = NULL;
4208 }
4209
4210 curpath[len] = '\0';
4211 }
4212
4213 closedir(dir);
4214
4215 return success;
4216 }
4217
4218 void tagcache_screensync_event(void)
4219 {
4220 tc_stat.curentry = NULL;
4221 }
4222
4223 void tagcache_screensync_enable(bool state)
4224 {
4225 tc_stat.syncscreen = state;
4226 }
4227
4228 void tagcache_build(const char *path)
4229 {
4230 struct tagcache_header header;
4231 bool ret;
4232
4233 curpath[0] = '\0';
4234 data_size = 0;
4235 total_entry_count = 0;
4236 processed_dir_count = 0;
4237
4238 #ifdef HAVE_DIRCACHE
4239 while (dircache_is_initializing())
4240 sleep(1);
4241 #endif
4242
4243 logf("updating tagcache");
4244
4245 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4246 if (cachefd >= 0)
4247 {
4248 logf("skipping, cache already waiting for commit");
4249 close(cachefd);
4250 return ;
4251 }
4252
4253 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4254 if (cachefd < 0)
4255 {
4256 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4257 return ;
4258 }
4259
4260 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4261
4262 cpu_boost(true);
4263
4264 logf("Scanning files...");
4265 /* Scan for new files. */
4266 memset(&header, 0, sizeof(struct tagcache_header));
4267 write(cachefd, &header, sizeof(struct tagcache_header));
4268
4269 if (strcmp("/", path) != 0)
4270 strcpy(curpath, path);
4271 ret = check_dir(path, true);
4272
4273 /* Write the header. */
4274 header.magic = TAGCACHE_MAGIC;
4275 header.datasize = data_size;
4276 header.entry_count = total_entry_count;
4277 lseek(cachefd, 0, SEEK_SET);
4278 write(cachefd, &header, sizeof(struct tagcache_header));
4279 close(cachefd);
4280
4281 if (filenametag_fd >= 0)
4282 {
4283 close(filenametag_fd);
4284 filenametag_fd = -1;
4285 }
4286
4287 if (!ret)
4288 {
4289 logf("Aborted.");
4290 cpu_boost(false);
4291 return ;
4292 }
4293
4294 /* Commit changes to the database. */
4295 #ifdef __PCTOOL__
4296 allocate_tempbuf();
4297 #endif
4298 if (commit())
4299 {
4300 remove(TAGCACHE_FILE_TEMP);
4301 logf("tagcache built!");
4302 }
4303 #ifdef __PCTOOL__
4304 free_tempbuf();
4305 #endif
4306
4307 #ifdef HAVE_TC_RAMCACHE
4308 if (hdr)
4309 {
4310 /* Import runtime statistics if we just initialized the db. */
4311 if (hdr->h.serial == 0)
4312 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4313 }
4314 #endif
4315
4316 cpu_boost(false);
4317 }
4318
4319 #ifdef HAVE_TC_RAMCACHE
4320 static void load_ramcache(void)
4321 {
4322 if (!hdr)
4323 return ;
4324
4325 cpu_boost(true);
4326
4327 /* At first we should load the cache (if exists). */
4328 tc_stat.ramcache = load_tagcache();
4329
4330 if (!tc_stat.ramcache)
4331 {
4332 /* If loading failed, it must indicate some problem with the db
4333 * so disable it entirely to prevent further issues. */
4334 tc_stat.ready = false;
4335 hdr = NULL;
4336 }
4337
4338 cpu_boost(false);
4339 }
4340
4341 void tagcache_unload_ramcache(void)
4342 {
4343 tc_stat.ramcache = false;
4344 /* Just to make sure there is no statefile present. */
4345 // remove(TAGCACHE_STATEFILE);
4346 }
4347 #endif
4348
4349 #ifndef __PCTOOL__
4350 static void tagcache_thread(void)
4351 {
4352 struct queue_event ev;
4353 bool check_done = false;
4354
4355 /* If the previous cache build/update was interrupted, commit
4356 * the changes first in foreground. */
4357 cpu_boost(true);
4358 allocate_tempbuf();
4359 commit();
4360 free_tempbuf();
4361
4362 #ifdef HAVE_TC_RAMCACHE
4363 # ifdef HAVE_EEPROM_SETTINGS
4364 if (firmware_settings.initialized && firmware_settings.disk_clean)
4365 check_done = tagcache_dumpload();
4366
4367 remove(TAGCACHE_STATEFILE);
4368 # endif
4369
4370 /* Allocate space for the tagcache if found on disk. */
4371 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4372 allocate_tagcache();
4373 #endif
4374
4375 cpu_boost(false);
4376 tc_stat.initialized = true;
4377
4378 /* Don't delay bootup with the header check but do it on background. */
4379 sleep(HZ);
4380 tc_stat.ready = check_all_headers();
4381 tc_stat.readyvalid = true;
4382
4383 while (1)
4384 {
4385 run_command_queue(false);
4386
4387 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4388
4389 switch (ev.id)
4390 {
4391 case Q_IMPORT_CHANGELOG:
4392 tagcache_import_changelog();
4393 break;
4394
4395 case Q_REBUILD:
4396 remove_files();
4397 remove(TAGCACHE_FILE_TEMP);
4398 tagcache_build("/");
4399 break;
4400
4401 case Q_UPDATE:
4402 tagcache_build("/");
4403 #ifdef HAVE_TC_RAMCACHE
4404 load_ramcache();
4405 #endif
4406 check_deleted_files();
4407 break ;
4408
4409 case Q_START_SCAN:
4410 check_done = false;
4411 case SYS_TIMEOUT:
4412 if (check_done || !tc_stat.ready)
4413 break ;
4414
4415 #ifdef HAVE_TC_RAMCACHE
4416 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4417 {
4418 load_ramcache();
4419 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4420 tagcache_build("/");
4421 }
4422 else
4423 #endif
4424 if (global_settings.tagcache_autoupdate)
4425 {
4426 tagcache_build("/");
4427
4428 /* This will be very slow unless dircache is enabled
4429 or target is flash based, but do it anyway for
4430 consistency. */
4431 check_deleted_files();
4432 }
4433
4434 logf("tagcache check done");
4435
4436 check_done = true;
4437 break ;
4438
4439 case Q_STOP_SCAN:
4440 break ;
4441
4442 case SYS_POWEROFF:
4443 break ;
4444
4445 #ifndef SIMULATOR
4446 case SYS_USB_CONNECTED:
4447 logf("USB: TagCache");
4448 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4449 usb_wait_for_disconnect(&tagcache_queue);
4450 break ;
4451 #endif
4452 }
4453 }
4454 }
4455
4456 bool tagcache_prepare_shutdown(void)
4457 {
4458 if (tagcache_get_commit_step() > 0)
4459 return false;
4460
4461 tagcache_stop_scan();
4462 while (read_lock || write_lock)
4463 sleep(1);
4464
4465 return true;
4466 }
4467
4468 void tagcache_shutdown(void)
4469 {
4470 /* Flush the command queue. */
4471 run_command_queue(true);
4472
4473 #ifdef HAVE_EEPROM_SETTINGS
4474 if (tc_stat.ramcache)
4475 tagcache_dumpsave();
4476 #endif
4477 }
4478
4479 static int get_progress(void)
4480 {
4481 int total_count = -1;
4482
4483 #ifdef HAVE_DIRCACHE
4484 if (dircache_is_enabled())
4485 {
4486 total_count = dircache_get_entry_count();
4487 }
4488 else
4489 #endif
4490 #ifdef HAVE_TC_RAMCACHE
4491 {
4492 if (hdr && tc_stat.ramcache)
4493 total_count = hdr->h.tch.entry_count;
4494 }
4495 #endif
4496
4497 if (total_count < 0)
4498 return -1;
4499
4500 return processed_dir_count * 100 / total_count;
4501 }
4502
4503 struct tagcache_stat* tagcache_get_stat(void)
4504 {
4505 tc_stat.progress = get_progress();
4506 tc_stat.processed_entries = processed_dir_count;
4507
4508 return &tc_stat;
4509 }
4510
4511 void tagcache_start_scan(void)
4512 {
4513 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4514 }
4515
4516 bool tagcache_update(void)
4517 {
4518 if (!tc_stat.ready)
4519 return false;
4520
4521 queue_post(&tagcache_queue, Q_UPDATE, 0);
4522 return false;
4523 }
4524
4525 bool tagcache_rebuild()
4526 {
4527 queue_post(&tagcache_queue, Q_REBUILD, 0);
4528 return false;
4529 }
4530
4531 void tagcache_stop_scan(void)
4532 {
4533 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4534 }
4535
4536 #ifdef HAVE_TC_RAMCACHE
4537 bool tagcache_is_ramcache(void)
4538 {
4539 return tc_stat.ramcache;
4540 }
4541 #endif
4542
4543 #endif /* !__PCTOOL__ */
4544
4545
4546 void tagcache_init(void)
4547 {
4548 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4549 memset(&current_tcmh, 0, sizeof(struct master_header));
4550 filenametag_fd = -1;
4551 write_lock = read_lock = 0;
4552
4553 #ifndef __PCTOOL__
4554 mutex_init(&command_queue_mutex);
4555 queue_init(&tagcache_queue, true);
4556 create_thread(tagcache_thread, tagcache_stack,
4557 sizeof(tagcache_stack), 0, tagcache_thread_name
4558 IF_PRIO(, PRIORITY_BACKGROUND)
4559 IF_COP(, CPU));
4560 #else
4561 tc_stat.initialized = true;
4562 allocate_tempbuf();
4563 commit();
4564 free_tempbuf();
4565 tc_stat.ready = check_all_headers();
4566 #endif
4567 }
4568
4569 #ifdef __PCTOOL__
4570 void tagcache_reverse_scan(void)
4571 {
4572 logf("Checking for deleted files");
4573 check_deleted_files();
4574 }
4575 #endif
4576
4577 bool tagcache_is_initialized(void)
4578 {
4579 return tc_stat.initialized;
4580 }
4581 bool tagcache_is_usable(void)
4582 {
4583 return tc_stat.initialized && tc_stat.ready;
4584 }
4585 int tagcache_get_commit_step(void)
4586 {
4587 return tc_stat.commit_step;
4588 }
4589 int tagcache_get_max_commit_step(void)
4590 {
4591 return (int)(SORTED_TAGS_COUNT)+1;
4592 }
4593
My public Rockbox git repo