?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Correct the metadata's VBR flag for MP4 files. ALAC is native VBR, AAC very unlikely...
[kugel-rb.git] / apps / tagcache.c
blobea7768139b7fec5970221ccbc5bc418e9b316650
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /*#define LOGF_ENABLE*/
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #ifdef APPLICATION
64 #include <unistd.h> /* readlink() */
65 #include <limits.h> /* PATH_MAX */
66 #endif
67 #include "config.h"
68 #include "ata_idle_notify.h"
69 #include "thread.h"
70 #include "kernel.h"
71 #include "system.h"
72 #include "logf.h"
73 #include "string-extra.h"
74 #include "usb.h"
75 #include "metadata.h"
76 #include "tagcache.h"
77 #include "buffer.h"
78 #include "crc32.h"
79 #include "misc.h"
80 #include "settings.h"
81 #include "dir.h"
82 #include "filefuncs.h"
83 #include "structec.h"
84 #include "debug.h"
85
86 #ifndef __PCTOOL__
87 #include "lang.h"
88 #include "eeprom_settings.h"
89 #endif
90
91 #ifdef __PCTOOL__
92 #define yield() do { } while(0)
93 #define sim_sleep(timeout) do { } while(0)
94 #define do_timed_yield() do { } while(0)
95 #endif
96
97 #ifndef __PCTOOL__
98 /* Tag Cache thread. */
99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
101 static const char tagcache_thread_name[] = "tagcache";
102 #endif
103
104 /* Previous path when scanning directory tree recursively. */
105 static char curpath[TAG_MAXLEN+32];
106
107 /* Used when removing duplicates. */
108 static char *tempbuf; /* Allocated when needed. */
109 static long tempbufidx; /* Current location in buffer. */
110 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
111 static long tempbuf_left; /* Buffer space left. */
112 static long tempbuf_pos;
113
114 #define SORTED_TAGS_COUNT 8
115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
118 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
119 /* Tags we want to get sorted (loaded to the tempbuf). */
120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
121 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
122 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
123
124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
126 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
127 (1LU << tag_albumartist) | (1LU << tag_grouping))
128
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime" };
134
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat;
137
138 /* Queue commands. */
139 enum tagcache_queue {
140 Q_STOP_SCAN = 0,
141 Q_START_SCAN,
142 Q_IMPORT_CHANGELOG,
143 Q_UPDATE,
144 Q_REBUILD,
145
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER,
148 CMD_UPDATE_NUMERIC,
149 };
150
151 struct tagcache_command_entry {
152 int32_t command;
153 int32_t idx_id;
154 int32_t tag;
155 int32_t data;
156 };
157
158 #ifndef __PCTOOL__
159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
160 static volatile int command_queue_widx = 0;
161 static volatile int command_queue_ridx = 0;
162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
163 #endif
164
165 /* Tag database structures. */
166
167 /* Variable-length tag entry in tag files. */
168 struct tagfile_entry {
169 int32_t tag_length; /* Length of the data in bytes including '\0' */
170 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
171 char tag_data[0]; /* Begin of the tag data */
172 };
173
174 /* Fixed-size tag entry in master db index. */
175 struct index_entry {
176 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
177 int32_t flag; /* Status flags */
178 };
179
180 /* Header is the same in every file. */
181 struct tagcache_header {
182 int32_t magic; /* Header version number */
183 int32_t datasize; /* Data size in bytes */
184 int32_t entry_count; /* Number of entries in this file */
185 };
186
187 struct master_header {
188 struct tagcache_header tch;
189 int32_t serial; /* Increasing counting number */
190 int32_t commitid; /* Number of commits so far */
191 int32_t dirty;
192 };
193
194 /* For the endianess correction */
195 static const char *tagfile_entry_ec = "ll";
196 /**
197 Note: This should be (1 + TAG_COUNT) amount of l's.
198 */
199 static const char *index_entry_ec = "llllllllllllllllllllll";
200
201 static const char *tagcache_header_ec = "lll";
202 static const char *master_header_ec = "llllll";
203
204 static struct master_header current_tcmh;
205
206 #ifdef HAVE_TC_RAMCACHE
207 /* Header is created when loading database to ram. */
208 struct ramcache_header {
209 struct master_header h; /* Header from the master index */
210 struct index_entry *indices; /* Master index file content */
211 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
212 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
213 };
214
215 # ifdef HAVE_EEPROM_SETTINGS
216 struct statefile_header {
217 struct ramcache_header *hdr;
218 struct tagcache_stat tc_stat;
219 };
220 # endif
221
222 /* Pointer to allocated ramcache_header */
223 static struct ramcache_header *hdr;
224 #endif
225
226 /**
227 * Full tag entries stored in a temporary file waiting
228 * for commit to the cache. */
229 struct temp_file_entry {
230 long tag_offset[TAG_COUNT];
231 short tag_length[TAG_COUNT];
232 long flag;
233
234 long data_length;
235 };
236
237 struct tempbuf_id_list {
238 long id;
239 struct tempbuf_id_list *next;
240 };
241
242 struct tempbuf_searchidx {
243 long idx_id;
244 char *str;
245 int seek;
246 struct tempbuf_id_list idlist;
247 };
248
249 /* Lookup buffer for fixing messed up index while after sorting. */
250 static long commit_entry_count;
251 static long lookup_buffer_depth;
252 static struct tempbuf_searchidx **lookup;
253
254 /* Used when building the temporary file. */
255 static int cachefd = -1, filenametag_fd;
256 static int total_entry_count = 0;
257 static int data_size = 0;
258 static int processed_dir_count;
259
260 /* Thread safe locking */
261 static volatile int write_lock;
262 static volatile int read_lock;
263
264 static bool delete_entry(long idx_id);
265
266 const char* tagcache_tag_to_str(int tag)
267 {
268 return tags_str[tag];
269 }
270
271 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
272 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
273 {
274 return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
275 }
276
277 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
278 {
279 return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
280 }
281
282 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
283 {
284 return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
285 }
286
287 #ifdef HAVE_DIRCACHE
288 /**
289 * Returns true if specified flag is still present, i.e., dircache
290 * has not been reloaded.
291 */
292 static bool is_dircache_intact(void)
293 {
294 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
295 }
296 #endif
297
298 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
299 {
300 int fd;
301 char buf[MAX_PATH];
302 int rc;
303
304 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
305 return -1;
306
307 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
308
309 fd = open(buf, write ? O_RDWR : O_RDONLY);
310 if (fd < 0)
311 {
312 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
313 tc_stat.ready = false;
314 return fd;
315 }
316
317 /* Check the header. */
318 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
319 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
320 {
321 logf("header error");
322 tc_stat.ready = false;
323 close(fd);
324 return -2;
325 }
326
327 return fd;
328 }
329
330 static int open_master_fd(struct master_header *hdr, bool write)
331 {
332 int fd;
333 int rc;
334
335 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
336 if (fd < 0)
337 {
338 logf("master file open failed for R/W");
339 tc_stat.ready = false;
340 return fd;
341 }
342
343 tc_stat.econ = false;
344
345 /* Check the header. */
346 rc = read(fd, hdr, sizeof(struct master_header));
347 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
348 {
349 /* Success. */
350 return fd;
351 }
352
353 /* Trying to read again, this time with endianess correction enabled. */
354 lseek(fd, 0, SEEK_SET);
355
356 rc = ecread(fd, hdr, 1, master_header_ec, true);
357 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
358 {
359 logf("header error");
360 tc_stat.ready = false;
361 close(fd);
362 return -2;
363 }
364
365 tc_stat.econ = true;
366
367 return fd;
368 }
369
370 #ifndef __PCTOOL__
371 static bool do_timed_yield(void)
372 {
373 /* Sorting can lock up for quite a while, so yield occasionally */
374 static long wakeup_tick = 0;
375 if (TIME_AFTER(current_tick, wakeup_tick))
376 {
377 wakeup_tick = current_tick + (HZ/4);
378 yield();
379 return true;
380 }
381 return false;
382 }
383 #endif
384
385 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
386 static long find_entry_ram(const char *filename,
387 const struct dircache_entry *dc)
388 {
389 static long last_pos = 0;
390 int i;
391
392 /* Check if tagcache is loaded into ram. */
393 if (!tc_stat.ramcache)
394 return -1;
395
396 if (dc == NULL)
397 dc = dircache_get_entry_ptr(filename);
398
399 if (dc == NULL)
400 {
401 logf("tagcache: file not found.");
402 return -1;
403 }
404
405 try_again:
406
407 if (last_pos > 0)
408 i = last_pos;
409 else
410 i = 0;
411
412 for (; i < hdr->h.tch.entry_count; i++)
413 {
414 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
415 {
416 last_pos = MAX(0, i - 3);
417 return i;
418 }
419
420 do_timed_yield();
421 }
422
423 if (last_pos > 0)
424 {
425 last_pos = 0;
426 goto try_again;
427 }
428
429 return -1;
430 }
431 #endif
432
433 static long find_entry_disk(const char *filename, bool localfd)
434 {
435 struct tagcache_header tch;
436 static long last_pos = -1;
437 long pos_history[POS_HISTORY_COUNT];
438 long pos_history_idx = 0;
439 bool found = false;
440 struct tagfile_entry tfe;
441 int fd;
442 char buf[TAG_MAXLEN+32];
443 int i;
444 int pos = -1;
445
446 if (!tc_stat.ready)
447 return -2;
448
449 fd = filenametag_fd;
450 if (fd < 0 || localfd)
451 {
452 last_pos = -1;
453 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
454 return -1;
455 }
456
457 check_again:
458
459 if (last_pos > 0)
460 lseek(fd, last_pos, SEEK_SET);
461 else
462 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
463
464 while (true)
465 {
466 pos = lseek(fd, 0, SEEK_CUR);
467 for (i = pos_history_idx-1; i >= 0; i--)
468 pos_history[i+1] = pos_history[i];
469 pos_history[0] = pos;
470
471 if (ecread_tagfile_entry(fd, &tfe)
472 != sizeof(struct tagfile_entry))
473 {
474 break ;
475 }
476
477 if (tfe.tag_length >= (long)sizeof(buf))
478 {
479 logf("too long tag #1");
480 close(fd);
481 if (!localfd)
482 filenametag_fd = -1;
483 last_pos = -1;
484 return -2;
485 }
486
487 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
488 {
489 logf("read error #2");
490 close(fd);
491 if (!localfd)
492 filenametag_fd = -1;
493 last_pos = -1;
494 return -3;
495 }
496
497 if (!strcasecmp(filename, buf))
498 {
499 last_pos = pos_history[pos_history_idx];
500 found = true;
501 break ;
502 }
503
504 if (pos_history_idx < POS_HISTORY_COUNT - 1)
505 pos_history_idx++;
506 }
507
508 /* Not found? */
509 if (!found)
510 {
511 if (last_pos > 0)
512 {
513 last_pos = -1;
514 logf("seek again");
515 goto check_again;
516 }
517
518 if (fd != filenametag_fd || localfd)
519 close(fd);
520 return -4;
521 }
522
523 if (fd != filenametag_fd || localfd)
524 close(fd);
525
526 return tfe.idx_id;
527 }
528
529 static int find_index(const char *filename)
530 {
531 long idx_id = -1;
532
533 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
534 if (tc_stat.ramcache && is_dircache_intact())
535 idx_id = find_entry_ram(filename, NULL);
536 #endif
537
538 if (idx_id < 0)
539 idx_id = find_entry_disk(filename, true);
540
541 return idx_id;
542 }
543
544 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
545 {
546 int idx_id;
547
548 if (!tc_stat.ready)
549 return false;
550
551 idx_id = find_index(filename);
552 if (idx_id < 0)
553 return false;
554
555 if (!tagcache_search(tcs, tag_filename))
556 return false;
557
558 tcs->entry_count = 0;
559 tcs->idx_id = idx_id;
560
561 return true;
562 }
563
564 static bool get_index(int masterfd, int idxid,
565 struct index_entry *idx, bool use_ram)
566 {
567 bool localfd = false;
568
569 if (idxid < 0)
570 {
571 logf("Incorrect idxid: %d", idxid);
572 return false;
573 }
574
575 #ifdef HAVE_TC_RAMCACHE
576 if (tc_stat.ramcache && use_ram)
577 {
578 if (hdr->indices[idxid].flag & FLAG_DELETED)
579 return false;
580
581 # ifdef HAVE_DIRCACHE
582 if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE)
583 || is_dircache_intact())
584 #endif
585 {
586 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
587 return true;
588 }
589 }
590 #else
591 (void)use_ram;
592 #endif
593
594 if (masterfd < 0)
595 {
596 struct master_header tcmh;
597
598 localfd = true;
599 masterfd = open_master_fd(&tcmh, false);
600 if (masterfd < 0)
601 return false;
602 }
603
604 lseek(masterfd, idxid * sizeof(struct index_entry)
605 + sizeof(struct master_header), SEEK_SET);
606 if (ecread_index_entry(masterfd, idx)
607 != sizeof(struct index_entry))
608 {
609 logf("read error #3");
610 if (localfd)
611 close(masterfd);
612
613 return false;
614 }
615
616 if (localfd)
617 close(masterfd);
618
619 if (idx->flag & FLAG_DELETED)
620 return false;
621
622 return true;
623 }
624
625 #ifndef __PCTOOL__
626
627 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
628 {
629 /* We need to exclude all memory only flags & tags when writing to disk. */
630 if (idx->flag & FLAG_DIRCACHE)
631 {
632 logf("memory only flags!");
633 return false;
634 }
635
636 #ifdef HAVE_TC_RAMCACHE
637 /* Only update numeric data. Writing the whole index to RAM by memcpy
638 * destroys dircache pointers!
639 */
640 if (tc_stat.ramcache)
641 {
642 int tag;
643 struct index_entry *idx_ram = &hdr->indices[idxid];
644
645 for (tag = 0; tag < TAG_COUNT; tag++)
646 {
647 if (TAGCACHE_IS_NUMERIC(tag))
648 {
649 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
650 }
651 }
652
653 /* Don't touch the dircache flag or attributes. */
654 idx_ram->flag = (idx->flag & 0x0000ffff)
655 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
656 }
657 #endif
658
659 lseek(masterfd, idxid * sizeof(struct index_entry)
660 + sizeof(struct master_header), SEEK_SET);
661 if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
662 {
663 logf("write error #3");
664 logf("idxid: %d", idxid);
665 return false;
666 }
667
668 return true;
669 }
670
671 #endif /* !__PCTOOL__ */
672
673 static bool open_files(struct tagcache_search *tcs, int tag)
674 {
675 if (tcs->idxfd[tag] < 0)
676 {
677 char fn[MAX_PATH];
678
679 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
680 tcs->idxfd[tag] = open(fn, O_RDONLY);
681 }
682
683 if (tcs->idxfd[tag] < 0)
684 {
685 logf("File not open!");
686 return false;
687 }
688
689 return true;
690 }
691
692 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
693 int tag, char *buf, long size)
694 {
695 struct tagfile_entry tfe;
696 long seek;
697
698 *buf = '\0';
699
700 if (TAGCACHE_IS_NUMERIC(tag))
701 return false;
702
703 seek = idx->tag_seek[tag];
704 if (seek < 0)
705 {
706 logf("Retrieve failed");
707 return false;
708 }
709
710 #ifdef HAVE_TC_RAMCACHE
711 if (tcs->ramsearch)
712 {
713 struct tagfile_entry *ep;
714
715 # ifdef HAVE_DIRCACHE
716 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
717 && is_dircache_intact())
718 {
719 dircache_copy_path((struct dircache_entry *)seek,
720 buf, size);
721 return true;
722 }
723 else
724 # endif
725 if (tag != tag_filename)
726 {
727 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
728 strlcpy(buf, ep->tag_data, size);
729
730 return true;
731 }
732 }
733 #endif
734
735 if (!open_files(tcs, tag))
736 return false;
737
738 lseek(tcs->idxfd[tag], seek, SEEK_SET);
739 if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
740 != sizeof(struct tagfile_entry))
741 {
742 logf("read error #5");
743 return false;
744 }
745
746 if (tfe.tag_length >= size)
747 {
748 logf("too small buffer");
749 return false;
750 }
751
752 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
753 tfe.tag_length)
754 {
755 logf("read error #6");
756 return false;
757 }
758
759 buf[tfe.tag_length] = '\0';
760
761 return true;
762 }
763
764 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
765
766 static long read_numeric_tag(int tag, int idx_id, const struct index_entry *idx)
767 {
768 #ifndef __PCTOOL__
769 if (! COMMAND_QUEUE_IS_EMPTY)
770 {
771 /* Attempt to find tag data through store-to-load forwarding in
772 command queue */
773 long result = -1;
774
775 mutex_lock(&command_queue_mutex);
776
777 int ridx = command_queue_widx;
778
779 while (ridx != command_queue_ridx)
780 {
781 if (--ridx < 0)
782 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
783
784 if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
785 && command_queue[ridx].idx_id == idx_id
786 && command_queue[ridx].tag == tag)
787 {
788 result = command_queue[ridx].data;
789 break;
790 }
791 }
792
793 mutex_unlock(&command_queue_mutex);
794
795 if (result >= 0)
796 {
797 logf("read_numeric_tag: "
798 "Recovered tag %d value %lX from write queue",
799 tag, result);
800 return result;
801 }
802 }
803 #endif
804
805 return idx->tag_seek[tag];
806 }
807
808
809 static long check_virtual_tags(int tag, int idx_id,
810 const struct index_entry *idx)
811 {
812 long data = 0;
813
814 switch (tag)
815 {
816 case tag_virt_length_sec:
817 data = (read_numeric_tag(tag_length, idx_id, idx)/1000) % 60;
818 break;
819
820 case tag_virt_length_min:
821 data = (read_numeric_tag(tag_length, idx_id, idx)/1000) / 60;
822 break;
823
824 case tag_virt_playtime_sec:
825 data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) % 60;
826 break;
827
828 case tag_virt_playtime_min:
829 data = (read_numeric_tag(tag_playtime, idx_id, idx)/1000) / 60;
830 break;
831
832 case tag_virt_autoscore:
833 if (read_numeric_tag(tag_length, idx_id, idx) == 0
834 || read_numeric_tag(tag_playcount, idx_id, idx) == 0)
835 {
836 data = 0;
837 }
838 else
839 {
840 /* A straight calculus gives:
841 autoscore = 100 * playtime / length / playcout (1)
842 Now, consider the euclidian division of playtime by length:
843 playtime = alpha * length + beta
844 With:
845 0 <= beta < length
846 Now, (1) becomes:
847 autoscore = 100 * (alpha / playcout + beta / length / playcount)
848 Both terms should be small enough to avoid any overflow
849 */
850 data = 100 * (read_numeric_tag(tag_playtime, idx_id, idx)
851 / read_numeric_tag(tag_length, idx_id, idx))
852 + (100 * (read_numeric_tag(tag_playtime, idx_id, idx)
853 % read_numeric_tag(tag_length, idx_id, idx)))
854 / read_numeric_tag(tag_length, idx_id, idx);
855 data /= read_numeric_tag(tag_playcount, idx_id, idx);
856 }
857 break;
858
859 /* How many commits before the file has been added to the DB. */
860 case tag_virt_entryage:
861 data = current_tcmh.commitid
862 - read_numeric_tag(tag_commitid, idx_id, idx) - 1;
863 break;
864
865 default:
866 data = read_numeric_tag(tag, idx_id, idx);
867 }
868
869 return data;
870 }
871
872 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
873 {
874 struct index_entry idx;
875
876 if (!tc_stat.ready)
877 return false;
878
879 if (!TAGCACHE_IS_NUMERIC(tag))
880 return -1;
881
882 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
883 return -2;
884
885 return check_virtual_tags(tag, tcs->idx_id, &idx);
886 }
887
888 inline static bool str_ends_with(const char *str1, const char *str2)
889 {
890 int str_len = strlen(str1);
891 int clause_len = strlen(str2);
892
893 if (clause_len > str_len)
894 return false;
895
896 return !strcasecmp(&str1[str_len - clause_len], str2);
897 }
898
899 inline static bool str_oneof(const char *str, const char *list)
900 {
901 const char *sep;
902 int l, len = strlen(str);
903
904 while (*list)
905 {
906 sep = strchr(list, '|');
907 l = sep ? (long)sep - (long)list : (int)strlen(list);
908 if ((l==len) && !strncasecmp(str, list, len))
909 return true;
910 list += sep ? l + 1 : l;
911 }
912
913 return false;
914 }
915
916 static bool check_against_clause(long numeric, const char *str,
917 const struct tagcache_search_clause *clause)
918 {
919 if (clause->numeric)
920 {
921 switch (clause->type)
922 {
923 case clause_is:
924 return numeric == clause->numeric_data;
925 case clause_is_not:
926 return numeric != clause->numeric_data;
927 case clause_gt:
928 return numeric > clause->numeric_data;
929 case clause_gteq:
930 return numeric >= clause->numeric_data;
931 case clause_lt:
932 return numeric < clause->numeric_data;
933 case clause_lteq:
934 return numeric <= clause->numeric_data;
935 default:
936 logf("Incorrect numeric tag: %d", clause->type);
937 }
938 }
939 else
940 {
941 switch (clause->type)
942 {
943 case clause_is:
944 return !strcasecmp(clause->str, str);
945 case clause_is_not:
946 return strcasecmp(clause->str, str);
947 case clause_gt:
948 return 0>strcasecmp(clause->str, str);
949 case clause_gteq:
950 return 0>=strcasecmp(clause->str, str);
951 case clause_lt:
952 return 0<strcasecmp(clause->str, str);
953 case clause_lteq:
954 return 0<=strcasecmp(clause->str, str);
955 case clause_contains:
956 return (strcasestr(str, clause->str) != NULL);
957 case clause_not_contains:
958 return (strcasestr(str, clause->str) == NULL);
959 case clause_begins_with:
960 return (strcasestr(str, clause->str) == str);
961 case clause_not_begins_with:
962 return (strcasestr(str, clause->str) != str);
963 case clause_ends_with:
964 return str_ends_with(str, clause->str);
965 case clause_not_ends_with:
966 return !str_ends_with(str, clause->str);
967 case clause_oneof:
968 return str_oneof(str, clause->str);
969
970 default:
971 logf("Incorrect tag: %d", clause->type);
972 }
973 }
974
975 return false;
976 }
977
978 static bool check_clauses(struct tagcache_search *tcs,
979 struct index_entry *idx,
980 struct tagcache_search_clause **clause, int count)
981 {
982 int i;
983
984 #ifdef HAVE_TC_RAMCACHE
985 if (tcs->ramsearch)
986 {
987 /* Go through all conditional clauses. */
988 for (i = 0; i < count; i++)
989 {
990 struct tagfile_entry *tfe;
991 int seek;
992 char buf[256];
993 char *str = NULL;
994
995 seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
996
997 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
998 {
999 if (clause[i]->tag == tag_filename)
1000 {
1001 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1002 str = buf;
1003 }
1004 else
1005 {
1006 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
1007 str = tfe->tag_data;
1008 }
1009 }
1010
1011 if (!check_against_clause(seek, str, clause[i]))
1012 return false;
1013 }
1014 }
1015 else
1016 #endif
1017 {
1018 /* Check for conditions. */
1019 for (i = 0; i < count; i++)
1020 {
1021 struct tagfile_entry tfe;
1022 int seek;
1023 char str[256];
1024
1025 seek = check_virtual_tags(clause[i]->tag, tcs->idx_id, idx);
1026
1027 memset(str, 0, sizeof str);
1028 if (!TAGCACHE_IS_NUMERIC(clause[i]->tag))
1029 {
1030 int fd = tcs->idxfd[clause[i]->tag];
1031 lseek(fd, seek, SEEK_SET);
1032 ecread_tagfile_entry(fd, &tfe);
1033 if (tfe.tag_length >= (int)sizeof(str))
1034 {
1035 logf("Too long tag read!");
1036 break ;
1037 }
1038
1039 read(fd, str, tfe.tag_length);
1040
1041 /* Check if entry has been deleted. */
1042 if (str[0] == '\0')
1043 break;
1044 }
1045
1046 if (!check_against_clause(seek, str, clause[i]))
1047 return false;
1048 }
1049 }
1050
1051 return true;
1052 }
1053
1054 bool tagcache_check_clauses(struct tagcache_search *tcs,
1055 struct tagcache_search_clause **clause, int count)
1056 {
1057 struct index_entry idx;
1058
1059 if (count == 0)
1060 return true;
1061
1062 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1063 return false;
1064
1065 return check_clauses(tcs, &idx, clause, count);
1066 }
1067
1068 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1069 {
1070 int i;
1071
1072 /* If uniq buffer is not defined we must return true for search to work. */
1073 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1074 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1075 {
1076 return true;
1077 }
1078
1079 for (i = 0; i < tcs->unique_list_count; i++)
1080 {
1081 /* Return false if entry is found. */
1082 if (tcs->unique_list[i] == id)
1083 return false;
1084 }
1085
1086 if (tcs->unique_list_count < tcs->unique_list_capacity)
1087 {
1088 tcs->unique_list[i] = id;
1089 tcs->unique_list_count++;
1090 }
1091
1092 return true;
1093 }
1094
1095 static bool build_lookup_list(struct tagcache_search *tcs)
1096 {
1097 struct index_entry entry;
1098 int i, j;
1099
1100 tcs->seek_list_count = 0;
1101
1102 #ifdef HAVE_TC_RAMCACHE
1103 if (tcs->ramsearch
1104 # ifdef HAVE_DIRCACHE
1105 && (tcs->type != tag_filename || is_dircache_intact())
1106 # endif
1107 )
1108 {
1109 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1110 {
1111 struct tagcache_seeklist_entry *seeklist;
1112 struct index_entry *idx = &hdr->indices[i];
1113 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1114 break ;
1115
1116 /* Skip deleted files. */
1117 if (idx->flag & FLAG_DELETED)
1118 continue;
1119
1120 /* Go through all filters.. */
1121 for (j = 0; j < tcs->filter_count; j++)
1122 {
1123 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1124 {
1125 break ;
1126 }
1127 }
1128
1129 if (j < tcs->filter_count)
1130 continue ;
1131
1132 /* Check for conditions. */
1133 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1134 continue;
1135
1136 /* Add to the seek list if not already in uniq buffer. */
1137 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1138 continue;
1139
1140 /* Lets add it. */
1141 seeklist = &tcs->seeklist[tcs->seek_list_count];
1142 seeklist->seek = idx->tag_seek[tcs->type];
1143 seeklist->flag = idx->flag;
1144 seeklist->idx_id = i;
1145 tcs->seek_list_count++;
1146 }
1147
1148 tcs->seek_pos = i;
1149
1150 return tcs->seek_list_count > 0;
1151 }
1152 #endif
1153
1154 if (tcs->masterfd < 0)
1155 {
1156 struct master_header tcmh;
1157 tcs->masterfd = open_master_fd(&tcmh, false);
1158 }
1159
1160 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1161 sizeof(struct master_header), SEEK_SET);
1162
1163 while (ecread_index_entry(tcs->masterfd, &entry)
1164 == sizeof(struct index_entry))
1165 {
1166 struct tagcache_seeklist_entry *seeklist;
1167
1168 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1169 break ;
1170
1171 i = tcs->seek_pos;
1172 tcs->seek_pos++;
1173
1174 /* Check if entry has been deleted. */
1175 if (entry.flag & FLAG_DELETED)
1176 continue;
1177
1178 /* Go through all filters.. */
1179 for (j = 0; j < tcs->filter_count; j++)
1180 {
1181 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1182 break ;
1183 }
1184
1185 if (j < tcs->filter_count)
1186 continue ;
1187
1188 /* Check for conditions. */
1189 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1190 continue;
1191
1192 /* Add to the seek list if not already in uniq buffer. */
1193 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1194 continue;
1195
1196 /* Lets add it. */
1197 seeklist = &tcs->seeklist[tcs->seek_list_count];
1198 seeklist->seek = entry.tag_seek[tcs->type];
1199 seeklist->flag = entry.flag;
1200 seeklist->idx_id = i;
1201 tcs->seek_list_count++;
1202
1203 yield();
1204 }
1205
1206 return tcs->seek_list_count > 0;
1207 }
1208
1209
1210 static void remove_files(void)
1211 {
1212 int i;
1213 char buf[MAX_PATH];
1214
1215 tc_stat.ready = false;
1216 tc_stat.ramcache = false;
1217 tc_stat.econ = false;
1218 remove(TAGCACHE_FILE_MASTER);
1219 for (i = 0; i < TAG_COUNT; i++)
1220 {
1221 if (TAGCACHE_IS_NUMERIC(i))
1222 continue;
1223
1224 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1225 remove(buf);
1226 }
1227 }
1228
1229
1230 static bool check_all_headers(void)
1231 {
1232 struct master_header myhdr;
1233 struct tagcache_header tch;
1234 int tag;
1235 int fd;
1236
1237 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1238 return false;
1239
1240 close(fd);
1241 if (myhdr.dirty)
1242 {
1243 logf("tagcache is dirty!");
1244 return false;
1245 }
1246
1247 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1248
1249 for (tag = 0; tag < TAG_COUNT; tag++)
1250 {
1251 if (TAGCACHE_IS_NUMERIC(tag))
1252 continue;
1253
1254 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1255 return false;
1256
1257 close(fd);
1258 }
1259
1260 return true;
1261 }
1262
1263 bool tagcache_is_busy(void)
1264 {
1265 return read_lock || write_lock;
1266 }
1267
1268 bool tagcache_search(struct tagcache_search *tcs, int tag)
1269 {
1270 struct tagcache_header tag_hdr;
1271 struct master_header master_hdr;
1272 int i;
1273
1274 while (read_lock)
1275 sleep(1);
1276
1277 memset(tcs, 0, sizeof(struct tagcache_search));
1278 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1279 return false;
1280
1281 tcs->position = sizeof(struct tagcache_header);
1282 tcs->type = tag;
1283 tcs->seek_pos = 0;
1284 tcs->list_position = 0;
1285 tcs->seek_list_count = 0;
1286 tcs->filter_count = 0;
1287 tcs->masterfd = -1;
1288
1289 for (i = 0; i < TAG_COUNT; i++)
1290 tcs->idxfd[i] = -1;
1291
1292 #ifndef HAVE_TC_RAMCACHE
1293 tcs->ramsearch = false;
1294 #else
1295 tcs->ramsearch = tc_stat.ramcache;
1296 if (tcs->ramsearch)
1297 {
1298 tcs->entry_count = hdr->entry_count[tcs->type];
1299 }
1300 else
1301 #endif
1302 {
1303 /* Always open as R/W so we can pass tcs to functions that modify data also
1304 * without failing. */
1305 tcs->masterfd = open_master_fd(&master_hdr, true);
1306 if (tcs->masterfd < 0)
1307 return false;
1308
1309 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1310 {
1311 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1312 if (tcs->idxfd[tcs->type] < 0)
1313 return false;
1314
1315 tcs->entry_count = tag_hdr.entry_count;
1316 }
1317 else
1318 {
1319 tcs->entry_count = master_hdr.tch.entry_count;
1320 }
1321 }
1322
1323 tcs->valid = true;
1324 tcs->initialized = true;
1325 write_lock++;
1326
1327 return true;
1328 }
1329
1330 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1331 void *buffer, long length)
1332 {
1333 tcs->unique_list = (unsigned long *)buffer;
1334 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1335 tcs->unique_list_count = 0;
1336 }
1337
1338 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1339 int tag, int seek)
1340 {
1341 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1342 return false;
1343
1344 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1345 return false;
1346
1347 tcs->filter_tag[tcs->filter_count] = tag;
1348 tcs->filter_seek[tcs->filter_count] = seek;
1349 tcs->filter_count++;
1350
1351 return true;
1352 }
1353
1354 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1355 struct tagcache_search_clause *clause)
1356 {
1357 int i;
1358
1359 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1360 {
1361 logf("Too many clauses");
1362 return false;
1363 }
1364
1365 /* Check if there is already a similar filter in present (filters are
1366 * much faster than clauses).
1367 */
1368 for (i = 0; i < tcs->filter_count; i++)
1369 {
1370 if (tcs->filter_tag[i] == clause->tag)
1371 return true;
1372 }
1373
1374 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1375 {
1376 char buf[MAX_PATH];
1377
1378 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1379 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1380 }
1381
1382 tcs->clause[tcs->clause_count] = clause;
1383 tcs->clause_count++;
1384
1385 return true;
1386 }
1387
1388 static bool get_next(struct tagcache_search *tcs)
1389 {
1390 static char buf[TAG_MAXLEN+32];
1391 struct tagfile_entry entry;
1392 long flag = 0;
1393
1394 if (!tcs->valid || !tc_stat.ready)
1395 return false;
1396
1397 if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1398 #ifdef HAVE_TC_RAMCACHE
1399 && !tcs->ramsearch
1400 #endif
1401 )
1402 return false;
1403
1404 /* Relative fetch. */
1405 if (tcs->filter_count > 0 || tcs->clause_count > 0
1406 || TAGCACHE_IS_NUMERIC(tcs->type)
1407 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1408 /* We need to retrieve flag status for dircache. */
1409 || (tcs->ramsearch && tcs->type == tag_filename)
1410 #endif
1411 )
1412 {
1413 struct tagcache_seeklist_entry *seeklist;
1414
1415 /* Check for end of list. */
1416 if (tcs->list_position == tcs->seek_list_count)
1417 {
1418 tcs->list_position = 0;
1419
1420 /* Try to fetch more. */
1421 if (!build_lookup_list(tcs))
1422 {
1423 tcs->valid = false;
1424 return false;
1425 }
1426 }
1427
1428 seeklist = &tcs->seeklist[tcs->list_position];
1429 flag = seeklist->flag;
1430 tcs->position = seeklist->seek;
1431 tcs->idx_id = seeklist->idx_id;
1432 tcs->list_position++;
1433 }
1434 else
1435 {
1436 if (tcs->entry_count == 0)
1437 {
1438 tcs->valid = false;
1439 return false;
1440 }
1441
1442 tcs->entry_count--;
1443 }
1444
1445 tcs->result_seek = tcs->position;
1446
1447 if (TAGCACHE_IS_NUMERIC(tcs->type))
1448 {
1449 snprintf(buf, sizeof(buf), "%ld", tcs->position);
1450 tcs->result = buf;
1451 tcs->result_len = strlen(buf) + 1;
1452 return true;
1453 }
1454
1455 /* Direct fetch. */
1456 #ifdef HAVE_TC_RAMCACHE
1457 if (tcs->ramsearch)
1458 {
1459
1460 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1461 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1462 && is_dircache_intact())
1463 {
1464 dircache_copy_path((struct dircache_entry *)tcs->position,
1465 buf, sizeof buf);
1466 tcs->result = buf;
1467 tcs->result_len = strlen(buf) + 1;
1468 tcs->ramresult = false;
1469
1470 return true;
1471 }
1472 else
1473 #endif
1474 if (tcs->type != tag_filename)
1475 {
1476 struct tagfile_entry *ep;
1477
1478 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1479 tcs->result = ep->tag_data;
1480 tcs->result_len = strlen(tcs->result) + 1;
1481 tcs->idx_id = ep->idx_id;
1482 tcs->ramresult = true;
1483
1484 /* Increase position for the next run. This may get overwritten. */
1485 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1486
1487 return true;
1488 }
1489 }
1490 #endif
1491
1492 if (!open_files(tcs, tcs->type))
1493 {
1494 tcs->valid = false;
1495 return false;
1496 }
1497
1498 /* Seek stream to the correct position and continue to direct fetch. */
1499 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1500
1501 if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1502 {
1503 logf("read error #5");
1504 tcs->valid = false;
1505 return false;
1506 }
1507
1508 if (entry.tag_length > (long)sizeof(buf))
1509 {
1510 tcs->valid = false;
1511 logf("too long tag #2");
1512 logf("P:%lX/%lX", tcs->position, entry.tag_length);
1513 return false;
1514 }
1515
1516 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1517 {
1518 tcs->valid = false;
1519 logf("read error #4");
1520 return false;
1521 }
1522
1523 /**
1524 Update the position for the next read (this may be overridden
1525 if filters or clauses are being used).
1526 */
1527 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1528 tcs->result = buf;
1529 tcs->result_len = strlen(tcs->result) + 1;
1530 tcs->idx_id = entry.idx_id;
1531 tcs->ramresult = false;
1532
1533 return true;
1534 }
1535
1536 bool tagcache_get_next(struct tagcache_search *tcs)
1537 {
1538 while (get_next(tcs))
1539 {
1540 if (tcs->result_len > 1)
1541 return true;
1542 }
1543
1544 return false;
1545 }
1546
1547 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1548 int tag, char *buf, long size)
1549 {
1550 struct index_entry idx;
1551
1552 *buf = '\0';
1553 if (!get_index(tcs->masterfd, idxid, &idx, true))
1554 return false;
1555
1556 return retrieve(tcs, &idx, tag, buf, size);
1557 }
1558
1559 static bool update_master_header(void)
1560 {
1561 struct master_header myhdr;
1562 int fd;
1563
1564 if (!tc_stat.ready)
1565 return false;
1566
1567 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1568 return false;
1569
1570 myhdr.serial = current_tcmh.serial;
1571 myhdr.commitid = current_tcmh.commitid;
1572 myhdr.dirty = current_tcmh.dirty;
1573
1574 /* Write it back */
1575 lseek(fd, 0, SEEK_SET);
1576 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1577 close(fd);
1578
1579 #ifdef HAVE_TC_RAMCACHE
1580 if (hdr)
1581 {
1582 hdr->h.serial = current_tcmh.serial;
1583 hdr->h.commitid = current_tcmh.commitid;
1584 hdr->h.dirty = current_tcmh.dirty;
1585 }
1586 #endif
1587
1588 return true;
1589 }
1590
1591 #if 0
1592
1593 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1594 {
1595 struct tagentry *entry;
1596
1597 if (tcs->type != tag_title)
1598 return ;
1599
1600 /* We will need reserve buffer for this. */
1601 if (tcs->ramcache)
1602 {
1603 struct tagfile_entry *ep;
1604
1605 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1606 tcs->seek_list[tcs->seek_list_count];
1607 }
1608
1609 entry = find_entry_ram();
1610
1611 }
1612 #endif
1613
1614 void tagcache_search_finish(struct tagcache_search *tcs)
1615 {
1616 int i;
1617
1618 if (!tcs->initialized)
1619 return;
1620
1621 if (tcs->masterfd >= 0)
1622 {
1623 close(tcs->masterfd);
1624 tcs->masterfd = -1;
1625 }
1626
1627 for (i = 0; i < TAG_COUNT; i++)
1628 {
1629 if (tcs->idxfd[i] >= 0)
1630 {
1631 close(tcs->idxfd[i]);
1632 tcs->idxfd[i] = -1;
1633 }
1634 }
1635
1636 tcs->ramsearch = false;
1637 tcs->valid = false;
1638 tcs->initialized = 0;
1639 if (write_lock > 0)
1640 write_lock--;
1641 }
1642
1643 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1644 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1645 {
1646 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1647 }
1648
1649 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1650 {
1651 return check_virtual_tags(tag, idx_id, entry);
1652 }
1653
1654 static char* get_tag_string(const struct index_entry *entry, int tag)
1655 {
1656 char* s = get_tag(entry, tag)->tag_data;
1657 return strcmp(s, UNTAGGED) ? s : NULL;
1658 }
1659
1660 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1661 {
1662 struct index_entry *entry;
1663 int idx_id;
1664
1665 if (!tc_stat.ready || !tc_stat.ramcache)
1666 return false;
1667
1668 /* Find the corresponding entry in tagcache. */
1669 idx_id = find_entry_ram(filename, NULL);
1670 if (idx_id < 0)
1671 return false;
1672
1673 entry = &hdr->indices[idx_id];
1674
1675 memset(id3, 0, sizeof(struct mp3entry));
1676
1677 id3->title = get_tag_string(entry, tag_title);
1678 id3->artist = get_tag_string(entry, tag_artist);
1679 id3->album = get_tag_string(entry, tag_album);
1680 id3->genre_string = get_tag_string(entry, tag_genre);
1681 id3->composer = get_tag_string(entry, tag_composer);
1682 id3->comment = get_tag_string(entry, tag_comment);
1683 id3->albumartist = get_tag_string(entry, tag_albumartist);
1684 id3->grouping = get_tag_string(entry, tag_grouping);
1685
1686 id3->length = get_tag_numeric(entry, tag_length, idx_id);
1687 id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id);
1688 id3->rating = get_tag_numeric(entry, tag_rating, idx_id);
1689 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1690 id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1691 id3->year = get_tag_numeric(entry, tag_year, idx_id);
1692
1693 id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1694 id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1695 id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1696 if (id3->bitrate == 0)
1697 id3->bitrate = 1;
1698
1699 #if CONFIG_CODEC == SWCODEC
1700 if (global_settings.autoresume_enable)
1701 {
1702 id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1703 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1704 id3->title, id3->offset);
1705 }
1706 #endif
1707
1708 return true;
1709 }
1710 #endif
1711
1712 static inline void write_item(const char *item)
1713 {
1714 int len = strlen(item) + 1;
1715
1716 data_size += len;
1717 write(cachefd, item, len);
1718 }
1719
1720 static int check_if_empty(char **tag)
1721 {
1722 int length;
1723
1724 if (*tag == NULL || **tag == '\0')
1725 {
1726 *tag = UNTAGGED;
1727 return sizeof(UNTAGGED); /* Tag length */
1728 }
1729
1730 length = strlen(*tag);
1731 if (length > TAG_MAXLEN)
1732 {
1733 logf("over length tag: %s", *tag);
1734 length = TAG_MAXLEN;
1735 (*tag)[length] = '\0';
1736 }
1737
1738 return length + 1;
1739 }
1740
1741 #define ADD_TAG(entry,tag,data) \
1742 /* Adding tag */ \
1743 entry.tag_offset[tag] = offset; \
1744 entry.tag_length[tag] = check_if_empty(data); \
1745 offset += entry.tag_length[tag]
1746 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1747 * idea, as it uses lots of stack and is called from a recursive function
1748 * (check_dir).
1749 */
1750 static void __attribute__ ((noinline)) add_tagcache(char *path,
1751 unsigned long mtime
1752 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1753 ,const struct dircache_entry *dc
1754 #endif
1755 )
1756 {
1757 struct mp3entry id3;
1758 struct temp_file_entry entry;
1759 bool ret;
1760 int fd;
1761 int idx_id = -1;
1762 char tracknumfix[3];
1763 int offset = 0;
1764 int path_length = strlen(path);
1765 bool has_albumartist;
1766 bool has_grouping;
1767
1768 #ifdef SIMULATOR
1769 /* Crude logging for the sim - to aid in debugging */
1770 int logfd = open(ROCKBOX_DIR "/database.log",
1771 O_WRONLY | O_APPEND | O_CREAT, 0666);
1772 if (logfd >= 0) {
1773 write(logfd, path, strlen(path));
1774 write(logfd, "\n", 1);
1775 close(logfd);
1776 }
1777 #endif
1778
1779 if (cachefd < 0)
1780 return ;
1781
1782 /* Check for overlength file path. */
1783 if (path_length > TAG_MAXLEN)
1784 {
1785 /* Path can't be shortened. */
1786 logf("Too long path: %s", path);
1787 return ;
1788 }
1789
1790 /* Check if the file is supported. */
1791 if (probe_file_format(path) == AFMT_UNKNOWN)
1792 return ;
1793
1794 /* Check if the file is already cached. */
1795 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1796 if (tc_stat.ramcache && is_dircache_intact())
1797 {
1798 idx_id = find_entry_ram(path, dc);
1799 }
1800 #endif
1801
1802 /* Be sure the entry doesn't exist. */
1803 if (filenametag_fd >= 0 && idx_id < 0)
1804 idx_id = find_entry_disk(path, false);
1805
1806 /* Check if file has been modified. */
1807 if (idx_id >= 0)
1808 {
1809 struct index_entry idx;
1810
1811 /* TODO: Mark that the index exists (for fast reverse scan) */
1812 //found_idx[idx_id/8] |= idx_id%8;
1813
1814 if (!get_index(-1, idx_id, &idx, true))
1815 {
1816 logf("failed to retrieve index entry");
1817 return ;
1818 }
1819
1820 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1821 {
1822 /* No changes to file. */
1823 return ;
1824 }
1825
1826 /* Metadata might have been changed. Delete the entry. */
1827 logf("Re-adding: %s", path);
1828 if (!delete_entry(idx_id))
1829 {
1830 logf("delete_entry failed: %d", idx_id);
1831 return ;
1832 }
1833 }
1834
1835 fd = open(path, O_RDONLY);
1836 if (fd < 0)
1837 {
1838 logf("open fail: %s", path);
1839 return ;
1840 }
1841
1842 memset(&id3, 0, sizeof(struct mp3entry));
1843 memset(&entry, 0, sizeof(struct temp_file_entry));
1844 memset(&tracknumfix, 0, sizeof(tracknumfix));
1845 ret = get_metadata(&id3, fd, path);
1846 close(fd);
1847
1848 if (!ret)
1849 return ;
1850
1851 logf("-> %s", path);
1852
1853 /* Generate track number if missing. */
1854 if (id3.tracknum <= 0)
1855 {
1856 const char *p = strrchr(path, '.');
1857
1858 if (p == NULL)
1859 p = &path[strlen(path)-1];
1860
1861 while (*p != '/')
1862 {
1863 if (isdigit(*p) && isdigit(*(p-1)))
1864 {
1865 tracknumfix[1] = *p--;
1866 tracknumfix[0] = *p;
1867 break;
1868 }
1869 p--;
1870 }
1871
1872 if (tracknumfix[0] != '\0')
1873 {
1874 id3.tracknum = atoi(tracknumfix);
1875 /* Set a flag to indicate track number has been generated. */
1876 entry.flag |= FLAG_TRKNUMGEN;
1877 }
1878 else
1879 {
1880 /* Unable to generate track number. */
1881 id3.tracknum = -1;
1882 }
1883 }
1884
1885 /* Numeric tags */
1886 entry.tag_offset[tag_year] = id3.year;
1887 entry.tag_offset[tag_discnumber] = id3.discnum;
1888 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1889 entry.tag_offset[tag_length] = id3.length;
1890 entry.tag_offset[tag_bitrate] = id3.bitrate;
1891 entry.tag_offset[tag_mtime] = mtime;
1892
1893 /* String tags. */
1894 has_albumartist = id3.albumartist != NULL
1895 && strlen(id3.albumartist) > 0;
1896 has_grouping = id3.grouping != NULL
1897 && strlen(id3.grouping) > 0;
1898
1899 ADD_TAG(entry, tag_filename, &path);
1900 ADD_TAG(entry, tag_title, &id3.title);
1901 ADD_TAG(entry, tag_artist, &id3.artist);
1902 ADD_TAG(entry, tag_album, &id3.album);
1903 ADD_TAG(entry, tag_genre, &id3.genre_string);
1904 ADD_TAG(entry, tag_composer, &id3.composer);
1905 ADD_TAG(entry, tag_comment, &id3.comment);
1906 if (has_albumartist)
1907 {
1908 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1909 }
1910 else
1911 {
1912 ADD_TAG(entry, tag_albumartist, &id3.artist);
1913 }
1914 if (has_grouping)
1915 {
1916 ADD_TAG(entry, tag_grouping, &id3.grouping);
1917 }
1918 else
1919 {
1920 ADD_TAG(entry, tag_grouping, &id3.title);
1921 }
1922 entry.data_length = offset;
1923
1924 /* Write the header */
1925 write(cachefd, &entry, sizeof(struct temp_file_entry));
1926
1927 /* And tags also... Correct order is critical */
1928 write_item(path);
1929 write_item(id3.title);
1930 write_item(id3.artist);
1931 write_item(id3.album);
1932 write_item(id3.genre_string);
1933 write_item(id3.composer);
1934 write_item(id3.comment);
1935 if (has_albumartist)
1936 {
1937 write_item(id3.albumartist);
1938 }
1939 else
1940 {
1941 write_item(id3.artist);
1942 }
1943 if (has_grouping)
1944 {
1945 write_item(id3.grouping);
1946 }
1947 else
1948 {
1949 write_item(id3.title);
1950 }
1951 total_entry_count++;
1952 }
1953
1954 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1955 {
1956 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1957 int len = strlen(str)+1;
1958 int i;
1959 unsigned crc32;
1960 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1961 char buf[TAG_MAXLEN+32];
1962
1963 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1964 buf[i] = tolower(str[i]);
1965 buf[i] = '\0';
1966
1967 crc32 = crc_32(buf, i, 0xffffffff);
1968
1969 if (unique)
1970 {
1971 /* Check if the crc does not exist -> entry does not exist for sure. */
1972 for (i = 0; i < tempbufidx; i++)
1973 {
1974 if (crcbuf[-i] != crc32)
1975 continue;
1976
1977 if (!strcasecmp(str, index[i].str))
1978 {
1979 if (id < 0 || id >= lookup_buffer_depth)
1980 {
1981 logf("lookup buf overf.: %d", id);
1982 return false;
1983 }
1984
1985 lookup[id] = &index[i];
1986 return true;
1987 }
1988 }
1989 }
1990
1991 /* Insert to CRC buffer. */
1992 crcbuf[-tempbufidx] = crc32;
1993 tempbuf_left -= 4;
1994
1995 /* Insert it to the buffer. */
1996 tempbuf_left -= len;
1997 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1998 return false;
1999
2000 if (id >= lookup_buffer_depth)
2001 {
2002 logf("lookup buf overf. #2: %d", id);
2003 return false;
2004 }
2005
2006 if (id >= 0)
2007 {
2008 lookup[id] = &index[tempbufidx];
2009 index[tempbufidx].idlist.id = id;
2010 }
2011 else
2012 index[tempbufidx].idlist.id = -1;
2013
2014 index[tempbufidx].idlist.next = NULL;
2015 index[tempbufidx].idx_id = idx_id;
2016 index[tempbufidx].seek = -1;
2017 index[tempbufidx].str = &tempbuf[tempbuf_pos];
2018 memcpy(index[tempbufidx].str, str, len);
2019 tempbuf_pos += len;
2020 tempbufidx++;
2021
2022 return true;
2023 }
2024
2025 static int compare(const void *p1, const void *p2)
2026 {
2027 do_timed_yield();
2028
2029 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2030 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2031
2032 if (strcmp(e1->str, UNTAGGED) == 0)
2033 {
2034 if (strcmp(e2->str, UNTAGGED) == 0)
2035 return 0;
2036 return -1;
2037 }
2038 else if (strcmp(e2->str, UNTAGGED) == 0)
2039 return 1;
2040
2041 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2042 }
2043
2044 static int tempbuf_sort(int fd)
2045 {
2046 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2047 struct tagfile_entry fe;
2048 int i;
2049 int length;
2050
2051 /* Generate reverse lookup entries. */
2052 for (i = 0; i < lookup_buffer_depth; i++)
2053 {
2054 struct tempbuf_id_list *idlist;
2055
2056 if (!lookup[i])
2057 continue;
2058
2059 if (lookup[i]->idlist.id == i)
2060 continue;
2061
2062 idlist = &lookup[i]->idlist;
2063 while (idlist->next != NULL)
2064 idlist = idlist->next;
2065
2066 tempbuf_left -= sizeof(struct tempbuf_id_list);
2067 if (tempbuf_left - 4 < 0)
2068 return -1;
2069
2070 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2071 if (tempbuf_pos & 0x03)
2072 {
2073 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2074 tempbuf_left -= 3;
2075 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2076 }
2077 tempbuf_pos += sizeof(struct tempbuf_id_list);
2078
2079 idlist = idlist->next;
2080 idlist->id = i;
2081 idlist->next = NULL;
2082
2083 do_timed_yield();
2084 }
2085
2086 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2087 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2088
2089 for (i = 0; i < tempbufidx; i++)
2090 {
2091 struct tempbuf_id_list *idlist = &index[i].idlist;
2092
2093 /* Fix the lookup list. */
2094 while (idlist != NULL)
2095 {
2096 if (idlist->id >= 0)
2097 lookup[idlist->id] = &index[i];
2098 idlist = idlist->next;
2099 }
2100
2101 index[i].seek = lseek(fd, 0, SEEK_CUR);
2102 length = strlen(index[i].str) + 1;
2103 fe.tag_length = length;
2104 fe.idx_id = index[i].idx_id;
2105
2106 /* Check the chunk alignment. */
2107 if ((fe.tag_length + sizeof(struct tagfile_entry))
2108 % TAGFILE_ENTRY_CHUNK_LENGTH)
2109 {
2110 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2111 ((fe.tag_length + sizeof(struct tagfile_entry))
2112 % TAGFILE_ENTRY_CHUNK_LENGTH);
2113 }
2114
2115 #ifdef TAGCACHE_STRICT_ALIGN
2116 /* Make sure the entry is long aligned. */
2117 if (index[i].seek & 0x03)
2118 {
2119 logf("tempbuf_sort: alignment error!");
2120 return -3;
2121 }
2122 #endif
2123
2124 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2125 sizeof(struct tagfile_entry))
2126 {
2127 logf("tempbuf_sort: write error #1");
2128 return -1;
2129 }
2130
2131 if (write(fd, index[i].str, length) != length)
2132 {
2133 logf("tempbuf_sort: write error #2");
2134 return -2;
2135 }
2136
2137 /* Write some padding. */
2138 if (fe.tag_length - length > 0)
2139 write(fd, "XXXXXXXX", fe.tag_length - length);
2140 }
2141
2142 return i;
2143 }
2144
2145 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2146 {
2147 if (id < 0 || id >= lookup_buffer_depth)
2148 return NULL;
2149
2150 return lookup[id];
2151 }
2152
2153
2154 inline static int tempbuf_find_location(int id)
2155 {
2156 struct tempbuf_searchidx *entry;
2157
2158 entry = tempbuf_locate(id);
2159 if (entry == NULL)
2160 return -1;
2161
2162 return entry->seek;
2163 }
2164
2165 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2166 {
2167 struct master_header tcmh;
2168 struct index_entry idx;
2169 int masterfd;
2170 int masterfd_pos;
2171 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2172 int max_entries;
2173 int entries_processed = 0;
2174 int i, j;
2175 char buf[TAG_MAXLEN];
2176
2177 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2178
2179 logf("Building numeric indices...");
2180 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2181
2182 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2183 return false;
2184
2185 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2186 SEEK_CUR);
2187 if (masterfd_pos == filesize(masterfd))
2188 {
2189 logf("we can't append!");
2190 close(masterfd);
2191 return false;
2192 }
2193
2194 while (entries_processed < h->entry_count)
2195 {
2196 int count = MIN(h->entry_count - entries_processed, max_entries);
2197
2198 /* Read in as many entries as possible. */
2199 for (i = 0; i < count; i++)
2200 {
2201 struct temp_file_entry *tfe = &entrybuf[i];
2202 int datastart;
2203
2204 /* Read in numeric data. */
2205 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2206 sizeof(struct temp_file_entry))
2207 {
2208 logf("read fail #1");
2209 close(masterfd);
2210 return false;
2211 }
2212
2213 datastart = lseek(tmpfd, 0, SEEK_CUR);
2214
2215 /**
2216 * Read string data from the following tags:
2217 * - tag_filename
2218 * - tag_artist
2219 * - tag_album
2220 * - tag_title
2221 *
2222 * A crc32 hash is calculated from the read data
2223 * and stored back to the data offset field kept in memory.
2224 */
2225 #define tmpdb_read_string_tag(tag) \
2226 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2227 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2228 { \
2229 logf("read fail: buffer overflow"); \
2230 close(masterfd); \
2231 return false; \
2232 } \
2233 \
2234 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2235 tfe->tag_length[tag]) \
2236 { \
2237 logf("read fail #2"); \
2238 close(masterfd); \
2239 return false; \
2240 } \
2241 \
2242 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2243 lseek(tmpfd, datastart, SEEK_SET)
2244
2245 tmpdb_read_string_tag(tag_filename);
2246 tmpdb_read_string_tag(tag_artist);
2247 tmpdb_read_string_tag(tag_album);
2248 tmpdb_read_string_tag(tag_title);
2249
2250 /* Seek to the end of the string data. */
2251 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2252 }
2253
2254 /* Backup the master index position. */
2255 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2256 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2257
2258 /* Check if we can resurrect some deleted runtime statistics data. */
2259 for (i = 0; i < tcmh.tch.entry_count; i++)
2260 {
2261 /* Read the index entry. */
2262 if (ecread_index_entry(masterfd, &idx)
2263 != sizeof(struct index_entry))
2264 {
2265 logf("read fail #3");
2266 close(masterfd);
2267 return false;
2268 }
2269
2270 /**
2271 * Skip unless the entry is marked as being deleted
2272 * or the data has already been resurrected.
2273 */
2274 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2275 continue;
2276
2277 /* Now try to match the entry. */
2278 /**
2279 * To succesfully match a song, the following conditions
2280 * must apply:
2281 *
2282 * For numeric fields: tag_length
2283 * - Full identical match is required
2284 *
2285 * If tag_filename matches, no further checking necessary.
2286 *
2287 * For string hashes: tag_artist, tag_album, tag_title
2288 * - Two of these must match
2289 */
2290 for (j = 0; j < count; j++)
2291 {
2292 struct temp_file_entry *tfe = &entrybuf[j];
2293
2294 /* Try to match numeric fields first. */
2295 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2296 continue;
2297
2298 /* Now it's time to do the hash matching. */
2299 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2300 {
2301 int match_count = 0;
2302
2303 /* No filename match, check if we can match two other tags. */
2304 #define tmpdb_match(tag) \
2305 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2306 match_count++
2307
2308 tmpdb_match(tag_artist);
2309 tmpdb_match(tag_album);
2310 tmpdb_match(tag_title);
2311
2312 if (match_count < 2)
2313 {
2314 /* Still no match found, give up. */
2315 continue;
2316 }
2317 }
2318
2319 /* A match found, now copy & resurrect the statistical data. */
2320 #define tmpdb_copy_tag(tag) \
2321 tfe->tag_offset[tag] = idx.tag_seek[tag]
2322
2323 tmpdb_copy_tag(tag_playcount);
2324 tmpdb_copy_tag(tag_rating);
2325 tmpdb_copy_tag(tag_playtime);
2326 tmpdb_copy_tag(tag_lastplayed);
2327 tmpdb_copy_tag(tag_commitid);
2328 tmpdb_copy_tag(tag_lastoffset);
2329
2330 /* Avoid processing this entry again. */
2331 idx.flag |= FLAG_RESURRECTED;
2332
2333 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2334 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2335 {
2336 logf("masterfd writeback fail #1");
2337 close(masterfd);
2338 return false;
2339 }
2340
2341 logf("Entry resurrected");
2342 }
2343 }
2344
2345
2346 /* Restore the master index position. */
2347 lseek(masterfd, masterfd_pos, SEEK_SET);
2348
2349 /* Commit the data to the index. */
2350 for (i = 0; i < count; i++)
2351 {
2352 int loc = lseek(masterfd, 0, SEEK_CUR);
2353
2354 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2355 {
2356 logf("read fail #3");
2357 close(masterfd);
2358 return false;
2359 }
2360
2361 for (j = 0; j < TAG_COUNT; j++)
2362 {
2363 if (!TAGCACHE_IS_NUMERIC(j))
2364 continue;
2365
2366 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2367 }
2368 idx.flag = entrybuf[i].flag;
2369
2370 if (idx.tag_seek[tag_commitid])
2371 {
2372 /* Data has been resurrected. */
2373 idx.flag |= FLAG_DIRTYNUM;
2374 }
2375 else if (tc_stat.ready && current_tcmh.commitid > 0)
2376 {
2377 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2378 idx.flag |= FLAG_DIRTYNUM;
2379 }
2380
2381 /* Write back the updated index. */
2382 lseek(masterfd, loc, SEEK_SET);
2383 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2384 {
2385 logf("write fail");
2386 close(masterfd);
2387 return false;
2388 }
2389 }
2390
2391 entries_processed += count;
2392 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2393 }
2394
2395 close(masterfd);
2396
2397 return true;
2398 }
2399
2400 /**
2401 * Return values:
2402 * > 0 success
2403 * == 0 temporary failure
2404 * < 0 fatal error
2405 */
2406 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2407 {
2408 int i;
2409 struct tagcache_header tch;
2410 struct master_header tcmh;
2411 struct index_entry idxbuf[IDX_BUF_DEPTH];
2412 int idxbuf_pos;
2413 char buf[TAG_MAXLEN+32];
2414 int fd = -1, masterfd;
2415 bool error = false;
2416 int init;
2417 int masterfd_pos;
2418
2419 logf("Building index: %d", index_type);
2420
2421 /* Check the number of entries we need to allocate ram for. */
2422 commit_entry_count = h->entry_count + 1;
2423
2424 masterfd = open_master_fd(&tcmh, false);
2425 if (masterfd >= 0)
2426 {
2427 commit_entry_count += tcmh.tch.entry_count;
2428 close(masterfd);
2429 }
2430 else
2431 remove_files(); /* Just to be sure we are clean. */
2432
2433 /* Open the index file, which contains the tag names. */
2434 fd = open_tag_fd(&tch, index_type, true);
2435 if (fd >= 0)
2436 {
2437 logf("tch.datasize=%ld", tch.datasize);
2438 lookup_buffer_depth = 1 +
2439 /* First part */ commit_entry_count +
2440 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2441 }
2442 else
2443 {
2444 lookup_buffer_depth = 1 +
2445 /* First part */ commit_entry_count +
2446 /* Second part */ 0;
2447 }
2448
2449 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2450 logf("commit_entry_count=%ld", commit_entry_count);
2451
2452 /* Allocate buffer for all index entries from both old and new
2453 * tag files. */
2454 tempbufidx = 0;
2455 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2456
2457 /* Allocate lookup buffer. The first portion of commit_entry_count
2458 * contains the new tags in the temporary file and the second
2459 * part for locating entries already in the db.
2460 *
2461 * New tags Old tags
2462 * +---------+---------------------------+
2463 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2464 * +---------+---------------------------+
2465 *
2466 * Old tags are inserted to a temporary buffer with position:
2467 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2468 * And new tags with index:
2469 * tempbuf_insert(idx, ...);
2470 *
2471 * The buffer is sorted and written into tag file:
2472 * tempbuf_sort(...);
2473 * leaving master index locations messed up.
2474 *
2475 * That is fixed using the lookup buffer for old tags:
2476 * new_seek = tempbuf_find_location(old_seek, ...);
2477 * and for new tags:
2478 * new_seek = tempbuf_find_location(idx);
2479 */
2480 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2481 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2482 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2483
2484 /* And calculate the remaining data space used mainly for storing
2485 * tag data (strings). */
2486 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2487 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2488 {
2489 logf("Buffer way too small!");
2490 return 0;
2491 }
2492
2493 if (fd >= 0)
2494 {
2495 /**
2496 * If tag file contains unique tags (sorted index), we will load
2497 * it entirely into memory so we can resort it later for use with
2498 * chunked browsing.
2499 */
2500 if (TAGCACHE_IS_SORTED(index_type))
2501 {
2502 logf("loading tags...");
2503 for (i = 0; i < tch.entry_count; i++)
2504 {
2505 struct tagfile_entry entry;
2506 int loc = lseek(fd, 0, SEEK_CUR);
2507 bool ret;
2508
2509 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2510 {
2511 logf("read error #7");
2512 close(fd);
2513 return -2;
2514 }
2515
2516 if (entry.tag_length >= (int)sizeof(buf))
2517 {
2518 logf("too long tag #3");
2519 close(fd);
2520 return -2;
2521 }
2522
2523 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2524 {
2525 logf("read error #8");
2526 close(fd);
2527 return -2;
2528 }
2529
2530 /* Skip deleted entries. */
2531 if (buf[0] == '\0')
2532 continue;
2533
2534 /**
2535 * Save the tag and tag id in the memory buffer. Tag id
2536 * is saved so we can later reindex the master lookup
2537 * table when the index gets resorted.
2538 */
2539 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2540 + commit_entry_count, entry.idx_id,
2541 TAGCACHE_IS_UNIQUE(index_type));
2542 if (!ret)
2543 {
2544 close(fd);
2545 return -3;
2546 }
2547 do_timed_yield();
2548 }
2549 logf("done");
2550 }
2551 else
2552 tempbufidx = tch.entry_count;
2553 }
2554 else
2555 {
2556 /**
2557 * Creating new index file to store the tags. No need to preload
2558 * anything whether the index type is sorted or not.
2559 */
2560 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2561 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2562 if (fd < 0)
2563 {
2564 logf("%s open fail", buf);
2565 return -2;
2566 }
2567
2568 tch.magic = TAGCACHE_MAGIC;
2569 tch.entry_count = 0;
2570 tch.datasize = 0;
2571
2572 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2573 != sizeof(struct tagcache_header))
2574 {
2575 logf("header write failed");
2576 close(fd);
2577 return -2;
2578 }
2579 }
2580
2581 /* Loading the tag lookup file as "master file". */
2582 logf("Loading index file");
2583 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2584
2585 if (masterfd < 0)
2586 {
2587 logf("Creating new DB");
2588 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2589
2590 if (masterfd < 0)
2591 {
2592 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2593 close(fd);
2594 return -2;
2595 }
2596
2597 /* Write the header (write real values later). */
2598 memset(&tcmh, 0, sizeof(struct master_header));
2599 tcmh.tch = *h;
2600 tcmh.tch.entry_count = 0;
2601 tcmh.tch.datasize = 0;
2602 tcmh.dirty = true;
2603 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2604 init = true;
2605 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2606 }
2607 else
2608 {
2609 /**
2610 * Master file already exists so we need to process the current
2611 * file first.
2612 */
2613 init = false;
2614
2615 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2616 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2617 {
2618 logf("header error");
2619 close(fd);
2620 close(masterfd);
2621 return -2;
2622 }
2623
2624 /**
2625 * If we reach end of the master file, we need to expand it to
2626 * hold new tags. If the current index is not sorted, we can
2627 * simply append new data to end of the file.
2628 * However, if the index is sorted, we need to update all tag
2629 * pointers in the master file for the current index.
2630 */
2631 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2632 SEEK_CUR);
2633 if (masterfd_pos == filesize(masterfd))
2634 {
2635 logf("appending...");
2636 init = true;
2637 }
2638 }
2639
2640 /**
2641 * Load new unique tags in memory to be sorted later and added
2642 * to the master lookup file.
2643 */
2644 if (TAGCACHE_IS_SORTED(index_type))
2645 {
2646 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2647 /* h is the header of the temporary file containing new tags. */
2648 logf("inserting new tags...");
2649 for (i = 0; i < h->entry_count; i++)
2650 {
2651 struct temp_file_entry entry;
2652
2653 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2654 sizeof(struct temp_file_entry))
2655 {
2656 logf("read fail #3");
2657 error = true;
2658 goto error_exit;
2659 }
2660
2661 /* Read data. */
2662 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2663 {
2664 logf("too long entry!");
2665 error = true;
2666 goto error_exit;
2667 }
2668
2669 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2670 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2671 entry.tag_length[index_type])
2672 {
2673 logf("read fail #4");
2674 error = true;
2675 goto error_exit;
2676 }
2677
2678 if (TAGCACHE_IS_UNIQUE(index_type))
2679 error = !tempbuf_insert(buf, i, -1, true);
2680 else
2681 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2682
2683 if (error)
2684 {
2685 logf("insert error");
2686 goto error_exit;
2687 }
2688
2689 /* Skip to next. */
2690 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2691 entry.tag_length[index_type], SEEK_CUR);
2692 do_timed_yield();
2693 }
2694 logf("done");
2695
2696 /* Sort the buffer data and write it to the index file. */
2697 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2698 /**
2699 * We need to truncate the index file now. There can be junk left
2700 * at the end of file (however, we _should_ always follow the
2701 * entry_count and don't crash with that).
2702 */
2703 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2704
2705 i = tempbuf_sort(fd);
2706 if (i < 0)
2707 goto error_exit;
2708 logf("sorted %d tags", i);
2709
2710 /**
2711 * Now update all indexes in the master lookup file.
2712 */
2713 logf("updating indices...");
2714 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2715 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2716 {
2717 int j;
2718 int loc = lseek(masterfd, 0, SEEK_CUR);
2719
2720 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2721
2722 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2723 != (int)sizeof(struct index_entry)*idxbuf_pos)
2724 {
2725 logf("read fail #5");
2726 error = true;
2727 goto error_exit ;
2728 }
2729 lseek(masterfd, loc, SEEK_SET);
2730
2731 for (j = 0; j < idxbuf_pos; j++)
2732 {
2733 if (idxbuf[j].flag & FLAG_DELETED)
2734 {
2735 /* We can just ignore deleted entries. */
2736 // idxbuf[j].tag_seek[index_type] = 0;
2737 continue;
2738 }
2739
2740 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2741 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2742 + commit_entry_count);
2743
2744 if (idxbuf[j].tag_seek[index_type] < 0)
2745 {
2746 logf("update error: %ld/%d/%ld",
2747 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2748 error = true;
2749 goto error_exit;
2750 }
2751
2752 do_timed_yield();
2753 }
2754
2755 /* Write back the updated index. */
2756 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2757 index_entry_ec, tc_stat.econ) !=
2758 (int)sizeof(struct index_entry)*idxbuf_pos)
2759 {
2760 logf("write fail");
2761 error = true;
2762 goto error_exit;
2763 }
2764 }
2765 logf("done");
2766 }
2767
2768 /**
2769 * Walk through the temporary file containing the new tags.
2770 */
2771 // build_normal_index(h, tmpfd, masterfd, idx);
2772 logf("updating new indices...");
2773 lseek(masterfd, masterfd_pos, SEEK_SET);
2774 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2775 lseek(fd, 0, SEEK_END);
2776 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2777 {
2778 int j;
2779
2780 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2781 if (init)
2782 {
2783 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2784 }
2785 else
2786 {
2787 int loc = lseek(masterfd, 0, SEEK_CUR);
2788
2789 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2790 != (int)sizeof(struct index_entry)*idxbuf_pos)
2791 {
2792 logf("read fail #6");
2793 error = true;
2794 break ;
2795 }
2796 lseek(masterfd, loc, SEEK_SET);
2797 }
2798
2799 /* Read entry headers. */
2800 for (j = 0; j < idxbuf_pos; j++)
2801 {
2802 if (!TAGCACHE_IS_SORTED(index_type))
2803 {
2804 struct temp_file_entry entry;
2805 struct tagfile_entry fe;
2806
2807 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2808 sizeof(struct temp_file_entry))
2809 {
2810 logf("read fail #7");
2811 error = true;
2812 break ;
2813 }
2814
2815 /* Read data. */
2816 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2817 {
2818 logf("too long entry!");
2819 logf("length=%d", entry.tag_length[index_type]);
2820 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2821 error = true;
2822 break ;
2823 }
2824
2825 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2826 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2827 entry.tag_length[index_type])
2828 {
2829 logf("read fail #8");
2830 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2831 logf("length=0x%02x", entry.tag_length[index_type]);
2832 error = true;
2833 break ;
2834 }
2835
2836 /* Write to index file. */
2837 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2838 fe.tag_length = entry.tag_length[index_type];
2839 fe.idx_id = tcmh.tch.entry_count + i + j;
2840 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2841 write(fd, buf, fe.tag_length);
2842 tempbufidx++;
2843
2844 /* Skip to next. */
2845 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2846 entry.tag_length[index_type], SEEK_CUR);
2847 }
2848 else
2849 {
2850 /* Locate the correct entry from the sorted array. */
2851 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2852 if (idxbuf[j].tag_seek[index_type] < 0)
2853 {
2854 logf("entry not found (%d)", j);
2855 error = true;
2856 break ;
2857 }
2858 }
2859 }
2860
2861 /* Write index. */
2862 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2863 index_entry_ec, tc_stat.econ) !=
2864 (int)sizeof(struct index_entry)*idxbuf_pos)
2865 {
2866 logf("tagcache: write fail #4");
2867 error = true;
2868 break ;
2869 }
2870
2871 do_timed_yield();
2872 }
2873 logf("done");
2874
2875 /* Finally write the header. */
2876 tch.magic = TAGCACHE_MAGIC;
2877 tch.entry_count = tempbufidx;
2878 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2879 lseek(fd, 0, SEEK_SET);
2880 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2881
2882 if (index_type != tag_filename)
2883 h->datasize += tch.datasize;
2884 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2885 error_exit:
2886
2887 close(fd);
2888 close(masterfd);
2889
2890 if (error)
2891 return -2;
2892
2893 return 1;
2894 }
2895
2896 static bool commit(void)
2897 {
2898 struct tagcache_header tch;
2899 struct master_header tcmh;
2900 int i, len, rc;
2901 int tmpfd;
2902 int masterfd;
2903 #ifdef HAVE_DIRCACHE
2904 bool dircache_buffer_stolen = false;
2905 #endif
2906 bool local_allocation = false;
2907
2908 logf("committing tagcache");
2909
2910 while (write_lock)
2911 sleep(1);
2912
2913 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2914 if (tmpfd < 0)
2915 {
2916 logf("nothing to commit");
2917 return true;
2918 }
2919
2920
2921 /* Load the header. */
2922 len = sizeof(struct tagcache_header);
2923 rc = read(tmpfd, &tch, len);
2924
2925 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2926 {
2927 logf("incorrect tmpheader");
2928 close(tmpfd);
2929 remove(TAGCACHE_FILE_TEMP);
2930 return false;
2931 }
2932
2933 if (tch.entry_count == 0)
2934 {
2935 logf("nothing to commit");
2936 close(tmpfd);
2937 remove(TAGCACHE_FILE_TEMP);
2938 return true;
2939 }
2940
2941 /* Fully initialize existing headers (if any) before going further. */
2942 tc_stat.ready = check_all_headers();
2943
2944 #ifdef HAVE_EEPROM_SETTINGS
2945 remove(TAGCACHE_STATEFILE);
2946 #endif
2947
2948 /* At first be sure to unload the ramcache! */
2949 #ifdef HAVE_TC_RAMCACHE
2950 tc_stat.ramcache = false;
2951 #endif
2952
2953 read_lock++;
2954
2955 /* Try to steal every buffer we can :) */
2956 if (tempbuf_size == 0)
2957 local_allocation = true;
2958
2959 #ifdef HAVE_DIRCACHE
2960 if (tempbuf_size == 0)
2961 {
2962 /* Try to steal the dircache buffer. */
2963 tempbuf = dircache_steal_buffer(&tempbuf_size);
2964 tempbuf_size &= ~0x03;
2965
2966 if (tempbuf_size > 0)
2967 {
2968 dircache_buffer_stolen = true;
2969 }
2970 }
2971 #endif
2972
2973 #ifdef HAVE_TC_RAMCACHE
2974 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2975 {
2976 tempbuf = (char *)(hdr + 1);
2977 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2978 tempbuf_size &= ~0x03;
2979 }
2980 #endif
2981
2982 /* And finally fail if there are no buffers available. */
2983 if (tempbuf_size == 0)
2984 {
2985 logf("delaying commit until next boot");
2986 tc_stat.commit_delayed = true;
2987 close(tmpfd);
2988 read_lock--;
2989 return false;
2990 }
2991
2992 logf("commit %ld entries...", tch.entry_count);
2993
2994 /* Mark DB dirty so it will stay disabled if commit fails. */
2995 current_tcmh.dirty = true;
2996 update_master_header();
2997
2998 /* Now create the index files. */
2999 tc_stat.commit_step = 0;
3000 tch.datasize = 0;
3001 tc_stat.commit_delayed = false;
3002
3003 for (i = 0; i < TAG_COUNT; i++)
3004 {
3005 int ret;
3006
3007 if (TAGCACHE_IS_NUMERIC(i))
3008 continue;
3009
3010 tc_stat.commit_step++;
3011 ret = build_index(i, &tch, tmpfd);
3012 if (ret <= 0)
3013 {
3014 close(tmpfd);
3015 logf("tagcache failed init");
3016 if (ret == 0)
3017 tc_stat.commit_delayed = true;
3018
3019 tc_stat.commit_step = 0;
3020 read_lock--;
3021 return false;
3022 }
3023 }
3024
3025 if (!build_numeric_indices(&tch, tmpfd))
3026 {
3027 logf("Failure to commit numeric indices");
3028 close(tmpfd);
3029 tc_stat.commit_step = 0;
3030 read_lock--;
3031 return false;
3032 }
3033
3034 close(tmpfd);
3035 remove(TAGCACHE_FILE_TEMP);
3036
3037 tc_stat.commit_step = 0;
3038
3039 /* Update the master index headers. */
3040 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3041 {
3042 read_lock--;
3043 return false;
3044 }
3045
3046 tcmh.tch.entry_count += tch.entry_count;
3047 tcmh.tch.datasize = sizeof(struct master_header)
3048 + sizeof(struct index_entry) * tcmh.tch.entry_count
3049 + tch.datasize;
3050 tcmh.dirty = false;
3051 tcmh.commitid++;
3052
3053 lseek(masterfd, 0, SEEK_SET);
3054 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3055 close(masterfd);
3056
3057 logf("tagcache committed");
3058 tc_stat.ready = check_all_headers();
3059 tc_stat.readyvalid = true;
3060
3061 if (local_allocation)
3062 {
3063 tempbuf = NULL;
3064 tempbuf_size = 0;
3065 }
3066
3067 #ifdef HAVE_DIRCACHE
3068 /* Rebuild the dircache, if we stole the buffer. */
3069 if (dircache_buffer_stolen)
3070 dircache_build(0);
3071 #endif
3072
3073 #ifdef HAVE_TC_RAMCACHE
3074 /* Reload tagcache. */
3075 if (tc_stat.ramcache_allocated > 0)
3076 tagcache_start_scan();
3077 #endif
3078
3079 read_lock--;
3080
3081 return true;
3082 }
3083
3084 static void allocate_tempbuf(void)
3085 {
3086 /* Yeah, malloc would be really nice now :) */
3087 #ifdef __PCTOOL__
3088 tempbuf_size = 32*1024*1024;
3089 tempbuf = malloc(tempbuf_size);
3090 #else
3091 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
3092 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
3093 audiobuf += tempbuf_size;
3094 #endif
3095 }
3096
3097 static void free_tempbuf(void)
3098 {
3099 if (tempbuf_size == 0)
3100 return ;
3101
3102 #ifdef __PCTOOL__
3103 free(tempbuf);
3104 #else
3105 audiobuf -= tempbuf_size;
3106 #endif
3107 tempbuf = NULL;
3108 tempbuf_size = 0;
3109 }
3110
3111 #ifndef __PCTOOL__
3112
3113 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3114 {
3115 struct index_entry idx;
3116
3117 if (!tc_stat.ready)
3118 return false;
3119
3120 if (!TAGCACHE_IS_NUMERIC(tag))
3121 return false;
3122
3123 if (!get_index(masterfd, idx_id, &idx, false))
3124 return false;
3125
3126 idx.tag_seek[tag] = data;
3127 idx.flag |= FLAG_DIRTYNUM;
3128
3129 return write_index(masterfd, idx_id, &idx);
3130 }
3131
3132 #if 0
3133 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3134 int tag, long data)
3135 {
3136 struct master_header myhdr;
3137
3138 if (tcs->masterfd < 0)
3139 {
3140 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3141 return false;
3142 }
3143
3144 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3145 }
3146 #endif
3147
3148 static bool command_queue_is_full(void)
3149 {
3150 int next;
3151
3152 next = command_queue_widx + 1;
3153 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3154 next = 0;
3155
3156 return (next == command_queue_ridx);
3157 }
3158
3159 static void command_queue_sync_callback(void *data)
3160 {
3161 (void)data;
3162 struct master_header myhdr;
3163 int masterfd;
3164
3165 mutex_lock(&command_queue_mutex);
3166
3167 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3168 return;
3169
3170 while (command_queue_ridx != command_queue_widx)
3171 {
3172 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3173
3174 switch (ce->command)
3175 {
3176 case CMD_UPDATE_MASTER_HEADER:
3177 {
3178 close(masterfd);
3179 update_master_header();
3180
3181 /* Re-open the masterfd. */
3182 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3183 return;
3184
3185 break;
3186 }
3187 case CMD_UPDATE_NUMERIC:
3188 {
3189 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3190 break;
3191 }
3192 }
3193
3194 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3195 command_queue_ridx = 0;
3196 }
3197
3198 close(masterfd);
3199
3200 tc_stat.queue_length = 0;
3201 mutex_unlock(&command_queue_mutex);
3202 }
3203
3204 static void run_command_queue(bool force)
3205 {
3206 if (COMMAND_QUEUE_IS_EMPTY)
3207 return;
3208
3209 if (force || command_queue_is_full())
3210 command_queue_sync_callback(NULL);
3211 else
3212 register_storage_idle_func(command_queue_sync_callback);
3213 }
3214
3215 static void queue_command(int cmd, long idx_id, int tag, long data)
3216 {
3217 while (1)
3218 {
3219 int next;
3220
3221 mutex_lock(&command_queue_mutex);
3222 next = command_queue_widx + 1;
3223 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3224 next = 0;
3225
3226 /* Make sure queue is not full. */
3227 if (next != command_queue_ridx)
3228 {
3229 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3230
3231 ce->command = cmd;
3232 ce->idx_id = idx_id;
3233 ce->tag = tag;
3234 ce->data = data;
3235
3236 command_queue_widx = next;
3237
3238 tc_stat.queue_length++;
3239
3240 mutex_unlock(&command_queue_mutex);
3241 break;
3242 }
3243
3244 /* Queue is full, try again later... */
3245 mutex_unlock(&command_queue_mutex);
3246 sleep(1);
3247 }
3248 }
3249
3250 long tagcache_increase_serial(void)
3251 {
3252 long old;
3253
3254 if (!tc_stat.ready)
3255 return -2;
3256
3257 while (read_lock)
3258 sleep(1);
3259
3260 old = current_tcmh.serial++;
3261 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3262
3263 return old;
3264 }
3265
3266 void tagcache_update_numeric(int idx_id, int tag, long data)
3267 {
3268 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3269 }
3270 #endif /* !__PCTOOL__ */
3271
3272 long tagcache_get_serial(void)
3273 {
3274 return current_tcmh.serial;
3275 }
3276
3277 long tagcache_get_commitid(void)
3278 {
3279 return current_tcmh.commitid;
3280 }
3281
3282 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3283 {
3284 char buf[512];
3285 int i;
3286
3287 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3288 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3289 {
3290 if (*datastr == '\0')
3291 break;
3292
3293 if (*datastr == '"' || *datastr == '\\')
3294 buf[i++] = '\\';
3295
3296 buf[i] = *(datastr++);
3297 }
3298
3299 strcpy(&buf[i], "\" ");
3300
3301 write(fd, buf, i + 2);
3302
3303 return true;
3304 }
3305
3306 #ifndef __PCTOOL__
3307
3308 static bool read_tag(char *dest, long size,
3309 const char *src, const char *tagstr)
3310 {
3311 int pos;
3312 char current_tag[32];
3313
3314 while (*src != '\0')
3315 {
3316 /* Skip all whitespace */
3317 while (*src == ' ')
3318 src++;
3319
3320 if (*src == '\0')
3321 break;
3322
3323 pos = 0;
3324 /* Read in tag name */
3325 while (*src != '=' && *src != ' ')
3326 {
3327 current_tag[pos] = *src;
3328 src++;
3329 pos++;
3330
3331 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3332 return false;
3333 }
3334 current_tag[pos] = '\0';
3335
3336 /* Read in tag data */
3337
3338 /* Find the start. */
3339 while (*src != '"' && *src != '\0')
3340 src++;
3341
3342 if (*src == '\0' || *(++src) == '\0')
3343 return false;
3344
3345 /* Read the data. */
3346 for (pos = 0; pos < size; pos++)
3347 {
3348 if (*src == '\0')
3349 break;
3350
3351 if (*src == '\\')
3352 {
3353 dest[pos] = *(src+1);
3354 src += 2;
3355 continue;
3356 }
3357
3358 dest[pos] = *src;
3359
3360 if (*src == '"')
3361 {
3362 src++;
3363 break;
3364 }
3365
3366 if (*src == '\0')
3367 break;
3368
3369 src++;
3370 }
3371 dest[pos] = '\0';
3372
3373 if (!strcasecmp(tagstr, current_tag))
3374 return true;
3375 }
3376
3377 return false;
3378 }
3379
3380 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3381 {
3382 struct index_entry idx;
3383 char tag_data[TAG_MAXLEN+32];
3384 int idx_id;
3385 long masterfd = (long)parameters;
3386 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3387 tag_commitid };
3388 int i;
3389 (void)line_n;
3390
3391 if (*buf == '#')
3392 return 0;
3393
3394 logf("%d/%s", line_n, buf);
3395 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3396 {
3397 logf("filename missing");
3398 logf("-> %s", buf);
3399 return 0;
3400 }
3401
3402 idx_id = find_index(tag_data);
3403 if (idx_id < 0)
3404 {
3405 logf("entry not found");
3406 return 0;
3407 }
3408
3409 if (!get_index(masterfd, idx_id, &idx, false))
3410 {
3411 logf("failed to retrieve index entry");
3412 return 0;
3413 }
3414
3415 /* Stop if tag has already been modified. */
3416 if (idx.flag & FLAG_DIRTYNUM)
3417 return 0;
3418
3419 logf("import: %s", tag_data);
3420
3421 idx.flag |= FLAG_DIRTYNUM;
3422 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3423 {
3424 int data;
3425
3426 if (!read_tag(tag_data, sizeof tag_data, buf,
3427 tagcache_tag_to_str(import_tags[i])))
3428 {
3429 continue;
3430 }
3431
3432 data = atoi(tag_data);
3433 if (data < 0)
3434 continue;
3435
3436 idx.tag_seek[import_tags[i]] = data;
3437
3438 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3439 current_tcmh.serial = data + 1;
3440 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3441 current_tcmh.commitid = data + 1;
3442 }
3443
3444 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3445 }
3446
3447 bool tagcache_import_changelog(void)
3448 {
3449 struct master_header myhdr;
3450 struct tagcache_header tch;
3451 int clfd;
3452 long masterfd;
3453 char buf[2048];
3454
3455 if (!tc_stat.ready)
3456 return false;
3457
3458 while (read_lock)
3459 sleep(1);
3460
3461 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3462 if (clfd < 0)
3463 {
3464 logf("failure to open changelog");
3465 return false;
3466 }
3467
3468 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3469 {
3470 close(clfd);
3471 return false;
3472 }
3473
3474 write_lock++;
3475
3476 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3477
3478 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3479 parse_changelog_line);
3480
3481 close(clfd);
3482 close(masterfd);
3483
3484 if (filenametag_fd >= 0)
3485 {
3486 close(filenametag_fd);
3487 filenametag_fd = -1;
3488 }
3489
3490 write_lock--;
3491
3492 update_master_header();
3493
3494 return true;
3495 }
3496
3497 #endif /* !__PCTOOL__ */
3498
3499 bool tagcache_create_changelog(struct tagcache_search *tcs)
3500 {
3501 struct master_header myhdr;
3502 struct index_entry idx;
3503 char buf[TAG_MAXLEN+32];
3504 char temp[32];
3505 int clfd;
3506 int i, j;
3507
3508 if (!tc_stat.ready)
3509 return false;
3510
3511 if (!tagcache_search(tcs, tag_filename))
3512 return false;
3513
3514 /* Initialize the changelog */
3515 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3516 if (clfd < 0)
3517 {
3518 logf("failure to open changelog");
3519 return false;
3520 }
3521
3522 if (tcs->masterfd < 0)
3523 {
3524 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3525 return false;
3526 }
3527 else
3528 {
3529 lseek(tcs->masterfd, 0, SEEK_SET);
3530 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3531 }
3532
3533 write(clfd, "## Changelog version 1\n", 23);
3534
3535 for (i = 0; i < myhdr.tch.entry_count; i++)
3536 {
3537 if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3538 {
3539 logf("read error #9");
3540 tagcache_search_finish(tcs);
3541 close(clfd);
3542 return false;
3543 }
3544
3545 /* Skip until the entry found has been modified. */
3546 if (! (idx.flag & FLAG_DIRTYNUM) )
3547 continue;
3548
3549 /* Skip deleted entries too. */
3550 if (idx.flag & FLAG_DELETED)
3551 continue;
3552
3553 /* Now retrieve all tags. */
3554 for (j = 0; j < TAG_COUNT; j++)
3555 {
3556 if (TAGCACHE_IS_NUMERIC(j))
3557 {
3558 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3559 write_tag(clfd, tagcache_tag_to_str(j), temp);
3560 continue;
3561 }
3562
3563 tcs->type = j;
3564 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3565 write_tag(clfd, tagcache_tag_to_str(j), buf);
3566 }
3567
3568 write(clfd, "\n", 1);
3569 do_timed_yield();
3570 }
3571
3572 close(clfd);
3573
3574 tagcache_search_finish(tcs);
3575
3576 return true;
3577 }
3578
3579 static bool delete_entry(long idx_id)
3580 {
3581 int fd = -1;
3582 int masterfd = -1;
3583 int tag, i;
3584 struct index_entry idx, myidx;
3585 struct master_header myhdr;
3586 char buf[TAG_MAXLEN+32];
3587 int in_use[TAG_COUNT];
3588
3589 logf("delete_entry(): %ld", idx_id);
3590
3591 #ifdef HAVE_TC_RAMCACHE
3592 /* At first mark the entry removed from ram cache. */
3593 if (tc_stat.ramcache)
3594 hdr->indices[idx_id].flag |= FLAG_DELETED;
3595 #endif
3596
3597 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3598 return false;
3599
3600 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3601 if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3602 {
3603 logf("delete_entry(): read error");
3604 goto cleanup;
3605 }
3606
3607 if (myidx.flag & FLAG_DELETED)
3608 {
3609 logf("delete_entry(): already deleted!");
3610 goto cleanup;
3611 }
3612
3613 myidx.flag |= FLAG_DELETED;
3614 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3615 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3616 {
3617 logf("delete_entry(): write_error #1");
3618 goto cleanup;
3619 }
3620
3621 /* Now check which tags are no longer in use (if any) */
3622 for (tag = 0; tag < TAG_COUNT; tag++)
3623 in_use[tag] = 0;
3624
3625 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3626 for (i = 0; i < myhdr.tch.entry_count; i++)
3627 {
3628 struct index_entry *idxp;
3629
3630 #ifdef HAVE_TC_RAMCACHE
3631 /* Use RAM DB if available for greater speed */
3632 if (tc_stat.ramcache)
3633 idxp = &hdr->indices[i];
3634 else
3635 #endif
3636 {
3637 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3638 {
3639 logf("delete_entry(): read error #2");
3640 goto cleanup;
3641 }
3642 idxp = &idx;
3643 }
3644
3645 if (idxp->flag & FLAG_DELETED)
3646 continue;
3647
3648 for (tag = 0; tag < TAG_COUNT; tag++)
3649 {
3650 if (TAGCACHE_IS_NUMERIC(tag))
3651 continue;
3652
3653 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3654 in_use[tag]++;
3655 }
3656 }
3657
3658 /* Now delete all tags no longer in use. */
3659 for (tag = 0; tag < TAG_COUNT; tag++)
3660 {
3661 struct tagcache_header tch;
3662 int oldseek = myidx.tag_seek[tag];
3663
3664 if (TAGCACHE_IS_NUMERIC(tag))
3665 continue;
3666
3667 /**
3668 * Replace tag seek with a hash value of the field string data.
3669 * That way runtime statistics of moved or altered files can be
3670 * resurrected.
3671 */
3672 #ifdef HAVE_TC_RAMCACHE
3673 if (tc_stat.ramcache && tag != tag_filename)
3674 {
3675 struct tagfile_entry *tfe;
3676 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3677
3678 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3679 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3680 myidx.tag_seek[tag] = *seek;
3681 }
3682 else
3683 #endif
3684 {
3685 struct tagfile_entry tfe;
3686
3687 /* Open the index file, which contains the tag names. */
3688 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3689 goto cleanup;
3690
3691 /* Skip the header block */
3692 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3693 if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3694 {
3695 logf("delete_entry(): read error #3");
3696 goto cleanup;
3697 }
3698
3699 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3700 {
3701 logf("delete_entry(): read error #4");
3702 goto cleanup;
3703 }
3704
3705 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3706 }
3707
3708 if (in_use[tag])
3709 {
3710 logf("in use: %d/%d", tag, in_use[tag]);
3711 if (fd >= 0)
3712 {
3713 close(fd);
3714 fd = -1;
3715 }
3716 continue;
3717 }
3718
3719 #ifdef HAVE_TC_RAMCACHE
3720 /* Delete from ram. */
3721 if (tc_stat.ramcache && tag != tag_filename)
3722 {
3723 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3724 tagentry->tag_data[0] = '\0';
3725 }
3726 #endif
3727
3728 /* Open the index file, which contains the tag names. */
3729 if (fd < 0)
3730 {
3731 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3732 goto cleanup;
3733 }
3734
3735 /* Skip the header block */
3736 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3737
3738 /* Debug, print 10 first characters of the tag
3739 read(fd, buf, 10);
3740 buf[10]='\0';
3741 logf("TAG:%s", buf);
3742 lseek(fd, -10, SEEK_CUR);
3743 */
3744
3745 /* Write first data byte in tag as \0 */
3746 write(fd, "", 1);
3747
3748 /* Now tag data has been removed */
3749 close(fd);
3750 fd = -1;
3751 }
3752
3753 /* Write index entry back into master index. */
3754 lseek(masterfd, sizeof(struct master_header) +
3755 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3756 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3757 {
3758 logf("delete_entry(): write_error #2");
3759 goto cleanup;
3760 }
3761
3762 close(masterfd);
3763
3764 return true;
3765
3766 cleanup:
3767 if (fd >= 0)
3768 close(fd);
3769 if (masterfd >= 0)
3770 close(masterfd);
3771
3772 return false;
3773 }
3774
3775 #ifndef __PCTOOL__
3776 /**
3777 * Returns true if there is an event waiting in the queue
3778 * that requires the current operation to be aborted.
3779 */
3780 static bool check_event_queue(void)
3781 {
3782 struct queue_event ev;
3783
3784 if(!queue_peek(&tagcache_queue, &ev))
3785 return false;
3786
3787 switch (ev.id)
3788 {
3789 case Q_STOP_SCAN:
3790 case SYS_POWEROFF:
3791 case SYS_USB_CONNECTED:
3792 return true;
3793 }
3794
3795 return false;
3796 }
3797 #endif
3798
3799 #ifdef HAVE_TC_RAMCACHE
3800 static bool allocate_tagcache(void)
3801 {
3802 struct master_header tcmh;
3803 int fd;
3804
3805 /* Load the header. */
3806 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3807 {
3808 hdr = NULL;
3809 return false;
3810 }
3811
3812 close(fd);
3813
3814 /**
3815 * Now calculate the required cache size plus
3816 * some extra space for alignment fixes.
3817 */
3818 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3819 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3820 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3821 memset(hdr, 0, sizeof(struct ramcache_header));
3822 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3823 hdr->indices = (struct index_entry *)(hdr + 1);
3824 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3825
3826 return true;
3827 }
3828
3829 # ifdef HAVE_EEPROM_SETTINGS
3830 static bool tagcache_dumpload(void)
3831 {
3832 struct statefile_header shdr;
3833 int fd, rc;
3834 long offpos;
3835 int i;
3836
3837 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3838 if (fd < 0)
3839 {
3840 logf("no tagcache statedump");
3841 return false;
3842 }
3843
3844 /* Check the statefile memory placement */
3845 hdr = buffer_alloc(0);
3846 rc = read(fd, &shdr, sizeof(struct statefile_header));
3847 if (rc != sizeof(struct statefile_header)
3848 /* || (long)hdr != (long)shdr.hdr */)
3849 {
3850 logf("incorrect statefile");
3851 hdr = NULL;
3852 close(fd);
3853 return false;
3854 }
3855
3856 offpos = (long)hdr - (long)shdr.hdr;
3857
3858 /* Lets allocate real memory and load it */
3859 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3860 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3861 close(fd);
3862
3863 if (rc != shdr.tc_stat.ramcache_allocated)
3864 {
3865 logf("read failure!");
3866 hdr = NULL;
3867 return false;
3868 }
3869
3870 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3871
3872 /* Now fix the pointers */
3873 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3874 for (i = 0; i < TAG_COUNT; i++)
3875 hdr->tags[i] += offpos;
3876
3877 return true;
3878 }
3879
3880 static bool tagcache_dumpsave(void)
3881 {
3882 struct statefile_header shdr;
3883 int fd;
3884
3885 if (!tc_stat.ramcache)
3886 return false;
3887
3888 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3889 if (fd < 0)
3890 {
3891 logf("failed to create a statedump");
3892 return false;
3893 }
3894
3895 /* Create the header */
3896 shdr.hdr = hdr;
3897 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3898 write(fd, &shdr, sizeof(struct statefile_header));
3899
3900 /* And dump the data too */
3901 write(fd, hdr, tc_stat.ramcache_allocated);
3902 close(fd);
3903
3904 return true;
3905 }
3906 # endif
3907
3908 static bool load_tagcache(void)
3909 {
3910 struct tagcache_header *tch;
3911 long bytesleft = tc_stat.ramcache_allocated;
3912 struct index_entry *idx;
3913 int rc, fd;
3914 char *p;
3915 int i, tag;
3916
3917 # ifdef HAVE_DIRCACHE
3918 while (dircache_is_initializing())
3919 sleep(1);
3920
3921 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3922 # endif
3923
3924 logf("loading tagcache to ram...");
3925
3926 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3927 if (fd < 0)
3928 {
3929 logf("tagcache open failed");
3930 return false;
3931 }
3932
3933 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3934 != sizeof(struct master_header)
3935 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3936 {
3937 logf("incorrect header");
3938 return false;
3939 }
3940
3941 idx = hdr->indices;
3942
3943 /* Load the master index table. */
3944 for (i = 0; i < hdr->h.tch.entry_count; i++)
3945 {
3946 rc = ecread_index_entry(fd, idx);
3947 if (rc != sizeof(struct index_entry))
3948 {
3949 logf("read error #10");
3950 close(fd);
3951 return false;
3952 }
3953
3954 bytesleft -= sizeof(struct index_entry);
3955 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3956 {
3957 logf("too big tagcache.");
3958 close(fd);
3959 return false;
3960 }
3961
3962 idx++;
3963 }
3964
3965 close(fd);
3966
3967 /* Load the tags. */
3968 p = (char *)idx;
3969 for (tag = 0; tag < TAG_COUNT; tag++)
3970 {
3971 struct tagfile_entry *fe;
3972 char buf[TAG_MAXLEN+32];
3973
3974 if (TAGCACHE_IS_NUMERIC(tag))
3975 continue ;
3976
3977 //p = ((void *)p+1);
3978 p = (char *)((long)p & ~0x03) + 0x04;
3979 hdr->tags[tag] = p;
3980
3981 /* Check the header. */
3982 tch = (struct tagcache_header *)p;
3983 p += sizeof(struct tagcache_header);
3984
3985 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3986 return false;
3987
3988 for (hdr->entry_count[tag] = 0;
3989 hdr->entry_count[tag] < tch->entry_count;
3990 hdr->entry_count[tag]++)
3991 {
3992 long pos;
3993
3994 if (do_timed_yield())
3995 {
3996 /* Abort if we got a critical event in queue */
3997 if (check_event_queue())
3998 return false;
3999 }
4000
4001 fe = (struct tagfile_entry *)p;
4002 pos = lseek(fd, 0, SEEK_CUR);
4003 rc = ecread_tagfile_entry(fd, fe);
4004 if (rc != sizeof(struct tagfile_entry))
4005 {
4006 /* End of lookup table. */
4007 logf("read error #11");
4008 close(fd);
4009 return false;
4010 }
4011
4012 /* We have a special handling for the filename tags. */
4013 if (tag == tag_filename)
4014 {
4015 # ifdef HAVE_DIRCACHE
4016 const struct dircache_entry *dc;
4017 # endif
4018
4019 // FIXME: This is wrong!
4020 // idx = &hdr->indices[hdr->entry_count[i]];
4021 idx = &hdr->indices[fe->idx_id];
4022
4023 if (fe->tag_length >= (long)sizeof(buf)-1)
4024 {
4025 read(fd, buf, 10);
4026 buf[10] = '\0';
4027 logf("TAG:%s", buf);
4028 logf("too long filename");
4029 close(fd);
4030 return false;
4031 }
4032
4033 rc = read(fd, buf, fe->tag_length);
4034 if (rc != fe->tag_length)
4035 {
4036 logf("read error #12");
4037 close(fd);
4038 return false;
4039 }
4040
4041 /* Check if the entry has already been removed */
4042 if (idx->flag & FLAG_DELETED)
4043 continue;
4044
4045 /* This flag must not be used yet. */
4046 if (idx->flag & FLAG_DIRCACHE)
4047 {
4048 logf("internal error!");
4049 close(fd);
4050 return false;
4051 }
4052
4053 if (idx->tag_seek[tag] != pos)
4054 {
4055 logf("corrupt data structures!");
4056 close(fd);
4057 return false;
4058 }
4059
4060 # ifdef HAVE_DIRCACHE
4061 if (dircache_is_enabled())
4062 {
4063 dc = dircache_get_entry_ptr(buf);
4064 if (dc == NULL)
4065 {
4066 logf("Entry no longer valid.");
4067 logf("-> %s", buf);
4068 if (global_settings.tagcache_autoupdate)
4069 delete_entry(fe->idx_id);
4070 continue ;
4071 }
4072
4073 idx->flag |= FLAG_DIRCACHE;
4074 idx->tag_seek[tag_filename] = (long)dc;
4075 }
4076 else
4077 # endif
4078 {
4079 /* This will be very slow unless dircache is enabled
4080 or target is flash based, but do it anyway for
4081 consistency. */
4082 /* Check if entry has been removed. */
4083 if (global_settings.tagcache_autoupdate)
4084 {
4085 if (!file_exists(buf))
4086 {
4087 logf("Entry no longer valid.");
4088 logf("-> %s", buf);
4089 delete_entry(fe->idx_id);
4090 continue;
4091 }
4092 }
4093 }
4094
4095 continue ;
4096 }
4097
4098 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4099 if (bytesleft < 0)
4100 {
4101 logf("too big tagcache #2");
4102 logf("tl: %ld", fe->tag_length);
4103 logf("bl: %ld", bytesleft);
4104 close(fd);
4105 return false;
4106 }
4107
4108 p = fe->tag_data;
4109 rc = read(fd, fe->tag_data, fe->tag_length);
4110 p += rc;
4111
4112 if (rc != fe->tag_length)
4113 {
4114 logf("read error #13");
4115 logf("rc=0x%04x", rc); // 0x431
4116 logf("len=0x%04lx", fe->tag_length); // 0x4000
4117 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4118 logf("tag=0x%02x", tag); // 0x00
4119 close(fd);
4120 return false;
4121 }
4122 }
4123 close(fd);
4124 }
4125
4126 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4127 logf("tagcache loaded into ram!");
4128
4129 return true;
4130 }
4131 #endif /* HAVE_TC_RAMCACHE */
4132
4133 static bool check_deleted_files(void)
4134 {
4135 int fd;
4136 char buf[TAG_MAXLEN+32];
4137 struct tagfile_entry tfe;
4138
4139 logf("reverse scan...");
4140 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4141 fd = open(buf, O_RDONLY);
4142
4143 if (fd < 0)
4144 {
4145 logf("%s open fail", buf);
4146 return false;
4147 }
4148
4149 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4150 while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4151 #ifndef __PCTOOL__
4152 && !check_event_queue()
4153 #endif
4154 )
4155 {
4156 if (tfe.tag_length >= (long)sizeof(buf)-1)
4157 {
4158 logf("too long tag");
4159 close(fd);
4160 return false;
4161 }
4162
4163 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4164 {
4165 logf("read error #14");
4166 close(fd);
4167 return false;
4168 }
4169
4170 /* Check if the file has already deleted from the db. */
4171 if (*buf == '\0')
4172 continue;
4173
4174 /* Now check if the file exists. */
4175 if (!file_exists(buf))
4176 {
4177 logf("Entry no longer valid.");
4178 logf("-> %s / %ld", buf, tfe.tag_length);
4179 delete_entry(tfe.idx_id);
4180 }
4181 }
4182
4183 close(fd);
4184
4185 logf("done");
4186
4187 return true;
4188 }
4189
4190
4191 /* Note that this function must not be inlined, otherwise the whole point
4192 * of having the code in a separate function is lost.
4193 */
4194 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4195 int *ignore, int *unignore)
4196 {
4197 char newpath[MAX_PATH];
4198
4199 /* check for a database.ignore file */
4200 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4201 *ignore = file_exists(newpath);
4202 /* check for a database.unignore file */
4203 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4204 *unignore = file_exists(newpath);
4205 }
4206
4207 static struct search_roots_ll {
4208 const char *path;
4209 struct search_roots_ll * next;
4210 } roots_ll;
4211
4212 #ifdef APPLICATION
4213 /*
4214 * This adds a path to the search roots, possibly during traveling through
4215 * the filesystem. It only adds if the path is not inside an already existing
4216 * search root.
4217 *
4218 * Returns true if it added the path to the search roots
4219 *
4220 * Windows 2000 and greater supports symlinks, but they don't provide
4221 * realpath() or readlink(), and symlinks are rarely used on them so
4222 * ignore this for windows for now
4223 **/
4224 static bool add_search_root(const char *name)
4225 {
4226 (void)name;
4227 #ifndef WIN32
4228 struct search_roots_ll *this, *prev = NULL;
4229 char target[MAX_PATH];
4230 /* Okay, realpath() is almost completely broken on android
4231 *
4232 * It doesn't accept NULL for resolved_name to dynamically allocate
4233 * the resulting path; and it assumes resolved_name to be PATH_MAX
4234 * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4235 * and blindly writes to the end if it
4236 *
4237 * therefore use sufficiently large static storage here
4238 * Note that PATH_MAX != MAX_PATH
4239 **/
4240 static char abs_target[PATH_MAX];
4241 ssize_t len;
4242
4243 len = readlink(name, target, sizeof(target));
4244 if (len < 0)
4245 return false;
4246
4247 target[len] = '\0';
4248 if (realpath(target, abs_target) == NULL)
4249 return false;
4250
4251 for(this = &roots_ll; this; prev = this, this = this->next)
4252 {
4253 size_t root_len = strlen(this->path);
4254 /* check if the link target is inside of an existing search root
4255 * don't add if target is inside, we'll scan it later */
4256 if (!strncmp(this->path, abs_target, root_len))
4257 return false;
4258 }
4259
4260 if (prev)
4261 {
4262 size_t len = strlen(abs_target) + 1; /* count \0 */
4263 this = malloc(sizeof(struct search_roots_ll) + len );
4264 if (!this || len > MAX_PATH)
4265 {
4266 logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4267 free(this);
4268 prev->next = NULL;
4269 return false;
4270 }
4271 this->path = ((char*)this) + sizeof(struct search_roots_ll);
4272 strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4273 this->next = NULL;
4274 prev->next = this;
4275 logf("Added %s to the search roots\n", abs_target);
4276 return true;
4277 }
4278 #endif
4279 return false;
4280 }
4281
4282 static int free_search_roots(struct search_roots_ll * start)
4283 {
4284 int ret = 0;
4285 if (start->next)
4286 {
4287 ret += free_search_roots(start->next);
4288 ret += sizeof(struct search_roots_ll);
4289 free(start->next);
4290 }
4291 return ret;
4292 }
4293 #else /* native, simulator */
4294 #define add_search_root(a) do {} while(0)
4295 #define free_search_roots(a) do {} while(0)
4296 #endif
4297
4298 static bool check_dir(const char *dirname, int add_files)
4299 {
4300 DIR *dir;
4301 int len;
4302 int success = false;
4303 int ignore, unignore;
4304
4305 dir = opendir(dirname);
4306 if (!dir)
4307 {
4308 logf("tagcache: opendir(%s) failed", dirname);
4309 return false;
4310 }
4311 /* check for a database.ignore and database.unignore */
4312 check_ignore(dirname, &ignore, &unignore);
4313
4314 /* don't do anything if both ignore and unignore are there */
4315 if (ignore != unignore)
4316 add_files = unignore;
4317
4318 /* Recursively scan the dir. */
4319 #ifdef __PCTOOL__
4320 while (1)
4321 #else
4322 while (!check_event_queue())
4323 #endif
4324 {
4325 struct dirent *entry = readdir(dir);
4326 if (entry == NULL)
4327 {
4328 success = true;
4329 break;
4330 }
4331
4332 if (!strcmp((char *)entry->d_name, ".") ||
4333 !strcmp((char *)entry->d_name, ".."))
4334 continue;
4335
4336 struct dirinfo info = dir_get_info(dir, entry);
4337
4338 yield();
4339
4340 len = strlen(curpath);
4341 /* don't add an extra / for curpath == / */
4342 if (len <= 1) len = 0;
4343 snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4344
4345 processed_dir_count++;
4346 if (info.attribute & ATTR_DIRECTORY)
4347 { /* don't follow symlinks to dirs, but try to add it as a search root
4348 * this makes able to avoid looping in recursive symlinks */
4349 if (info.attribute & ATTR_LINK)
4350 add_search_root(curpath);
4351 else
4352 check_dir(curpath, add_files);
4353 }
4354 else if (add_files)
4355 {
4356 tc_stat.curentry = curpath;
4357
4358 /* Add a new entry to the temporary db file. */
4359 add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4360 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4361 , dir->internal_entry
4362 #endif
4363 );
4364
4365 /* Wait until current path for debug screen is read and unset. */
4366 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4367 yield();
4368
4369 tc_stat.curentry = NULL;
4370 }
4371
4372 curpath[len] = '\0';
4373 }
4374
4375 closedir(dir);
4376
4377 return success;
4378 }
4379
4380 void tagcache_screensync_event(void)
4381 {
4382 tc_stat.curentry = NULL;
4383 }
4384
4385 void tagcache_screensync_enable(bool state)
4386 {
4387 tc_stat.syncscreen = state;
4388 }
4389
4390 void tagcache_build(const char *path)
4391 {
4392 struct tagcache_header header;
4393 bool ret;
4394
4395 curpath[0] = '\0';
4396 data_size = 0;
4397 total_entry_count = 0;
4398 processed_dir_count = 0;
4399
4400 #ifdef HAVE_DIRCACHE
4401 while (dircache_is_initializing())
4402 sleep(1);
4403 #endif
4404
4405 logf("updating tagcache");
4406
4407 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4408 if (cachefd >= 0)
4409 {
4410 logf("skipping, cache already waiting for commit");
4411 close(cachefd);
4412 return ;
4413 }
4414
4415 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4416 if (cachefd < 0)
4417 {
4418 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4419 return ;
4420 }
4421
4422 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4423
4424 cpu_boost(true);
4425
4426 logf("Scanning files...");
4427 /* Scan for new files. */
4428 memset(&header, 0, sizeof(struct tagcache_header));
4429 write(cachefd, &header, sizeof(struct tagcache_header));
4430
4431 ret = true;
4432 roots_ll.path = path;
4433 roots_ll.next = NULL;
4434 struct search_roots_ll * this;
4435 /* check_dir might add new roots */
4436 for(this = &roots_ll; this; this = this->next)
4437 {
4438 strcpy(curpath, this->path);
4439 ret = ret && check_dir(this->path, true);
4440 }
4441 if (roots_ll.next)
4442 free_search_roots(roots_ll.next);
4443
4444 /* Write the header. */
4445 header.magic = TAGCACHE_MAGIC;
4446 header.datasize = data_size;
4447 header.entry_count = total_entry_count;
4448 lseek(cachefd, 0, SEEK_SET);
4449 write(cachefd, &header, sizeof(struct tagcache_header));
4450 close(cachefd);
4451
4452 if (filenametag_fd >= 0)
4453 {
4454 close(filenametag_fd);
4455 filenametag_fd = -1;
4456 }
4457
4458 if (!ret)
4459 {
4460 logf("Aborted.");
4461 cpu_boost(false);
4462 return ;
4463 }
4464
4465 /* Commit changes to the database. */
4466 #ifdef __PCTOOL__
4467 allocate_tempbuf();
4468 #endif
4469 if (commit())
4470 {
4471 remove(TAGCACHE_FILE_TEMP);
4472 logf("tagcache built!");
4473 }
4474 #ifdef __PCTOOL__
4475 free_tempbuf();
4476 #endif
4477
4478 #ifdef HAVE_TC_RAMCACHE
4479 if (hdr)
4480 {
4481 /* Import runtime statistics if we just initialized the db. */
4482 if (hdr->h.serial == 0)
4483 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4484 }
4485 #endif
4486
4487 cpu_boost(false);
4488 }
4489
4490 #ifdef HAVE_TC_RAMCACHE
4491 static void load_ramcache(void)
4492 {
4493 if (!hdr)
4494 return ;
4495
4496 cpu_boost(true);
4497
4498 /* At first we should load the cache (if exists). */
4499 tc_stat.ramcache = load_tagcache();
4500
4501 if (!tc_stat.ramcache)
4502 {
4503 /* If loading failed, it must indicate some problem with the db
4504 * so disable it entirely to prevent further issues. */
4505 tc_stat.ready = false;
4506 hdr = NULL;
4507 }
4508
4509 cpu_boost(false);
4510 }
4511
4512 void tagcache_unload_ramcache(void)
4513 {
4514 tc_stat.ramcache = false;
4515 /* Just to make sure there is no statefile present. */
4516 // remove(TAGCACHE_STATEFILE);
4517 }
4518 #endif
4519
4520 #ifndef __PCTOOL__
4521 static void tagcache_thread(void)
4522 {
4523 struct queue_event ev;
4524 bool check_done = false;
4525
4526 /* If the previous cache build/update was interrupted, commit
4527 * the changes first in foreground. */
4528 cpu_boost(true);
4529 allocate_tempbuf();
4530 commit();
4531 free_tempbuf();
4532
4533 #ifdef HAVE_TC_RAMCACHE
4534 # ifdef HAVE_EEPROM_SETTINGS
4535 if (firmware_settings.initialized && firmware_settings.disk_clean
4536 && global_settings.tagcache_ram)
4537 {
4538 check_done = tagcache_dumpload();
4539 }
4540
4541 remove(TAGCACHE_STATEFILE);
4542 # endif
4543
4544 /* Allocate space for the tagcache if found on disk. */
4545 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4546 allocate_tagcache();
4547 #endif
4548
4549 cpu_boost(false);
4550 tc_stat.initialized = true;
4551
4552 /* Don't delay bootup with the header check but do it on background. */
4553 if (!tc_stat.ready)
4554 {
4555 sleep(HZ);
4556 tc_stat.ready = check_all_headers();
4557 tc_stat.readyvalid = true;
4558 }
4559
4560 while (1)
4561 {
4562 run_command_queue(false);
4563
4564 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4565
4566 switch (ev.id)
4567 {
4568 case Q_IMPORT_CHANGELOG:
4569 tagcache_import_changelog();
4570 break;
4571
4572 case Q_REBUILD:
4573 remove_files();
4574 remove(TAGCACHE_FILE_TEMP);
4575 tagcache_build("/");
4576 break;
4577
4578 case Q_UPDATE:
4579 tagcache_build("/");
4580 #ifdef HAVE_TC_RAMCACHE
4581 load_ramcache();
4582 #endif
4583 check_deleted_files();
4584 break ;
4585
4586 case Q_START_SCAN:
4587 check_done = false;
4588 case SYS_TIMEOUT:
4589 if (check_done || !tc_stat.ready)
4590 break ;
4591
4592 #ifdef HAVE_TC_RAMCACHE
4593 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4594 {
4595 load_ramcache();
4596 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4597 tagcache_build("/");
4598 }
4599 else
4600 #endif
4601 if (global_settings.tagcache_autoupdate)
4602 {
4603 tagcache_build("/");
4604
4605 /* This will be very slow unless dircache is enabled
4606 or target is flash based, but do it anyway for
4607 consistency. */
4608 check_deleted_files();
4609 }
4610
4611 logf("tagcache check done");
4612
4613 check_done = true;
4614 break ;
4615
4616 case Q_STOP_SCAN:
4617 break ;
4618
4619 case SYS_POWEROFF:
4620 break ;
4621
4622 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4623 case SYS_USB_CONNECTED:
4624 logf("USB: TagCache");
4625 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4626 usb_wait_for_disconnect(&tagcache_queue);
4627 break ;
4628 #endif
4629 }
4630 }
4631 }
4632
4633 bool tagcache_prepare_shutdown(void)
4634 {
4635 if (tagcache_get_commit_step() > 0)
4636 return false;
4637
4638 tagcache_stop_scan();
4639 while (read_lock || write_lock)
4640 sleep(1);
4641
4642 return true;
4643 }
4644
4645 void tagcache_shutdown(void)
4646 {
4647 /* Flush the command queue. */
4648 run_command_queue(true);
4649
4650 #ifdef HAVE_EEPROM_SETTINGS
4651 if (tc_stat.ramcache)
4652 tagcache_dumpsave();
4653 #endif
4654 }
4655
4656 static int get_progress(void)
4657 {
4658 int total_count = -1;
4659
4660 #ifdef HAVE_DIRCACHE
4661 if (dircache_is_enabled())
4662 {
4663 total_count = dircache_get_entry_count();
4664 }
4665 else
4666 #endif
4667 #ifdef HAVE_TC_RAMCACHE
4668 {
4669 if (hdr && tc_stat.ramcache)
4670 total_count = hdr->h.tch.entry_count;
4671 }
4672 #endif
4673
4674 if (total_count < 0)
4675 return -1;
4676
4677 return processed_dir_count * 100 / total_count;
4678 }
4679
4680 struct tagcache_stat* tagcache_get_stat(void)
4681 {
4682 tc_stat.progress = get_progress();
4683 tc_stat.processed_entries = processed_dir_count;
4684
4685 return &tc_stat;
4686 }
4687
4688 void tagcache_start_scan(void)
4689 {
4690 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4691 }
4692
4693 bool tagcache_update(void)
4694 {
4695 if (!tc_stat.ready)
4696 return false;
4697
4698 queue_post(&tagcache_queue, Q_UPDATE, 0);
4699 return false;
4700 }
4701
4702 bool tagcache_rebuild()
4703 {
4704 queue_post(&tagcache_queue, Q_REBUILD, 0);
4705 return false;
4706 }
4707
4708 void tagcache_stop_scan(void)
4709 {
4710 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4711 }
4712
4713 #ifdef HAVE_TC_RAMCACHE
4714 bool tagcache_is_ramcache(void)
4715 {
4716 return tc_stat.ramcache;
4717 }
4718 #endif
4719
4720 #endif /* !__PCTOOL__ */
4721
4722
4723 void tagcache_init(void)
4724 {
4725 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4726 memset(&current_tcmh, 0, sizeof(struct master_header));
4727 filenametag_fd = -1;
4728 write_lock = read_lock = 0;
4729
4730 #ifndef __PCTOOL__
4731 mutex_init(&command_queue_mutex);
4732 queue_init(&tagcache_queue, true);
4733 create_thread(tagcache_thread, tagcache_stack,
4734 sizeof(tagcache_stack), 0, tagcache_thread_name
4735 IF_PRIO(, PRIORITY_BACKGROUND)
4736 IF_COP(, CPU));
4737 #else
4738 tc_stat.initialized = true;
4739 allocate_tempbuf();
4740 commit();
4741 free_tempbuf();
4742 tc_stat.ready = check_all_headers();
4743 #endif
4744 }
4745
4746 #ifdef __PCTOOL__
4747 void tagcache_reverse_scan(void)
4748 {
4749 logf("Checking for deleted files");
4750 check_deleted_files();
4751 }
4752 #endif
4753
4754 bool tagcache_is_initialized(void)
4755 {
4756 return tc_stat.initialized;
4757 }
4758 bool tagcache_is_fully_initialized(void)
4759 {
4760 return tc_stat.readyvalid;
4761 }
4762 bool tagcache_is_usable(void)
4763 {
4764 return tc_stat.initialized && tc_stat.ready;
4765 }
4766 int tagcache_get_commit_step(void)
4767 {
4768 return tc_stat.commit_step;
4769 }
4770 int tagcache_get_max_commit_step(void)
4771 {
4772 return (int)(SORTED_TAGS_COUNT)+1;
4773 }
4774
My public Rockbox git repo