?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
GSoC/Buflib: Add buflib memory alocator to the core.
[kugel-rb.git] / apps / tagcache.c
blob1adfc759e6a2d3949e82bfe26880ada03edb9ec8
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /*#define LOGF_ENABLE*/
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #ifdef APPLICATION
64 #include <unistd.h> /* readlink() */
65 #include <limits.h> /* PATH_MAX */
66 #endif
67 #include "config.h"
68 #include "ata_idle_notify.h"
69 #include "thread.h"
70 #include "kernel.h"
71 #include "system.h"
72 #include "logf.h"
73 #include "string-extra.h"
74 #include "usb.h"
75 #include "metadata.h"
76 #include "tagcache.h"
77 #include "core_alloc.h"
78 #include "crc32.h"
79 #include "misc.h"
80 #include "settings.h"
81 #include "dir.h"
82 #include "filefuncs.h"
83 #include "structec.h"
84 #include "debug.h"
85
86 #ifndef __PCTOOL__
87 #include "lang.h"
88 #include "eeprom_settings.h"
89 #endif
90
91 #ifdef __PCTOOL__
92 #define yield() do { } while(0)
93 #define sim_sleep(timeout) do { } while(0)
94 #define do_timed_yield() do { } while(0)
95 #endif
96
97 #ifndef __PCTOOL__
98 /* Tag Cache thread. */
99 static struct event_queue tagcache_queue SHAREDBSS_ATTR;
100 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
101 static const char tagcache_thread_name[] = "tagcache";
102 #endif
103
104 /* Previous path when scanning directory tree recursively. */
105 static char curpath[TAG_MAXLEN+32];
106
107 /* Used when removing duplicates. */
108 static char *tempbuf; /* Allocated when needed. */
109 static long tempbufidx; /* Current location in buffer. */
110 static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
111 static long tempbuf_left; /* Buffer space left. */
112 static long tempbuf_pos;
113
114 #define SORTED_TAGS_COUNT 8
115 #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS)
116 #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS)
117 #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \
118 (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS))
119 /* Tags we want to get sorted (loaded to the tempbuf). */
120 #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
121 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
122 (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title))
123
124 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
125 #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \
126 (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \
127 (1LU << tag_albumartist) | (1LU << tag_grouping))
128
129 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
130 static const char *tags_str[] = { "artist", "album", "genre", "title",
131 "filename", "composer", "comment", "albumartist", "grouping", "year",
132 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
133 "playtime", "lastplayed", "commitid", "mtime", "lastoffset" };
134
135 /* Status information of the tagcache. */
136 static struct tagcache_stat tc_stat;
137
138 /* Queue commands. */
139 enum tagcache_queue {
140 Q_STOP_SCAN = 0,
141 Q_START_SCAN,
142 Q_IMPORT_CHANGELOG,
143 Q_UPDATE,
144 Q_REBUILD,
145
146 /* Internal tagcache command queue. */
147 CMD_UPDATE_MASTER_HEADER,
148 CMD_UPDATE_NUMERIC,
149 };
150
151 struct tagcache_command_entry {
152 int32_t command;
153 int32_t idx_id;
154 int32_t tag;
155 int32_t data;
156 };
157
158 #ifndef __PCTOOL__
159 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
160 static volatile int command_queue_widx = 0;
161 static volatile int command_queue_ridx = 0;
162 static struct mutex command_queue_mutex SHAREDBSS_ATTR;
163 #endif
164
165 /* Tag database structures. */
166
167 /* Variable-length tag entry in tag files. */
168 struct tagfile_entry {
169 int32_t tag_length; /* Length of the data in bytes including '\0' */
170 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
171 char tag_data[0]; /* Begin of the tag data */
172 };
173
174 /* Fixed-size tag entry in master db index. */
175 struct index_entry {
176 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
177 int32_t flag; /* Status flags */
178 };
179
180 /* Header is the same in every file. */
181 struct tagcache_header {
182 int32_t magic; /* Header version number */
183 int32_t datasize; /* Data size in bytes */
184 int32_t entry_count; /* Number of entries in this file */
185 };
186
187 struct master_header {
188 struct tagcache_header tch;
189 int32_t serial; /* Increasing counting number */
190 int32_t commitid; /* Number of commits so far */
191 int32_t dirty;
192 };
193
194 /* For the endianess correction */
195 static const char *tagfile_entry_ec = "ll";
196 /**
197 Note: This should be (1 + TAG_COUNT) amount of l's.
198 */
199 static const char *index_entry_ec = "llllllllllllllllllllll";
200
201 static const char *tagcache_header_ec = "lll";
202 static const char *master_header_ec = "llllll";
203
204 static struct master_header current_tcmh;
205
206 #ifdef HAVE_TC_RAMCACHE
207 /* Header is created when loading database to ram. */
208 struct ramcache_header {
209 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
210 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
211 struct index_entry indices[0]; /* Master index file content */
212 };
213
214 # ifdef HAVE_EEPROM_SETTINGS
215 struct statefile_header {
216 int32_t magic; /* Statefile version number */
217 struct master_header mh; /* Header from the master index */
218 struct ramcache_header *hdr; /* Old load address of hdr for relocation */
219 struct tagcache_stat tc_stat;
220 };
221 # endif
222
223 /* Pointer to allocated ramcache_header */
224 static struct ramcache_header *ramcache_hdr;
225 #endif
226
227 /**
228 * Full tag entries stored in a temporary file waiting
229 * for commit to the cache. */
230 struct temp_file_entry {
231 long tag_offset[TAG_COUNT];
232 short tag_length[TAG_COUNT];
233 long flag;
234
235 long data_length;
236 };
237
238 struct tempbuf_id_list {
239 long id;
240 struct tempbuf_id_list *next;
241 };
242
243 struct tempbuf_searchidx {
244 long idx_id;
245 char *str;
246 int seek;
247 struct tempbuf_id_list idlist;
248 };
249
250 /* Lookup buffer for fixing messed up index while after sorting. */
251 static long commit_entry_count;
252 static long lookup_buffer_depth;
253 static struct tempbuf_searchidx **lookup;
254
255 /* Used when building the temporary file. */
256 static int cachefd = -1, filenametag_fd;
257 static int total_entry_count = 0;
258 static int data_size = 0;
259 static int processed_dir_count;
260
261 /* Thread safe locking */
262 static volatile int write_lock;
263 static volatile int read_lock;
264
265 static bool delete_entry(long idx_id);
266
267 const char* tagcache_tag_to_str(int tag)
268 {
269 return tags_str[tag];
270 }
271
272 /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */
273 static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf)
274 {
275 return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ);
276 }
277
278 static ssize_t ecread_index_entry(int fd, struct index_entry *buf)
279 {
280 return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ);
281 }
282
283 static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf)
284 {
285 return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ);
286 }
287
288 #ifdef HAVE_DIRCACHE
289 /**
290 * Returns true if specified flag is still present, i.e., dircache
291 * has not been reloaded.
292 */
293 static bool is_dircache_intact(void)
294 {
295 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
296 }
297 #endif
298
299 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
300 {
301 int fd;
302 char buf[MAX_PATH];
303 int rc;
304
305 if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT)
306 return -1;
307
308 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
309
310 fd = open(buf, write ? O_RDWR : O_RDONLY);
311 if (fd < 0)
312 {
313 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
314 tc_stat.ready = false;
315 return fd;
316 }
317
318 /* Check the header. */
319 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
320 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
321 {
322 logf("header error");
323 tc_stat.ready = false;
324 close(fd);
325 return -2;
326 }
327
328 return fd;
329 }
330
331 static int open_master_fd(struct master_header *hdr, bool write)
332 {
333 int fd;
334 int rc;
335
336 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
337 if (fd < 0)
338 {
339 logf("master file open failed for R/W");
340 tc_stat.ready = false;
341 return fd;
342 }
343
344 tc_stat.econ = false;
345
346 /* Check the header. */
347 rc = read(fd, hdr, sizeof(struct master_header));
348 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
349 {
350 /* Success. */
351 return fd;
352 }
353
354 /* Trying to read again, this time with endianess correction enabled. */
355 lseek(fd, 0, SEEK_SET);
356
357 rc = ecread(fd, hdr, 1, master_header_ec, true);
358 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
359 {
360 logf("header error");
361 tc_stat.ready = false;
362 close(fd);
363 return -2;
364 }
365
366 tc_stat.econ = true;
367
368 return fd;
369 }
370
371 #ifndef __PCTOOL__
372 static bool do_timed_yield(void)
373 {
374 /* Sorting can lock up for quite a while, so yield occasionally */
375 static long wakeup_tick = 0;
376 if (TIME_AFTER(current_tick, wakeup_tick))
377 {
378 wakeup_tick = current_tick + (HZ/4);
379 yield();
380 return true;
381 }
382 return false;
383 }
384 #endif
385
386 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
387 /* find the ramcache entry corresponding to the file indicated by
388 * filename and dc (it's corresponding dircache id). */
389 static long find_entry_ram(const char *filename, int dc)
390 {
391 static long last_pos = 0;
392 int i;
393
394 /* Check if tagcache is loaded into ram. */
395 if (!tc_stat.ramcache)
396 return -1;
397
398 if (dc < 0)
399 dc = dircache_get_entry_id(filename);
400
401 if (dc < 0)
402 {
403 logf("tagcache: file not found.");
404 return -1;
405 }
406
407 try_again:
408
409 if (last_pos > 0)
410 i = last_pos;
411 else
412 i = 0;
413
414 for (; i < current_tcmh.tch.entry_count; i++)
415 {
416 if (ramcache_hdr->indices[i].tag_seek[tag_filename] == dc)
417 {
418 last_pos = MAX(0, i - 3);
419 return i;
420 }
421
422 do_timed_yield();
423 }
424
425 if (last_pos > 0)
426 {
427 last_pos = 0;
428 goto try_again;
429 }
430
431 return -1;
432 }
433 #endif
434
435 static long find_entry_disk(const char *filename_raw, bool localfd)
436 {
437 struct tagcache_header tch;
438 static long last_pos = -1;
439 long pos_history[POS_HISTORY_COUNT];
440 long pos_history_idx = 0;
441 bool found = false;
442 struct tagfile_entry tfe;
443 int fd;
444 char buf[TAG_MAXLEN+32];
445 int i;
446 int pos = -1;
447
448 const char *filename = filename_raw;
449 #ifdef APPLICATION
450 char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */
451 if (realpath(filename, pathbuf) == pathbuf)
452 filename = pathbuf;
453 #endif
454
455 if (!tc_stat.ready)
456 return -2;
457
458 fd = filenametag_fd;
459 if (fd < 0 || localfd)
460 {
461 last_pos = -1;
462 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
463 return -1;
464 }
465
466 check_again:
467
468 if (last_pos > 0)
469 lseek(fd, last_pos, SEEK_SET);
470 else
471 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
472
473 while (true)
474 {
475 pos = lseek(fd, 0, SEEK_CUR);
476 for (i = pos_history_idx-1; i >= 0; i--)
477 pos_history[i+1] = pos_history[i];
478 pos_history[0] = pos;
479
480 if (ecread_tagfile_entry(fd, &tfe)
481 != sizeof(struct tagfile_entry))
482 {
483 break ;
484 }
485
486 if (tfe.tag_length >= (long)sizeof(buf))
487 {
488 logf("too long tag #1");
489 close(fd);
490 if (!localfd)
491 filenametag_fd = -1;
492 last_pos = -1;
493 return -2;
494 }
495
496 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
497 {
498 logf("read error #2");
499 close(fd);
500 if (!localfd)
501 filenametag_fd = -1;
502 last_pos = -1;
503 return -3;
504 }
505
506 if (!strcmp(filename, buf))
507 {
508 last_pos = pos_history[pos_history_idx];
509 found = true;
510 break ;
511 }
512
513 if (pos_history_idx < POS_HISTORY_COUNT - 1)
514 pos_history_idx++;
515 }
516
517 /* Not found? */
518 if (!found)
519 {
520 if (last_pos > 0)
521 {
522 last_pos = -1;
523 logf("seek again");
524 goto check_again;
525 }
526
527 if (fd != filenametag_fd || localfd)
528 close(fd);
529 return -4;
530 }
531
532 if (fd != filenametag_fd || localfd)
533 close(fd);
534
535 return tfe.idx_id;
536 }
537
538 static int find_index(const char *filename)
539 {
540 long idx_id = -1;
541
542 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
543 if (tc_stat.ramcache && is_dircache_intact())
544 idx_id = find_entry_ram(filename, -1);
545 #endif
546
547 if (idx_id < 0)
548 idx_id = find_entry_disk(filename, true);
549
550 return idx_id;
551 }
552
553 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
554 {
555 int idx_id;
556
557 if (!tc_stat.ready)
558 return false;
559
560 idx_id = find_index(filename);
561 if (idx_id < 0)
562 return false;
563
564 if (!tagcache_search(tcs, tag_filename))
565 return false;
566
567 tcs->entry_count = 0;
568 tcs->idx_id = idx_id;
569
570 return true;
571 }
572
573 static bool get_index(int masterfd, int idxid,
574 struct index_entry *idx, bool use_ram)
575 {
576 bool localfd = false;
577
578 if (idxid < 0)
579 {
580 logf("Incorrect idxid: %d", idxid);
581 return false;
582 }
583
584 #ifdef HAVE_TC_RAMCACHE
585 if (tc_stat.ramcache && use_ram)
586 {
587 if (ramcache_hdr->indices[idxid].flag & FLAG_DELETED)
588 return false;
589
590 # ifdef HAVE_DIRCACHE
591 if (!(ramcache_hdr->indices[idxid].flag & FLAG_DIRCACHE)
592 || is_dircache_intact())
593 #endif
594 {
595 memcpy(idx, &ramcache_hdr->indices[idxid], sizeof(struct index_entry));
596 return true;
597 }
598 }
599 #else
600 (void)use_ram;
601 #endif
602
603 if (masterfd < 0)
604 {
605 struct master_header tcmh;
606
607 localfd = true;
608 masterfd = open_master_fd(&tcmh, false);
609 if (masterfd < 0)
610 return false;
611 }
612
613 lseek(masterfd, idxid * sizeof(struct index_entry)
614 + sizeof(struct master_header), SEEK_SET);
615 if (ecread_index_entry(masterfd, idx)
616 != sizeof(struct index_entry))
617 {
618 logf("read error #3");
619 if (localfd)
620 close(masterfd);
621
622 return false;
623 }
624
625 if (localfd)
626 close(masterfd);
627
628 if (idx->flag & FLAG_DELETED)
629 return false;
630
631 return true;
632 }
633
634 #ifndef __PCTOOL__
635
636 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
637 {
638 /* We need to exclude all memory only flags & tags when writing to disk. */
639 if (idx->flag & FLAG_DIRCACHE)
640 {
641 logf("memory only flags!");
642 return false;
643 }
644
645 #ifdef HAVE_TC_RAMCACHE
646 /* Only update numeric data. Writing the whole index to RAM by memcpy
647 * destroys dircache pointers!
648 */
649 if (tc_stat.ramcache)
650 {
651 int tag;
652 struct index_entry *idx_ram = &ramcache_hdr->indices[idxid];
653
654 for (tag = 0; tag < TAG_COUNT; tag++)
655 {
656 if (TAGCACHE_IS_NUMERIC(tag))
657 {
658 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
659 }
660 }
661
662 /* Don't touch the dircache flag or attributes. */
663 idx_ram->flag = (idx->flag & 0x0000ffff)
664 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
665 }
666 #endif
667
668 lseek(masterfd, idxid * sizeof(struct index_entry)
669 + sizeof(struct master_header), SEEK_SET);
670 if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry))
671 {
672 logf("write error #3");
673 logf("idxid: %d", idxid);
674 return false;
675 }
676
677 return true;
678 }
679
680 #endif /* !__PCTOOL__ */
681
682 static bool open_files(struct tagcache_search *tcs, int tag)
683 {
684 if (tcs->idxfd[tag] < 0)
685 {
686 char fn[MAX_PATH];
687
688 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
689 tcs->idxfd[tag] = open(fn, O_RDONLY);
690 }
691
692 if (tcs->idxfd[tag] < 0)
693 {
694 logf("File not open!");
695 return false;
696 }
697
698 return true;
699 }
700
701 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
702 int tag, char *buf, long size)
703 {
704 struct tagfile_entry tfe;
705 long seek;
706
707 *buf = '\0';
708
709 if (TAGCACHE_IS_NUMERIC(tag))
710 return false;
711
712 seek = idx->tag_seek[tag];
713 if (seek < 0)
714 {
715 logf("Retrieve failed");
716 return false;
717 }
718
719 #ifdef HAVE_TC_RAMCACHE
720 if (tcs->ramsearch)
721 {
722 struct tagfile_entry *ep;
723
724 # ifdef HAVE_DIRCACHE
725 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
726 && is_dircache_intact())
727 {
728 /* for tag_filename, seek is a dircache index */
729 dircache_copy_path(seek, buf, size);
730 return true;
731 }
732 else
733 # endif
734 if (tag != tag_filename)
735 {
736 ep = (struct tagfile_entry *)&ramcache_hdr->tags[tag][seek];
737 strlcpy(buf, ep->tag_data, size);
738
739 return true;
740 }
741 }
742 #endif
743
744 if (!open_files(tcs, tag))
745 return false;
746
747 lseek(tcs->idxfd[tag], seek, SEEK_SET);
748 if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe)
749 != sizeof(struct tagfile_entry))
750 {
751 logf("read error #5");
752 return false;
753 }
754
755 if (tfe.tag_length >= size)
756 {
757 logf("too small buffer");
758 return false;
759 }
760
761 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
762 tfe.tag_length)
763 {
764 logf("read error #6");
765 return false;
766 }
767
768 buf[tfe.tag_length] = '\0';
769
770 return true;
771 }
772
773 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
774
775 static long find_tag(int tag, int idx_id, const struct index_entry *idx)
776 {
777 #ifndef __PCTOOL__
778 if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag))
779 {
780 /* Attempt to find tag data through store-to-load forwarding in
781 command queue */
782 long result = -1;
783
784 mutex_lock(&command_queue_mutex);
785
786 int ridx = command_queue_widx;
787
788 while (ridx != command_queue_ridx)
789 {
790 if (--ridx < 0)
791 ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1;
792
793 if (command_queue[ridx].command == CMD_UPDATE_NUMERIC
794 && command_queue[ridx].idx_id == idx_id
795 && command_queue[ridx].tag == tag)
796 {
797 result = command_queue[ridx].data;
798 break;
799 }
800 }
801
802 mutex_unlock(&command_queue_mutex);
803
804 if (result >= 0)
805 {
806 logf("find_tag: "
807 "Recovered tag %d value %lX from write queue",
808 tag, result);
809 return result;
810 }
811 }
812 #endif
813
814 return idx->tag_seek[tag];
815 }
816
817
818 static long check_virtual_tags(int tag, int idx_id,
819 const struct index_entry *idx)
820 {
821 long data = 0;
822
823 switch (tag)
824 {
825 case tag_virt_length_sec:
826 data = (find_tag(tag_length, idx_id, idx)/1000) % 60;
827 break;
828
829 case tag_virt_length_min:
830 data = (find_tag(tag_length, idx_id, idx)/1000) / 60;
831 break;
832
833 case tag_virt_playtime_sec:
834 data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60;
835 break;
836
837 case tag_virt_playtime_min:
838 data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60;
839 break;
840
841 case tag_virt_autoscore:
842 if (find_tag(tag_length, idx_id, idx) == 0
843 || find_tag(tag_playcount, idx_id, idx) == 0)
844 {
845 data = 0;
846 }
847 else
848 {
849 /* A straight calculus gives:
850 autoscore = 100 * playtime / length / playcout (1)
851 Now, consider the euclidian division of playtime by length:
852 playtime = alpha * length + beta
853 With:
854 0 <= beta < length
855 Now, (1) becomes:
856 autoscore = 100 * (alpha / playcout + beta / length / playcount)
857 Both terms should be small enough to avoid any overflow
858 */
859 data = 100 * (find_tag(tag_playtime, idx_id, idx)
860 / find_tag(tag_length, idx_id, idx))
861 + (100 * (find_tag(tag_playtime, idx_id, idx)
862 % find_tag(tag_length, idx_id, idx)))
863 / find_tag(tag_length, idx_id, idx);
864 data /= find_tag(tag_playcount, idx_id, idx);
865 }
866 break;
867
868 /* How many commits before the file has been added to the DB. */
869 case tag_virt_entryage:
870 data = current_tcmh.commitid
871 - find_tag(tag_commitid, idx_id, idx) - 1;
872 break;
873
874 case tag_virt_basename:
875 tag = tag_filename; /* return filename; caller handles basename */
876 /* FALLTHRU */
877
878 default:
879 data = find_tag(tag, idx_id, idx);
880 }
881
882 return data;
883 }
884
885 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
886 {
887 struct index_entry idx;
888
889 if (!tc_stat.ready)
890 return false;
891
892 if (!TAGCACHE_IS_NUMERIC(tag))
893 return -1;
894
895 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
896 return -2;
897
898 return check_virtual_tags(tag, tcs->idx_id, &idx);
899 }
900
901 inline static bool str_ends_with(const char *str1, const char *str2)
902 {
903 int str_len = strlen(str1);
904 int clause_len = strlen(str2);
905
906 if (clause_len > str_len)
907 return false;
908
909 return !strcasecmp(&str1[str_len - clause_len], str2);
910 }
911
912 inline static bool str_oneof(const char *str, const char *list)
913 {
914 const char *sep;
915 int l, len = strlen(str);
916
917 while (*list)
918 {
919 sep = strchr(list, '|');
920 l = sep ? (long)sep - (long)list : (int)strlen(list);
921 if ((l==len) && !strncasecmp(str, list, len))
922 return true;
923 list += sep ? l + 1 : l;
924 }
925
926 return false;
927 }
928
929 static bool check_against_clause(long numeric, const char *str,
930 const struct tagcache_search_clause *clause)
931 {
932 if (clause->numeric)
933 {
934 switch (clause->type)
935 {
936 case clause_is:
937 return numeric == clause->numeric_data;
938 case clause_is_not:
939 return numeric != clause->numeric_data;
940 case clause_gt:
941 return numeric > clause->numeric_data;
942 case clause_gteq:
943 return numeric >= clause->numeric_data;
944 case clause_lt:
945 return numeric < clause->numeric_data;
946 case clause_lteq:
947 return numeric <= clause->numeric_data;
948 default:
949 logf("Incorrect numeric tag: %d", clause->type);
950 }
951 }
952 else
953 {
954 switch (clause->type)
955 {
956 case clause_is:
957 return !strcasecmp(clause->str, str);
958 case clause_is_not:
959 return strcasecmp(clause->str, str);
960 case clause_gt:
961 return 0>strcasecmp(clause->str, str);
962 case clause_gteq:
963 return 0>=strcasecmp(clause->str, str);
964 case clause_lt:
965 return 0<strcasecmp(clause->str, str);
966 case clause_lteq:
967 return 0<=strcasecmp(clause->str, str);
968 case clause_contains:
969 return (strcasestr(str, clause->str) != NULL);
970 case clause_not_contains:
971 return (strcasestr(str, clause->str) == NULL);
972 case clause_begins_with:
973 return (strcasestr(str, clause->str) == str);
974 case clause_not_begins_with:
975 return (strcasestr(str, clause->str) != str);
976 case clause_ends_with:
977 return str_ends_with(str, clause->str);
978 case clause_not_ends_with:
979 return !str_ends_with(str, clause->str);
980 case clause_oneof:
981 return str_oneof(str, clause->str);
982
983 default:
984 logf("Incorrect tag: %d", clause->type);
985 }
986 }
987
988 return false;
989 }
990
991 static bool check_clauses(struct tagcache_search *tcs,
992 struct index_entry *idx,
993 struct tagcache_search_clause **clauses, int count)
994 {
995 int i;
996
997 /* Go through all conditional clauses. */
998 for (i = 0; i < count; i++)
999 {
1000 int seek;
1001 char buf[256];
1002 char *str = buf;
1003 struct tagcache_search_clause *clause = clauses[i];
1004
1005 if (clause->type == clause_logical_or)
1006 break; /* all conditions before logical-or satisfied --
1007 stop processing clauses */
1008
1009 seek = check_virtual_tags(clause->tag, tcs->idx_id, idx);
1010
1011 #ifdef HAVE_TC_RAMCACHE
1012 if (tcs->ramsearch)
1013 {
1014 struct tagfile_entry *tfe;
1015
1016 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1017 {
1018 if (clause->tag == tag_filename
1019 || clause->tag == tag_virt_basename)
1020 {
1021 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
1022 }
1023 else
1024 {
1025 tfe = (struct tagfile_entry *)
1026 &ramcache_hdr->tags[clause->tag][seek];
1027 str = tfe->tag_data;
1028 }
1029 }
1030 }
1031 else
1032 #endif
1033 {
1034 struct tagfile_entry tfe;
1035
1036 if (!TAGCACHE_IS_NUMERIC(clause->tag))
1037 {
1038 int tag = clause->tag;
1039 if (tag == tag_virt_basename)
1040 tag = tag_filename;
1041
1042 int fd = tcs->idxfd[tag];
1043 lseek(fd, seek, SEEK_SET);
1044 ecread_tagfile_entry(fd, &tfe);
1045 if (tfe.tag_length >= (int)sizeof(buf))
1046 {
1047 logf("Too long tag read!");
1048 return false;
1049 }
1050
1051 read(fd, str, tfe.tag_length);
1052 str[tfe.tag_length] = '\0';
1053
1054 /* Check if entry has been deleted. */
1055 if (str[0] == '\0')
1056 return false;
1057 }
1058 }
1059
1060 if (clause->tag == tag_virt_basename)
1061 {
1062 char *basename = strrchr(str, '/');
1063 if (basename)
1064 str = basename + 1;
1065 }
1066
1067 if (!check_against_clause(seek, str, clause))
1068 {
1069 /* Clause failed -- try finding a logical-or clause */
1070 while (++i < count)
1071 {
1072 if (clauses[i]->type == clause_logical_or)
1073 break;
1074 }
1075
1076 if (i < count) /* Found logical-or? */
1077 continue; /* Check clauses after logical-or */
1078
1079 return false;
1080 }
1081 }
1082
1083 return true;
1084 }
1085
1086 bool tagcache_check_clauses(struct tagcache_search *tcs,
1087 struct tagcache_search_clause **clause, int count)
1088 {
1089 struct index_entry idx;
1090
1091 if (count == 0)
1092 return true;
1093
1094 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1095 return false;
1096
1097 return check_clauses(tcs, &idx, clause, count);
1098 }
1099
1100 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1101 {
1102 int i;
1103
1104 /* If uniq buffer is not defined we must return true for search to work. */
1105 if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type)
1106 && !TAGCACHE_IS_NUMERIC(tcs->type)))
1107 {
1108 return true;
1109 }
1110
1111 for (i = 0; i < tcs->unique_list_count; i++)
1112 {
1113 /* Return false if entry is found. */
1114 if (tcs->unique_list[i] == id)
1115 return false;
1116 }
1117
1118 if (tcs->unique_list_count < tcs->unique_list_capacity)
1119 {
1120 tcs->unique_list[i] = id;
1121 tcs->unique_list_count++;
1122 }
1123
1124 return true;
1125 }
1126
1127 static bool build_lookup_list(struct tagcache_search *tcs)
1128 {
1129 struct index_entry entry;
1130 int i, j;
1131
1132 tcs->seek_list_count = 0;
1133
1134 #ifdef HAVE_TC_RAMCACHE
1135 if (tcs->ramsearch
1136 # ifdef HAVE_DIRCACHE
1137 && (tcs->type != tag_filename || is_dircache_intact())
1138 # endif
1139 )
1140 {
1141 for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++)
1142 {
1143 struct tagcache_seeklist_entry *seeklist;
1144 struct index_entry *idx = &ramcache_hdr->indices[i];
1145 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1146 break ;
1147
1148 /* Skip deleted files. */
1149 if (idx->flag & FLAG_DELETED)
1150 continue;
1151
1152 /* Go through all filters.. */
1153 for (j = 0; j < tcs->filter_count; j++)
1154 {
1155 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1156 {
1157 break ;
1158 }
1159 }
1160
1161 if (j < tcs->filter_count)
1162 continue ;
1163
1164 /* Check for conditions. */
1165 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1166 continue;
1167
1168 /* Add to the seek list if not already in uniq buffer. */
1169 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1170 continue;
1171
1172 /* Lets add it. */
1173 seeklist = &tcs->seeklist[tcs->seek_list_count];
1174 seeklist->seek = idx->tag_seek[tcs->type];
1175 seeklist->flag = idx->flag;
1176 seeklist->idx_id = i;
1177 tcs->seek_list_count++;
1178 }
1179
1180 tcs->seek_pos = i;
1181
1182 return tcs->seek_list_count > 0;
1183 }
1184 #endif
1185
1186 if (tcs->masterfd < 0)
1187 {
1188 struct master_header tcmh;
1189 tcs->masterfd = open_master_fd(&tcmh, false);
1190 }
1191
1192 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1193 sizeof(struct master_header), SEEK_SET);
1194
1195 while (ecread_index_entry(tcs->masterfd, &entry)
1196 == sizeof(struct index_entry))
1197 {
1198 struct tagcache_seeklist_entry *seeklist;
1199
1200 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1201 break ;
1202
1203 i = tcs->seek_pos;
1204 tcs->seek_pos++;
1205
1206 /* Check if entry has been deleted. */
1207 if (entry.flag & FLAG_DELETED)
1208 continue;
1209
1210 /* Go through all filters.. */
1211 for (j = 0; j < tcs->filter_count; j++)
1212 {
1213 if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1214 break ;
1215 }
1216
1217 if (j < tcs->filter_count)
1218 continue ;
1219
1220 /* Check for conditions. */
1221 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1222 continue;
1223
1224 /* Add to the seek list if not already in uniq buffer. */
1225 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1226 continue;
1227
1228 /* Lets add it. */
1229 seeklist = &tcs->seeklist[tcs->seek_list_count];
1230 seeklist->seek = entry.tag_seek[tcs->type];
1231 seeklist->flag = entry.flag;
1232 seeklist->idx_id = i;
1233 tcs->seek_list_count++;
1234
1235 yield();
1236 }
1237
1238 return tcs->seek_list_count > 0;
1239 }
1240
1241
1242 static void remove_files(void)
1243 {
1244 int i;
1245 char buf[MAX_PATH];
1246
1247 tc_stat.ready = false;
1248 tc_stat.ramcache = false;
1249 tc_stat.econ = false;
1250 remove(TAGCACHE_FILE_MASTER);
1251 for (i = 0; i < TAG_COUNT; i++)
1252 {
1253 if (TAGCACHE_IS_NUMERIC(i))
1254 continue;
1255
1256 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1257 remove(buf);
1258 }
1259 }
1260
1261
1262 static bool check_all_headers(void)
1263 {
1264 struct master_header myhdr;
1265 struct tagcache_header tch;
1266 int tag;
1267 int fd;
1268
1269 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1270 return false;
1271
1272 close(fd);
1273 if (myhdr.dirty)
1274 {
1275 logf("tagcache is dirty!");
1276 return false;
1277 }
1278
1279 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1280
1281 for (tag = 0; tag < TAG_COUNT; tag++)
1282 {
1283 if (TAGCACHE_IS_NUMERIC(tag))
1284 continue;
1285
1286 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1287 return false;
1288
1289 close(fd);
1290 }
1291
1292 return true;
1293 }
1294
1295 bool tagcache_is_busy(void)
1296 {
1297 return read_lock || write_lock;
1298 }
1299
1300 bool tagcache_search(struct tagcache_search *tcs, int tag)
1301 {
1302 struct tagcache_header tag_hdr;
1303 struct master_header master_hdr;
1304 int i;
1305
1306 while (read_lock)
1307 sleep(1);
1308
1309 memset(tcs, 0, sizeof(struct tagcache_search));
1310 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1311 return false;
1312
1313 tcs->position = sizeof(struct tagcache_header);
1314 tcs->type = tag;
1315 tcs->seek_pos = 0;
1316 tcs->list_position = 0;
1317 tcs->seek_list_count = 0;
1318 tcs->filter_count = 0;
1319 tcs->masterfd = -1;
1320
1321 for (i = 0; i < TAG_COUNT; i++)
1322 tcs->idxfd[i] = -1;
1323
1324 #ifndef HAVE_TC_RAMCACHE
1325 tcs->ramsearch = false;
1326 #else
1327 tcs->ramsearch = tc_stat.ramcache;
1328 if (tcs->ramsearch)
1329 {
1330 tcs->entry_count = ramcache_hdr->entry_count[tcs->type];
1331 }
1332 else
1333 #endif
1334 {
1335 /* Always open as R/W so we can pass tcs to functions that modify data also
1336 * without failing. */
1337 tcs->masterfd = open_master_fd(&master_hdr, true);
1338 if (tcs->masterfd < 0)
1339 return false;
1340
1341 if (!TAGCACHE_IS_NUMERIC(tcs->type))
1342 {
1343 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1344 if (tcs->idxfd[tcs->type] < 0)
1345 return false;
1346
1347 tcs->entry_count = tag_hdr.entry_count;
1348 }
1349 else
1350 {
1351 tcs->entry_count = master_hdr.tch.entry_count;
1352 }
1353 }
1354
1355 tcs->valid = true;
1356 tcs->initialized = true;
1357 write_lock++;
1358
1359 return true;
1360 }
1361
1362 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1363 void *buffer, long length)
1364 {
1365 tcs->unique_list = (unsigned long *)buffer;
1366 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1367 tcs->unique_list_count = 0;
1368 }
1369
1370 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1371 int tag, int seek)
1372 {
1373 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1374 return false;
1375
1376 if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag))
1377 return false;
1378
1379 tcs->filter_tag[tcs->filter_count] = tag;
1380 tcs->filter_seek[tcs->filter_count] = seek;
1381 tcs->filter_count++;
1382
1383 return true;
1384 }
1385
1386 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1387 struct tagcache_search_clause *clause)
1388 {
1389 int i;
1390
1391 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1392 {
1393 logf("Too many clauses");
1394 return false;
1395 }
1396
1397 if (clause->type != clause_logical_or)
1398 {
1399 /* Check if there is already a similar filter in present (filters are
1400 * much faster than clauses).
1401 */
1402 for (i = 0; i < tcs->filter_count; i++)
1403 {
1404 if (tcs->filter_tag[i] == clause->tag)
1405 return true;
1406 }
1407
1408 if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0)
1409 {
1410 char buf[MAX_PATH];
1411
1412 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1413 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1414 }
1415 }
1416
1417 tcs->clause[tcs->clause_count] = clause;
1418 tcs->clause_count++;
1419
1420 return true;
1421 }
1422
1423 static bool get_next(struct tagcache_search *tcs)
1424 {
1425 static char buf[TAG_MAXLEN+32];
1426 struct tagfile_entry entry;
1427 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1428 long flag = 0;
1429 #endif
1430
1431 if (!tcs->valid || !tc_stat.ready)
1432 return false;
1433
1434 if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type)
1435 #ifdef HAVE_TC_RAMCACHE
1436 && !tcs->ramsearch
1437 #endif
1438 )
1439 return false;
1440
1441 /* Relative fetch. */
1442 if (tcs->filter_count > 0 || tcs->clause_count > 0
1443 || TAGCACHE_IS_NUMERIC(tcs->type)
1444 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1445 /* We need to retrieve flag status for dircache. */
1446 || (tcs->ramsearch && tcs->type == tag_filename)
1447 #endif
1448 )
1449 {
1450 struct tagcache_seeklist_entry *seeklist;
1451
1452 /* Check for end of list. */
1453 if (tcs->list_position == tcs->seek_list_count)
1454 {
1455 tcs->list_position = 0;
1456
1457 /* Try to fetch more. */
1458 if (!build_lookup_list(tcs))
1459 {
1460 tcs->valid = false;
1461 return false;
1462 }
1463 }
1464
1465 seeklist = &tcs->seeklist[tcs->list_position];
1466 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1467 flag = seeklist->flag;
1468 #endif
1469 tcs->position = seeklist->seek;
1470 tcs->idx_id = seeklist->idx_id;
1471 tcs->list_position++;
1472 }
1473 else
1474 {
1475 if (tcs->entry_count == 0)
1476 {
1477 tcs->valid = false;
1478 return false;
1479 }
1480
1481 tcs->entry_count--;
1482 }
1483
1484 tcs->result_seek = tcs->position;
1485
1486 if (TAGCACHE_IS_NUMERIC(tcs->type))
1487 {
1488 snprintf(buf, sizeof(buf), "%ld", tcs->position);
1489 tcs->result = buf;
1490 tcs->result_len = strlen(buf) + 1;
1491 return true;
1492 }
1493
1494 /* Direct fetch. */
1495 #ifdef HAVE_TC_RAMCACHE
1496 if (tcs->ramsearch)
1497 {
1498
1499 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1500 if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)
1501 && is_dircache_intact())
1502 {
1503 size_t len = dircache_copy_path(tcs->position, buf, sizeof buf);
1504 tcs->result_len = len + 1;
1505 tcs->result = buf;
1506 tcs->ramresult = false;
1507
1508 return true;
1509 }
1510 else
1511 #endif
1512 if (tcs->type != tag_filename)
1513 {
1514 struct tagfile_entry *ep;
1515
1516 ep = (struct tagfile_entry *)&ramcache_hdr->tags[tcs->type][tcs->position];
1517 tcs->result = ep->tag_data;
1518 tcs->result_len = strlen(tcs->result) + 1;
1519 tcs->idx_id = ep->idx_id;
1520 tcs->ramresult = true;
1521
1522 /* Increase position for the next run. This may get overwritten. */
1523 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1524
1525 return true;
1526 }
1527 }
1528 #endif
1529
1530 if (!open_files(tcs, tcs->type))
1531 {
1532 tcs->valid = false;
1533 return false;
1534 }
1535
1536 /* Seek stream to the correct position and continue to direct fetch. */
1537 lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET);
1538
1539 if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry))
1540 {
1541 logf("read error #5");
1542 tcs->valid = false;
1543 return false;
1544 }
1545
1546 if (entry.tag_length > (long)sizeof(buf))
1547 {
1548 tcs->valid = false;
1549 logf("too long tag #2");
1550 logf("P:%lX/%lX", tcs->position, entry.tag_length);
1551 return false;
1552 }
1553
1554 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1555 {
1556 tcs->valid = false;
1557 logf("read error #4");
1558 return false;
1559 }
1560
1561 /**
1562 Update the position for the next read (this may be overridden
1563 if filters or clauses are being used).
1564 */
1565 tcs->position += sizeof(struct tagfile_entry) + entry.tag_length;
1566 tcs->result = buf;
1567 tcs->result_len = strlen(tcs->result) + 1;
1568 tcs->idx_id = entry.idx_id;
1569 tcs->ramresult = false;
1570
1571 return true;
1572 }
1573
1574 bool tagcache_get_next(struct tagcache_search *tcs)
1575 {
1576 while (get_next(tcs))
1577 {
1578 if (tcs->result_len > 1)
1579 return true;
1580 }
1581
1582 return false;
1583 }
1584
1585 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1586 int tag, char *buf, long size)
1587 {
1588 struct index_entry idx;
1589
1590 *buf = '\0';
1591 if (!get_index(tcs->masterfd, idxid, &idx, true))
1592 return false;
1593
1594 return retrieve(tcs, &idx, tag, buf, size);
1595 }
1596
1597 static bool update_master_header(void)
1598 {
1599 struct master_header myhdr;
1600 int fd;
1601
1602 if (!tc_stat.ready)
1603 return false;
1604
1605 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1606 return false;
1607
1608 myhdr.serial = current_tcmh.serial;
1609 myhdr.commitid = current_tcmh.commitid;
1610 myhdr.dirty = current_tcmh.dirty;
1611
1612 /* Write it back */
1613 lseek(fd, 0, SEEK_SET);
1614 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1615 close(fd);
1616
1617 return true;
1618 }
1619
1620 void tagcache_search_finish(struct tagcache_search *tcs)
1621 {
1622 int i;
1623
1624 if (!tcs->initialized)
1625 return;
1626
1627 if (tcs->masterfd >= 0)
1628 {
1629 close(tcs->masterfd);
1630 tcs->masterfd = -1;
1631 }
1632
1633 for (i = 0; i < TAG_COUNT; i++)
1634 {
1635 if (tcs->idxfd[i] >= 0)
1636 {
1637 close(tcs->idxfd[i]);
1638 tcs->idxfd[i] = -1;
1639 }
1640 }
1641
1642 tcs->ramsearch = false;
1643 tcs->valid = false;
1644 tcs->initialized = 0;
1645 if (write_lock > 0)
1646 write_lock--;
1647 }
1648
1649 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1650 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1651 {
1652 return (struct tagfile_entry *)&ramcache_hdr->tags[tag][entry->tag_seek[tag]];
1653 }
1654
1655 static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id)
1656 {
1657 return check_virtual_tags(tag, idx_id, entry);
1658 }
1659
1660 static char* get_tag_string(const struct index_entry *entry, int tag)
1661 {
1662 char* s = get_tag(entry, tag)->tag_data;
1663 return strcmp(s, UNTAGGED) ? s : NULL;
1664 }
1665
1666 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1667 {
1668 struct index_entry *entry;
1669 int idx_id;
1670
1671 if (!tc_stat.ready || !tc_stat.ramcache)
1672 return false;
1673
1674 /* Find the corresponding entry in tagcache. */
1675 idx_id = find_entry_ram(filename, -1);
1676 if (idx_id < 0)
1677 return false;
1678
1679 entry = &ramcache_hdr->indices[idx_id];
1680
1681 memset(id3, 0, sizeof(struct mp3entry));
1682
1683 id3->title = get_tag_string(entry, tag_title);
1684 id3->artist = get_tag_string(entry, tag_artist);
1685 id3->album = get_tag_string(entry, tag_album);
1686 id3->genre_string = get_tag_string(entry, tag_genre);
1687 id3->composer = get_tag_string(entry, tag_composer);
1688 id3->comment = get_tag_string(entry, tag_comment);
1689 id3->albumartist = get_tag_string(entry, tag_albumartist);
1690 id3->grouping = get_tag_string(entry, tag_grouping);
1691
1692 id3->length = get_tag_numeric(entry, tag_length, idx_id);
1693 id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id);
1694 id3->rating = get_tag_numeric(entry, tag_rating, idx_id);
1695 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id);
1696 id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10;
1697 id3->year = get_tag_numeric(entry, tag_year, idx_id);
1698
1699 id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id);
1700 id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id);
1701 id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id);
1702 if (id3->bitrate == 0)
1703 id3->bitrate = 1;
1704
1705 #if CONFIG_CODEC == SWCODEC
1706 if (global_settings.autoresume_enable)
1707 {
1708 id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id);
1709 logf("tagcache_fill_tags: Set offset for %s to %lX\n",
1710 id3->title, id3->offset);
1711 }
1712 #endif
1713
1714 return true;
1715 }
1716 #endif
1717
1718 static inline void write_item(const char *item)
1719 {
1720 int len = strlen(item) + 1;
1721
1722 data_size += len;
1723 write(cachefd, item, len);
1724 }
1725
1726 static int check_if_empty(char **tag)
1727 {
1728 int length;
1729
1730 if (*tag == NULL || **tag == '\0')
1731 {
1732 *tag = UNTAGGED;
1733 return sizeof(UNTAGGED); /* Tag length */
1734 }
1735
1736 length = strlen(*tag);
1737 if (length > TAG_MAXLEN)
1738 {
1739 logf("over length tag: %s", *tag);
1740 length = TAG_MAXLEN;
1741 (*tag)[length] = '\0';
1742 }
1743
1744 return length + 1;
1745 }
1746
1747 #define ADD_TAG(entry,tag,data) \
1748 /* Adding tag */ \
1749 entry.tag_offset[tag] = offset; \
1750 entry.tag_length[tag] = check_if_empty(data); \
1751 offset += entry.tag_length[tag]
1752 /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good
1753 * idea, as it uses lots of stack and is called from a recursive function
1754 * (check_dir).
1755 */
1756 static void __attribute__ ((noinline)) add_tagcache(char *path,
1757 unsigned long mtime
1758 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1759 ,int dc
1760 #endif
1761 )
1762 {
1763 struct mp3entry id3;
1764 struct temp_file_entry entry;
1765 bool ret;
1766 int fd;
1767 int idx_id = -1;
1768 char tracknumfix[3];
1769 int offset = 0;
1770 int path_length = strlen(path);
1771 bool has_albumartist;
1772 bool has_grouping;
1773
1774 #ifdef SIMULATOR
1775 /* Crude logging for the sim - to aid in debugging */
1776 int logfd = open(ROCKBOX_DIR "/database.log",
1777 O_WRONLY | O_APPEND | O_CREAT, 0666);
1778 if (logfd >= 0) {
1779 write(logfd, path, strlen(path));
1780 write(logfd, "\n", 1);
1781 close(logfd);
1782 }
1783 #endif
1784
1785 if (cachefd < 0)
1786 return ;
1787
1788 /* Check for overlength file path. */
1789 if (path_length > TAG_MAXLEN)
1790 {
1791 /* Path can't be shortened. */
1792 logf("Too long path: %s", path);
1793 return ;
1794 }
1795
1796 /* Check if the file is supported. */
1797 if (probe_file_format(path) == AFMT_UNKNOWN)
1798 return ;
1799
1800 /* Check if the file is already cached. */
1801 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1802 if (tc_stat.ramcache && is_dircache_intact())
1803 {
1804 idx_id = find_entry_ram(path, dc);
1805 }
1806 #endif
1807
1808 /* Be sure the entry doesn't exist. */
1809 if (filenametag_fd >= 0 && idx_id < 0)
1810 idx_id = find_entry_disk(path, false);
1811
1812 /* Check if file has been modified. */
1813 if (idx_id >= 0)
1814 {
1815 struct index_entry idx;
1816
1817 /* TODO: Mark that the index exists (for fast reverse scan) */
1818 //found_idx[idx_id/8] |= idx_id%8;
1819
1820 if (!get_index(-1, idx_id, &idx, true))
1821 {
1822 logf("failed to retrieve index entry");
1823 return ;
1824 }
1825
1826 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1827 {
1828 /* No changes to file. */
1829 return ;
1830 }
1831
1832 /* Metadata might have been changed. Delete the entry. */
1833 logf("Re-adding: %s", path);
1834 if (!delete_entry(idx_id))
1835 {
1836 logf("delete_entry failed: %d", idx_id);
1837 return ;
1838 }
1839 }
1840
1841 fd = open(path, O_RDONLY);
1842 if (fd < 0)
1843 {
1844 logf("open fail: %s", path);
1845 return ;
1846 }
1847
1848 memset(&id3, 0, sizeof(struct mp3entry));
1849 memset(&entry, 0, sizeof(struct temp_file_entry));
1850 memset(&tracknumfix, 0, sizeof(tracknumfix));
1851 ret = get_metadata(&id3, fd, path);
1852 close(fd);
1853
1854 if (!ret)
1855 return ;
1856
1857 logf("-> %s", path);
1858
1859 if (id3.tracknum <= 0) /* Track number missing? */
1860 {
1861 id3.tracknum = -1;
1862 }
1863
1864 /* Numeric tags */
1865 entry.tag_offset[tag_year] = id3.year;
1866 entry.tag_offset[tag_discnumber] = id3.discnum;
1867 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1868 entry.tag_offset[tag_length] = id3.length;
1869 entry.tag_offset[tag_bitrate] = id3.bitrate;
1870 entry.tag_offset[tag_mtime] = mtime;
1871
1872 /* String tags. */
1873 has_albumartist = id3.albumartist != NULL
1874 && strlen(id3.albumartist) > 0;
1875 has_grouping = id3.grouping != NULL
1876 && strlen(id3.grouping) > 0;
1877
1878 ADD_TAG(entry, tag_filename, &path);
1879 ADD_TAG(entry, tag_title, &id3.title);
1880 ADD_TAG(entry, tag_artist, &id3.artist);
1881 ADD_TAG(entry, tag_album, &id3.album);
1882 ADD_TAG(entry, tag_genre, &id3.genre_string);
1883 ADD_TAG(entry, tag_composer, &id3.composer);
1884 ADD_TAG(entry, tag_comment, &id3.comment);
1885 if (has_albumartist)
1886 {
1887 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1888 }
1889 else
1890 {
1891 ADD_TAG(entry, tag_albumartist, &id3.artist);
1892 }
1893 if (has_grouping)
1894 {
1895 ADD_TAG(entry, tag_grouping, &id3.grouping);
1896 }
1897 else
1898 {
1899 ADD_TAG(entry, tag_grouping, &id3.title);
1900 }
1901 entry.data_length = offset;
1902
1903 /* Write the header */
1904 write(cachefd, &entry, sizeof(struct temp_file_entry));
1905
1906 /* And tags also... Correct order is critical */
1907 write_item(path);
1908 write_item(id3.title);
1909 write_item(id3.artist);
1910 write_item(id3.album);
1911 write_item(id3.genre_string);
1912 write_item(id3.composer);
1913 write_item(id3.comment);
1914 if (has_albumartist)
1915 {
1916 write_item(id3.albumartist);
1917 }
1918 else
1919 {
1920 write_item(id3.artist);
1921 }
1922 if (has_grouping)
1923 {
1924 write_item(id3.grouping);
1925 }
1926 else
1927 {
1928 write_item(id3.title);
1929 }
1930 total_entry_count++;
1931 }
1932
1933 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1934 {
1935 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1936 int len = strlen(str)+1;
1937 int i;
1938 unsigned crc32;
1939 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1940 char buf[TAG_MAXLEN+32];
1941
1942 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1943 buf[i] = tolower(str[i]);
1944 buf[i] = '\0';
1945
1946 crc32 = crc_32(buf, i, 0xffffffff);
1947
1948 if (unique)
1949 {
1950 /* Check if the crc does not exist -> entry does not exist for sure. */
1951 for (i = 0; i < tempbufidx; i++)
1952 {
1953 if (crcbuf[-i] != crc32)
1954 continue;
1955
1956 if (!strcasecmp(str, index[i].str))
1957 {
1958 if (id < 0 || id >= lookup_buffer_depth)
1959 {
1960 logf("lookup buf overf.: %d", id);
1961 return false;
1962 }
1963
1964 lookup[id] = &index[i];
1965 return true;
1966 }
1967 }
1968 }
1969
1970 /* Insert to CRC buffer. */
1971 crcbuf[-tempbufidx] = crc32;
1972 tempbuf_left -= 4;
1973
1974 /* Insert it to the buffer. */
1975 tempbuf_left -= len;
1976 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1977 return false;
1978
1979 if (id >= lookup_buffer_depth)
1980 {
1981 logf("lookup buf overf. #2: %d", id);
1982 return false;
1983 }
1984
1985 if (id >= 0)
1986 {
1987 lookup[id] = &index[tempbufidx];
1988 index[tempbufidx].idlist.id = id;
1989 }
1990 else
1991 index[tempbufidx].idlist.id = -1;
1992
1993 index[tempbufidx].idlist.next = NULL;
1994 index[tempbufidx].idx_id = idx_id;
1995 index[tempbufidx].seek = -1;
1996 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1997 memcpy(index[tempbufidx].str, str, len);
1998 tempbuf_pos += len;
1999 tempbufidx++;
2000
2001 return true;
2002 }
2003
2004 static int compare(const void *p1, const void *p2)
2005 {
2006 do_timed_yield();
2007
2008 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
2009 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
2010
2011 if (strcmp(e1->str, UNTAGGED) == 0)
2012 {
2013 if (strcmp(e2->str, UNTAGGED) == 0)
2014 return 0;
2015 return -1;
2016 }
2017 else if (strcmp(e2->str, UNTAGGED) == 0)
2018 return 1;
2019
2020 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
2021 }
2022
2023 static int tempbuf_sort(int fd)
2024 {
2025 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
2026 struct tagfile_entry fe;
2027 int i;
2028 int length;
2029
2030 /* Generate reverse lookup entries. */
2031 for (i = 0; i < lookup_buffer_depth; i++)
2032 {
2033 struct tempbuf_id_list *idlist;
2034
2035 if (!lookup[i])
2036 continue;
2037
2038 if (lookup[i]->idlist.id == i)
2039 continue;
2040
2041 idlist = &lookup[i]->idlist;
2042 while (idlist->next != NULL)
2043 idlist = idlist->next;
2044
2045 tempbuf_left -= sizeof(struct tempbuf_id_list);
2046 if (tempbuf_left - 4 < 0)
2047 return -1;
2048
2049 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2050 if (tempbuf_pos & 0x03)
2051 {
2052 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
2053 tempbuf_left -= 3;
2054 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
2055 }
2056 tempbuf_pos += sizeof(struct tempbuf_id_list);
2057
2058 idlist = idlist->next;
2059 idlist->id = i;
2060 idlist->next = NULL;
2061
2062 do_timed_yield();
2063 }
2064
2065 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
2066 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
2067
2068 for (i = 0; i < tempbufidx; i++)
2069 {
2070 struct tempbuf_id_list *idlist = &index[i].idlist;
2071
2072 /* Fix the lookup list. */
2073 while (idlist != NULL)
2074 {
2075 if (idlist->id >= 0)
2076 lookup[idlist->id] = &index[i];
2077 idlist = idlist->next;
2078 }
2079
2080 index[i].seek = lseek(fd, 0, SEEK_CUR);
2081 length = strlen(index[i].str) + 1;
2082 fe.tag_length = length;
2083 fe.idx_id = index[i].idx_id;
2084
2085 /* Check the chunk alignment. */
2086 if ((fe.tag_length + sizeof(struct tagfile_entry))
2087 % TAGFILE_ENTRY_CHUNK_LENGTH)
2088 {
2089 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2090 ((fe.tag_length + sizeof(struct tagfile_entry))
2091 % TAGFILE_ENTRY_CHUNK_LENGTH);
2092 }
2093
2094 #ifdef TAGCACHE_STRICT_ALIGN
2095 /* Make sure the entry is long aligned. */
2096 if (index[i].seek & 0x03)
2097 {
2098 logf("tempbuf_sort: alignment error!");
2099 return -3;
2100 }
2101 #endif
2102
2103 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2104 sizeof(struct tagfile_entry))
2105 {
2106 logf("tempbuf_sort: write error #1");
2107 return -1;
2108 }
2109
2110 if (write(fd, index[i].str, length) != length)
2111 {
2112 logf("tempbuf_sort: write error #2");
2113 return -2;
2114 }
2115
2116 /* Write some padding. */
2117 if (fe.tag_length - length > 0)
2118 write(fd, "XXXXXXXX", fe.tag_length - length);
2119 }
2120
2121 return i;
2122 }
2123
2124 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2125 {
2126 if (id < 0 || id >= lookup_buffer_depth)
2127 return NULL;
2128
2129 return lookup[id];
2130 }
2131
2132
2133 inline static int tempbuf_find_location(int id)
2134 {
2135 struct tempbuf_searchidx *entry;
2136
2137 entry = tempbuf_locate(id);
2138 if (entry == NULL)
2139 return -1;
2140
2141 return entry->seek;
2142 }
2143
2144 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2145 {
2146 struct master_header tcmh;
2147 struct index_entry idx;
2148 int masterfd;
2149 int masterfd_pos;
2150 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2151 int max_entries;
2152 int entries_processed = 0;
2153 int i, j;
2154 char buf[TAG_MAXLEN];
2155
2156 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2157
2158 logf("Building numeric indices...");
2159 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2160
2161 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2162 return false;
2163
2164 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2165 SEEK_CUR);
2166 if (masterfd_pos == filesize(masterfd))
2167 {
2168 logf("we can't append!");
2169 close(masterfd);
2170 return false;
2171 }
2172
2173 while (entries_processed < h->entry_count)
2174 {
2175 int count = MIN(h->entry_count - entries_processed, max_entries);
2176
2177 /* Read in as many entries as possible. */
2178 for (i = 0; i < count; i++)
2179 {
2180 struct temp_file_entry *tfe = &entrybuf[i];
2181 int datastart;
2182
2183 /* Read in numeric data. */
2184 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2185 sizeof(struct temp_file_entry))
2186 {
2187 logf("read fail #1");
2188 close(masterfd);
2189 return false;
2190 }
2191
2192 datastart = lseek(tmpfd, 0, SEEK_CUR);
2193
2194 /**
2195 * Read string data from the following tags:
2196 * - tag_filename
2197 * - tag_artist
2198 * - tag_album
2199 * - tag_title
2200 *
2201 * A crc32 hash is calculated from the read data
2202 * and stored back to the data offset field kept in memory.
2203 */
2204 #define tmpdb_read_string_tag(tag) \
2205 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2206 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2207 { \
2208 logf("read fail: buffer overflow"); \
2209 close(masterfd); \
2210 return false; \
2211 } \
2212 \
2213 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2214 tfe->tag_length[tag]) \
2215 { \
2216 logf("read fail #2"); \
2217 close(masterfd); \
2218 return false; \
2219 } \
2220 \
2221 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2222 lseek(tmpfd, datastart, SEEK_SET)
2223
2224 tmpdb_read_string_tag(tag_filename);
2225 tmpdb_read_string_tag(tag_artist);
2226 tmpdb_read_string_tag(tag_album);
2227 tmpdb_read_string_tag(tag_title);
2228
2229 /* Seek to the end of the string data. */
2230 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2231 }
2232
2233 /* Backup the master index position. */
2234 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2235 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2236
2237 /* Check if we can resurrect some deleted runtime statistics data. */
2238 for (i = 0; i < tcmh.tch.entry_count; i++)
2239 {
2240 /* Read the index entry. */
2241 if (ecread_index_entry(masterfd, &idx)
2242 != sizeof(struct index_entry))
2243 {
2244 logf("read fail #3");
2245 close(masterfd);
2246 return false;
2247 }
2248
2249 /**
2250 * Skip unless the entry is marked as being deleted
2251 * or the data has already been resurrected.
2252 */
2253 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2254 continue;
2255
2256 /* Now try to match the entry. */
2257 /**
2258 * To succesfully match a song, the following conditions
2259 * must apply:
2260 *
2261 * For numeric fields: tag_length
2262 * - Full identical match is required
2263 *
2264 * If tag_filename matches, no further checking necessary.
2265 *
2266 * For string hashes: tag_artist, tag_album, tag_title
2267 * - All three of these must match
2268 */
2269 for (j = 0; j < count; j++)
2270 {
2271 struct temp_file_entry *tfe = &entrybuf[j];
2272
2273 /* Try to match numeric fields first. */
2274 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2275 continue;
2276
2277 /* Now it's time to do the hash matching. */
2278 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2279 {
2280 int match_count = 0;
2281
2282 /* No filename match, check if we can match two other tags. */
2283 #define tmpdb_match(tag) \
2284 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2285 match_count++
2286
2287 tmpdb_match(tag_artist);
2288 tmpdb_match(tag_album);
2289 tmpdb_match(tag_title);
2290
2291 if (match_count < 3)
2292 {
2293 /* Still no match found, give up. */
2294 continue;
2295 }
2296 }
2297
2298 /* A match found, now copy & resurrect the statistical data. */
2299 #define tmpdb_copy_tag(tag) \
2300 tfe->tag_offset[tag] = idx.tag_seek[tag]
2301
2302 tmpdb_copy_tag(tag_playcount);
2303 tmpdb_copy_tag(tag_rating);
2304 tmpdb_copy_tag(tag_playtime);
2305 tmpdb_copy_tag(tag_lastplayed);
2306 tmpdb_copy_tag(tag_commitid);
2307 tmpdb_copy_tag(tag_lastoffset);
2308
2309 /* Avoid processing this entry again. */
2310 idx.flag |= FLAG_RESURRECTED;
2311
2312 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2313 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2314 {
2315 logf("masterfd writeback fail #1");
2316 close(masterfd);
2317 return false;
2318 }
2319
2320 logf("Entry resurrected");
2321 }
2322 }
2323
2324
2325 /* Restore the master index position. */
2326 lseek(masterfd, masterfd_pos, SEEK_SET);
2327
2328 /* Commit the data to the index. */
2329 for (i = 0; i < count; i++)
2330 {
2331 int loc = lseek(masterfd, 0, SEEK_CUR);
2332
2333 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2334 {
2335 logf("read fail #3");
2336 close(masterfd);
2337 return false;
2338 }
2339
2340 for (j = 0; j < TAG_COUNT; j++)
2341 {
2342 if (!TAGCACHE_IS_NUMERIC(j))
2343 continue;
2344
2345 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2346 }
2347 idx.flag = entrybuf[i].flag;
2348
2349 if (idx.tag_seek[tag_commitid])
2350 {
2351 /* Data has been resurrected. */
2352 idx.flag |= FLAG_DIRTYNUM;
2353 }
2354 else if (tc_stat.ready && current_tcmh.commitid > 0)
2355 {
2356 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2357 idx.flag |= FLAG_DIRTYNUM;
2358 }
2359
2360 /* Write back the updated index. */
2361 lseek(masterfd, loc, SEEK_SET);
2362 if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry))
2363 {
2364 logf("write fail");
2365 close(masterfd);
2366 return false;
2367 }
2368 }
2369
2370 entries_processed += count;
2371 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2372 }
2373
2374 close(masterfd);
2375
2376 return true;
2377 }
2378
2379 /**
2380 * Return values:
2381 * > 0 success
2382 * == 0 temporary failure
2383 * < 0 fatal error
2384 */
2385 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2386 {
2387 int i;
2388 struct tagcache_header tch;
2389 struct master_header tcmh;
2390 struct index_entry idxbuf[IDX_BUF_DEPTH];
2391 int idxbuf_pos;
2392 char buf[TAG_MAXLEN+32];
2393 int fd = -1, masterfd;
2394 bool error = false;
2395 int init;
2396 int masterfd_pos;
2397
2398 logf("Building index: %d", index_type);
2399
2400 /* Check the number of entries we need to allocate ram for. */
2401 commit_entry_count = h->entry_count + 1;
2402
2403 masterfd = open_master_fd(&tcmh, false);
2404 if (masterfd >= 0)
2405 {
2406 commit_entry_count += tcmh.tch.entry_count;
2407 close(masterfd);
2408 }
2409 else
2410 remove_files(); /* Just to be sure we are clean. */
2411
2412 /* Open the index file, which contains the tag names. */
2413 fd = open_tag_fd(&tch, index_type, true);
2414 if (fd >= 0)
2415 {
2416 logf("tch.datasize=%ld", tch.datasize);
2417 lookup_buffer_depth = 1 +
2418 /* First part */ commit_entry_count +
2419 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2420 }
2421 else
2422 {
2423 lookup_buffer_depth = 1 +
2424 /* First part */ commit_entry_count +
2425 /* Second part */ 0;
2426 }
2427
2428 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2429 logf("commit_entry_count=%ld", commit_entry_count);
2430
2431 /* Allocate buffer for all index entries from both old and new
2432 * tag files. */
2433 tempbufidx = 0;
2434 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2435
2436 /* Allocate lookup buffer. The first portion of commit_entry_count
2437 * contains the new tags in the temporary file and the second
2438 * part for locating entries already in the db.
2439 *
2440 * New tags Old tags
2441 * +---------+---------------------------+
2442 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2443 * +---------+---------------------------+
2444 *
2445 * Old tags are inserted to a temporary buffer with position:
2446 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2447 * And new tags with index:
2448 * tempbuf_insert(idx, ...);
2449 *
2450 * The buffer is sorted and written into tag file:
2451 * tempbuf_sort(...);
2452 * leaving master index locations messed up.
2453 *
2454 * That is fixed using the lookup buffer for old tags:
2455 * new_seek = tempbuf_find_location(old_seek, ...);
2456 * and for new tags:
2457 * new_seek = tempbuf_find_location(idx);
2458 */
2459 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2460 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2461 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2462
2463 /* And calculate the remaining data space used mainly for storing
2464 * tag data (strings). */
2465 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2466 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2467 {
2468 logf("Buffer way too small!");
2469 return 0;
2470 }
2471
2472 if (fd >= 0)
2473 {
2474 /**
2475 * If tag file contains unique tags (sorted index), we will load
2476 * it entirely into memory so we can resort it later for use with
2477 * chunked browsing.
2478 */
2479 if (TAGCACHE_IS_SORTED(index_type))
2480 {
2481 logf("loading tags...");
2482 for (i = 0; i < tch.entry_count; i++)
2483 {
2484 struct tagfile_entry entry;
2485 int loc = lseek(fd, 0, SEEK_CUR);
2486 bool ret;
2487
2488 if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry))
2489 {
2490 logf("read error #7");
2491 close(fd);
2492 return -2;
2493 }
2494
2495 if (entry.tag_length >= (int)sizeof(buf))
2496 {
2497 logf("too long tag #3");
2498 close(fd);
2499 return -2;
2500 }
2501
2502 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2503 {
2504 logf("read error #8");
2505 close(fd);
2506 return -2;
2507 }
2508
2509 /* Skip deleted entries. */
2510 if (buf[0] == '\0')
2511 continue;
2512
2513 /**
2514 * Save the tag and tag id in the memory buffer. Tag id
2515 * is saved so we can later reindex the master lookup
2516 * table when the index gets resorted.
2517 */
2518 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2519 + commit_entry_count, entry.idx_id,
2520 TAGCACHE_IS_UNIQUE(index_type));
2521 if (!ret)
2522 {
2523 close(fd);
2524 return -3;
2525 }
2526 do_timed_yield();
2527 }
2528 logf("done");
2529 }
2530 else
2531 tempbufidx = tch.entry_count;
2532 }
2533 else
2534 {
2535 /**
2536 * Creating new index file to store the tags. No need to preload
2537 * anything whether the index type is sorted or not.
2538 */
2539 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2540 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2541 if (fd < 0)
2542 {
2543 logf("%s open fail", buf);
2544 return -2;
2545 }
2546
2547 tch.magic = TAGCACHE_MAGIC;
2548 tch.entry_count = 0;
2549 tch.datasize = 0;
2550
2551 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2552 != sizeof(struct tagcache_header))
2553 {
2554 logf("header write failed");
2555 close(fd);
2556 return -2;
2557 }
2558 }
2559
2560 /* Loading the tag lookup file as "master file". */
2561 logf("Loading index file");
2562 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2563
2564 if (masterfd < 0)
2565 {
2566 logf("Creating new DB");
2567 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666);
2568
2569 if (masterfd < 0)
2570 {
2571 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2572 close(fd);
2573 return -2;
2574 }
2575
2576 /* Write the header (write real values later). */
2577 memset(&tcmh, 0, sizeof(struct master_header));
2578 tcmh.tch = *h;
2579 tcmh.tch.entry_count = 0;
2580 tcmh.tch.datasize = 0;
2581 tcmh.dirty = true;
2582 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2583 init = true;
2584 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2585 }
2586 else
2587 {
2588 /**
2589 * Master file already exists so we need to process the current
2590 * file first.
2591 */
2592 init = false;
2593
2594 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2595 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2596 {
2597 logf("header error");
2598 close(fd);
2599 close(masterfd);
2600 return -2;
2601 }
2602
2603 /**
2604 * If we reach end of the master file, we need to expand it to
2605 * hold new tags. If the current index is not sorted, we can
2606 * simply append new data to end of the file.
2607 * However, if the index is sorted, we need to update all tag
2608 * pointers in the master file for the current index.
2609 */
2610 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2611 SEEK_CUR);
2612 if (masterfd_pos == filesize(masterfd))
2613 {
2614 logf("appending...");
2615 init = true;
2616 }
2617 }
2618
2619 /**
2620 * Load new unique tags in memory to be sorted later and added
2621 * to the master lookup file.
2622 */
2623 if (TAGCACHE_IS_SORTED(index_type))
2624 {
2625 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2626 /* h is the header of the temporary file containing new tags. */
2627 logf("inserting new tags...");
2628 for (i = 0; i < h->entry_count; i++)
2629 {
2630 struct temp_file_entry entry;
2631
2632 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2633 sizeof(struct temp_file_entry))
2634 {
2635 logf("read fail #3");
2636 error = true;
2637 goto error_exit;
2638 }
2639
2640 /* Read data. */
2641 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2642 {
2643 logf("too long entry!");
2644 error = true;
2645 goto error_exit;
2646 }
2647
2648 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2649 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2650 entry.tag_length[index_type])
2651 {
2652 logf("read fail #4");
2653 error = true;
2654 goto error_exit;
2655 }
2656
2657 if (TAGCACHE_IS_UNIQUE(index_type))
2658 error = !tempbuf_insert(buf, i, -1, true);
2659 else
2660 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2661
2662 if (error)
2663 {
2664 logf("insert error");
2665 goto error_exit;
2666 }
2667
2668 /* Skip to next. */
2669 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2670 entry.tag_length[index_type], SEEK_CUR);
2671 do_timed_yield();
2672 }
2673 logf("done");
2674
2675 /* Sort the buffer data and write it to the index file. */
2676 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2677 /**
2678 * We need to truncate the index file now. There can be junk left
2679 * at the end of file (however, we _should_ always follow the
2680 * entry_count and don't crash with that).
2681 */
2682 ftruncate(fd, lseek(fd, 0, SEEK_CUR));
2683
2684 i = tempbuf_sort(fd);
2685 if (i < 0)
2686 goto error_exit;
2687 logf("sorted %d tags", i);
2688
2689 /**
2690 * Now update all indexes in the master lookup file.
2691 */
2692 logf("updating indices...");
2693 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2694 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2695 {
2696 int j;
2697 int loc = lseek(masterfd, 0, SEEK_CUR);
2698
2699 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2700
2701 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2702 != (int)sizeof(struct index_entry)*idxbuf_pos)
2703 {
2704 logf("read fail #5");
2705 error = true;
2706 goto error_exit ;
2707 }
2708 lseek(masterfd, loc, SEEK_SET);
2709
2710 for (j = 0; j < idxbuf_pos; j++)
2711 {
2712 if (idxbuf[j].flag & FLAG_DELETED)
2713 {
2714 /* We can just ignore deleted entries. */
2715 // idxbuf[j].tag_seek[index_type] = 0;
2716 continue;
2717 }
2718
2719 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2720 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2721 + commit_entry_count);
2722
2723 if (idxbuf[j].tag_seek[index_type] < 0)
2724 {
2725 logf("update error: %ld/%d/%ld",
2726 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2727 error = true;
2728 goto error_exit;
2729 }
2730
2731 do_timed_yield();
2732 }
2733
2734 /* Write back the updated index. */
2735 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2736 index_entry_ec, tc_stat.econ) !=
2737 (int)sizeof(struct index_entry)*idxbuf_pos)
2738 {
2739 logf("write fail");
2740 error = true;
2741 goto error_exit;
2742 }
2743 }
2744 logf("done");
2745 }
2746
2747 /**
2748 * Walk through the temporary file containing the new tags.
2749 */
2750 // build_normal_index(h, tmpfd, masterfd, idx);
2751 logf("updating new indices...");
2752 lseek(masterfd, masterfd_pos, SEEK_SET);
2753 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2754 lseek(fd, 0, SEEK_END);
2755 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2756 {
2757 int j;
2758
2759 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2760 if (init)
2761 {
2762 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2763 }
2764 else
2765 {
2766 int loc = lseek(masterfd, 0, SEEK_CUR);
2767
2768 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2769 != (int)sizeof(struct index_entry)*idxbuf_pos)
2770 {
2771 logf("read fail #6");
2772 error = true;
2773 break ;
2774 }
2775 lseek(masterfd, loc, SEEK_SET);
2776 }
2777
2778 /* Read entry headers. */
2779 for (j = 0; j < idxbuf_pos; j++)
2780 {
2781 if (!TAGCACHE_IS_SORTED(index_type))
2782 {
2783 struct temp_file_entry entry;
2784 struct tagfile_entry fe;
2785
2786 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2787 sizeof(struct temp_file_entry))
2788 {
2789 logf("read fail #7");
2790 error = true;
2791 break ;
2792 }
2793
2794 /* Read data. */
2795 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2796 {
2797 logf("too long entry!");
2798 logf("length=%d", entry.tag_length[index_type]);
2799 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2800 error = true;
2801 break ;
2802 }
2803
2804 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2805 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2806 entry.tag_length[index_type])
2807 {
2808 logf("read fail #8");
2809 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2810 logf("length=0x%02x", entry.tag_length[index_type]);
2811 error = true;
2812 break ;
2813 }
2814
2815 /* Write to index file. */
2816 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2817 fe.tag_length = entry.tag_length[index_type];
2818 fe.idx_id = tcmh.tch.entry_count + i + j;
2819 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2820 write(fd, buf, fe.tag_length);
2821 tempbufidx++;
2822
2823 /* Skip to next. */
2824 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2825 entry.tag_length[index_type], SEEK_CUR);
2826 }
2827 else
2828 {
2829 /* Locate the correct entry from the sorted array. */
2830 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2831 if (idxbuf[j].tag_seek[index_type] < 0)
2832 {
2833 logf("entry not found (%d)", j);
2834 error = true;
2835 break ;
2836 }
2837 }
2838 }
2839
2840 /* Write index. */
2841 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2842 index_entry_ec, tc_stat.econ) !=
2843 (int)sizeof(struct index_entry)*idxbuf_pos)
2844 {
2845 logf("tagcache: write fail #4");
2846 error = true;
2847 break ;
2848 }
2849
2850 do_timed_yield();
2851 }
2852 logf("done");
2853
2854 /* Finally write the header. */
2855 tch.magic = TAGCACHE_MAGIC;
2856 tch.entry_count = tempbufidx;
2857 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2858 lseek(fd, 0, SEEK_SET);
2859 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2860
2861 if (index_type != tag_filename)
2862 h->datasize += tch.datasize;
2863 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2864 error_exit:
2865
2866 close(fd);
2867 close(masterfd);
2868
2869 if (error)
2870 return -2;
2871
2872 return 1;
2873 }
2874
2875 static bool commit(void)
2876 {
2877 struct tagcache_header tch;
2878 struct master_header tcmh;
2879 int i, len, rc;
2880 int tmpfd;
2881 int masterfd;
2882 #ifdef HAVE_DIRCACHE
2883 bool dircache_buffer_stolen = false;
2884 #endif
2885 bool local_allocation = false;
2886
2887 logf("committing tagcache");
2888
2889 while (write_lock)
2890 sleep(1);
2891
2892 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2893 if (tmpfd < 0)
2894 {
2895 logf("nothing to commit");
2896 return true;
2897 }
2898
2899
2900 /* Load the header. */
2901 len = sizeof(struct tagcache_header);
2902 rc = read(tmpfd, &tch, len);
2903
2904 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2905 {
2906 logf("incorrect tmpheader");
2907 close(tmpfd);
2908 remove(TAGCACHE_FILE_TEMP);
2909 return false;
2910 }
2911
2912 if (tch.entry_count == 0)
2913 {
2914 logf("nothing to commit");
2915 close(tmpfd);
2916 remove(TAGCACHE_FILE_TEMP);
2917 return true;
2918 }
2919
2920 /* Fully initialize existing headers (if any) before going further. */
2921 tc_stat.ready = check_all_headers();
2922
2923 #ifdef HAVE_EEPROM_SETTINGS
2924 remove(TAGCACHE_STATEFILE);
2925 #endif
2926
2927 /* At first be sure to unload the ramcache! */
2928 #ifdef HAVE_TC_RAMCACHE
2929 tc_stat.ramcache = false;
2930 #endif
2931
2932 read_lock++;
2933
2934 /* Try to steal every buffer we can :) */
2935 if (tempbuf_size == 0)
2936 local_allocation = true;
2937
2938 #ifdef HAVE_DIRCACHE
2939 if (tempbuf_size == 0)
2940 {
2941 /* Try to steal the dircache buffer. */
2942 tempbuf = dircache_steal_buffer(&tempbuf_size);
2943 tempbuf_size &= ~0x03;
2944
2945 if (tempbuf_size > 0)
2946 {
2947 dircache_buffer_stolen = true;
2948 }
2949 }
2950 #endif
2951
2952 #ifdef HAVE_TC_RAMCACHE
2953 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2954 {
2955 tempbuf = (char *)(ramcache_hdr + 1);
2956 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2957 tempbuf_size &= ~0x03;
2958 }
2959 #endif
2960
2961 /* And finally fail if there are no buffers available. */
2962 if (tempbuf_size == 0)
2963 {
2964 logf("delaying commit until next boot");
2965 tc_stat.commit_delayed = true;
2966 close(tmpfd);
2967 read_lock--;
2968 return false;
2969 }
2970
2971 logf("commit %ld entries...", tch.entry_count);
2972
2973 /* Mark DB dirty so it will stay disabled if commit fails. */
2974 current_tcmh.dirty = true;
2975 update_master_header();
2976
2977 /* Now create the index files. */
2978 tc_stat.commit_step = 0;
2979 tch.datasize = 0;
2980 tc_stat.commit_delayed = false;
2981
2982 for (i = 0; i < TAG_COUNT; i++)
2983 {
2984 int ret;
2985
2986 if (TAGCACHE_IS_NUMERIC(i))
2987 continue;
2988
2989 tc_stat.commit_step++;
2990 ret = build_index(i, &tch, tmpfd);
2991 if (ret <= 0)
2992 {
2993 close(tmpfd);
2994 logf("tagcache failed init");
2995 if (ret == 0)
2996 tc_stat.commit_delayed = true;
2997
2998 tc_stat.commit_step = 0;
2999 read_lock--;
3000 return false;
3001 }
3002 }
3003
3004 if (!build_numeric_indices(&tch, tmpfd))
3005 {
3006 logf("Failure to commit numeric indices");
3007 close(tmpfd);
3008 tc_stat.commit_step = 0;
3009 read_lock--;
3010 return false;
3011 }
3012
3013 close(tmpfd);
3014
3015 tc_stat.commit_step = 0;
3016
3017 /* Update the master index headers. */
3018 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
3019 {
3020 read_lock--;
3021 return false;
3022 }
3023
3024 remove(TAGCACHE_FILE_TEMP);
3025
3026 tcmh.tch.entry_count += tch.entry_count;
3027 tcmh.tch.datasize = sizeof(struct master_header)
3028 + sizeof(struct index_entry) * tcmh.tch.entry_count
3029 + tch.datasize;
3030 tcmh.dirty = false;
3031 tcmh.commitid++;
3032
3033 lseek(masterfd, 0, SEEK_SET);
3034 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
3035 close(masterfd);
3036
3037 logf("tagcache committed");
3038 tc_stat.ready = check_all_headers();
3039 tc_stat.readyvalid = true;
3040
3041 if (local_allocation)
3042 {
3043 tempbuf = NULL;
3044 tempbuf_size = 0;
3045 }
3046
3047 #ifdef HAVE_DIRCACHE
3048 /* Rebuild the dircache, if we stole the buffer. */
3049 if (dircache_buffer_stolen)
3050 dircache_build(0);
3051 #endif
3052
3053 #ifdef HAVE_TC_RAMCACHE
3054 /* Reload tagcache. */
3055 if (tc_stat.ramcache_allocated > 0)
3056 tagcache_start_scan();
3057 #endif
3058
3059 read_lock--;
3060
3061 return true;
3062 }
3063
3064 static int tempbuf_handle;
3065 static void allocate_tempbuf(void)
3066 {
3067 /* Yeah, malloc would be really nice now :) */
3068 #ifdef __PCTOOL__
3069 tempbuf_size = 32*1024*1024;
3070 tempbuf = malloc(tempbuf_size);
3071 #else
3072 tempbuf_handle = core_alloc_maximum("tc tempbuf", &tempbuf_size, NULL);
3073 tempbuf = core_get_data(tempbuf_handle);
3074 #endif
3075 }
3076
3077 static void free_tempbuf(void)
3078 {
3079 if (tempbuf_size == 0)
3080 return ;
3081
3082 #ifdef __PCTOOL__
3083 free(tempbuf);
3084 #else
3085 core_free(tempbuf_handle);
3086 #endif
3087 tempbuf = NULL;
3088 tempbuf_size = 0;
3089 }
3090
3091 #ifndef __PCTOOL__
3092
3093 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3094 {
3095 struct index_entry idx;
3096
3097 if (!tc_stat.ready)
3098 return false;
3099
3100 if (!TAGCACHE_IS_NUMERIC(tag))
3101 return false;
3102
3103 if (!get_index(masterfd, idx_id, &idx, false))
3104 return false;
3105
3106 idx.tag_seek[tag] = data;
3107 idx.flag |= FLAG_DIRTYNUM;
3108
3109 return write_index(masterfd, idx_id, &idx);
3110 }
3111
3112 #if 0
3113 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3114 int tag, long data)
3115 {
3116 struct master_header myhdr;
3117
3118 if (tcs->masterfd < 0)
3119 {
3120 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3121 return false;
3122 }
3123
3124 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3125 }
3126 #endif
3127
3128 static bool command_queue_is_full(void)
3129 {
3130 int next;
3131
3132 next = command_queue_widx + 1;
3133 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3134 next = 0;
3135
3136 return (next == command_queue_ridx);
3137 }
3138
3139 static void command_queue_sync_callback(void *data)
3140 {
3141 (void)data;
3142 struct master_header myhdr;
3143 int masterfd;
3144
3145 mutex_lock(&command_queue_mutex);
3146
3147 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3148 return;
3149
3150 while (command_queue_ridx != command_queue_widx)
3151 {
3152 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3153
3154 switch (ce->command)
3155 {
3156 case CMD_UPDATE_MASTER_HEADER:
3157 {
3158 close(masterfd);
3159 update_master_header();
3160
3161 /* Re-open the masterfd. */
3162 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3163 return;
3164
3165 break;
3166 }
3167 case CMD_UPDATE_NUMERIC:
3168 {
3169 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3170 break;
3171 }
3172 }
3173
3174 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3175 command_queue_ridx = 0;
3176 }
3177
3178 close(masterfd);
3179
3180 tc_stat.queue_length = 0;
3181 mutex_unlock(&command_queue_mutex);
3182 }
3183
3184 static void run_command_queue(bool force)
3185 {
3186 if (COMMAND_QUEUE_IS_EMPTY)
3187 return;
3188
3189 if (force || command_queue_is_full())
3190 command_queue_sync_callback(NULL);
3191 else
3192 register_storage_idle_func(command_queue_sync_callback);
3193 }
3194
3195 static void queue_command(int cmd, long idx_id, int tag, long data)
3196 {
3197 while (1)
3198 {
3199 int next;
3200
3201 mutex_lock(&command_queue_mutex);
3202 next = command_queue_widx + 1;
3203 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3204 next = 0;
3205
3206 /* Make sure queue is not full. */
3207 if (next != command_queue_ridx)
3208 {
3209 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3210
3211 ce->command = cmd;
3212 ce->idx_id = idx_id;
3213 ce->tag = tag;
3214 ce->data = data;
3215
3216 command_queue_widx = next;
3217
3218 tc_stat.queue_length++;
3219
3220 mutex_unlock(&command_queue_mutex);
3221 break;
3222 }
3223
3224 /* Queue is full, try again later... */
3225 mutex_unlock(&command_queue_mutex);
3226 sleep(1);
3227 }
3228 }
3229
3230 long tagcache_increase_serial(void)
3231 {
3232 long old;
3233
3234 if (!tc_stat.ready)
3235 return -2;
3236
3237 while (read_lock)
3238 sleep(1);
3239
3240 old = current_tcmh.serial++;
3241 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3242
3243 return old;
3244 }
3245
3246 void tagcache_update_numeric(int idx_id, int tag, long data)
3247 {
3248 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3249 }
3250 #endif /* !__PCTOOL__ */
3251
3252 long tagcache_get_serial(void)
3253 {
3254 return current_tcmh.serial;
3255 }
3256
3257 long tagcache_get_commitid(void)
3258 {
3259 return current_tcmh.commitid;
3260 }
3261
3262 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3263 {
3264 char buf[512];
3265 int i;
3266
3267 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3268 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3269 {
3270 if (*datastr == '\0')
3271 break;
3272
3273 if (*datastr == '"' || *datastr == '\\')
3274 buf[i++] = '\\';
3275
3276 else if (*datastr == '\n')
3277 {
3278 buf[i++] = '\\';
3279 buf[i] = 'n';
3280 continue;
3281 }
3282
3283 buf[i] = *(datastr++);
3284 }
3285
3286 strcpy(&buf[i], "\" ");
3287
3288 write(fd, buf, i + 2);
3289
3290 return true;
3291 }
3292
3293 #ifndef __PCTOOL__
3294
3295 static bool read_tag(char *dest, long size,
3296 const char *src, const char *tagstr)
3297 {
3298 int pos;
3299 char current_tag[32];
3300
3301 while (*src != '\0')
3302 {
3303 /* Skip all whitespace */
3304 while (*src == ' ')
3305 src++;
3306
3307 if (*src == '\0')
3308 break;
3309
3310 pos = 0;
3311 /* Read in tag name */
3312 while (*src != '=' && *src != ' ')
3313 {
3314 current_tag[pos] = *src;
3315 src++;
3316 pos++;
3317
3318 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3319 return false;
3320 }
3321 current_tag[pos] = '\0';
3322
3323 /* Read in tag data */
3324
3325 /* Find the start. */
3326 while (*src != '"' && *src != '\0')
3327 src++;
3328
3329 if (*src == '\0' || *(++src) == '\0')
3330 return false;
3331
3332 /* Read the data. */
3333 for (pos = 0; pos < size; pos++)
3334 {
3335 if (*src == '\0')
3336 break;
3337
3338 if (*src == '\\')
3339 {
3340 src++;
3341 if (*src == 'n')
3342 dest[pos] = '\n';
3343 else
3344 dest[pos] = *src;
3345
3346 src++;
3347 continue;
3348 }
3349
3350 if (*src == '\0')
3351 break;
3352
3353 if (*src == '"')
3354 {
3355 src++;
3356 break;
3357 }
3358
3359 dest[pos] = *(src++);
3360 }
3361
3362 dest[pos] = '\0';
3363
3364 if (!strcasecmp(tagstr, current_tag))
3365 return true;
3366 }
3367
3368 return false;
3369 }
3370
3371 static int parse_changelog_line(int line_n, char *buf, void *parameters)
3372 {
3373 struct index_entry idx;
3374 char tag_data[TAG_MAXLEN+32];
3375 int idx_id;
3376 long masterfd = (long)parameters;
3377 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime,
3378 tag_lastplayed, tag_commitid, tag_lastoffset };
3379 int i;
3380 (void)line_n;
3381
3382 if (*buf == '#')
3383 return 0;
3384
3385 /* logf("%d/%s", line_n, buf); */
3386 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3387 {
3388 logf("%d/filename missing", line_n);
3389 logf("-> %s", buf);
3390 return 0;
3391 }
3392
3393 idx_id = find_index(tag_data);
3394 if (idx_id < 0)
3395 {
3396 logf("%d/entry not found", line_n);
3397 return 0;
3398 }
3399
3400 if (!get_index(masterfd, idx_id, &idx, false))
3401 {
3402 logf("%d/failed to retrieve index entry", line_n);
3403 return 0;
3404 }
3405
3406 /* Stop if tag has already been modified. */
3407 if (idx.flag & FLAG_DIRTYNUM)
3408 return 0;
3409
3410 logf("%d/import: %s", line_n, tag_data);
3411
3412 idx.flag |= FLAG_DIRTYNUM;
3413 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3414 {
3415 int data;
3416
3417 if (!read_tag(tag_data, sizeof tag_data, buf,
3418 tagcache_tag_to_str(import_tags[i])))
3419 {
3420 continue;
3421 }
3422
3423 data = atoi(tag_data);
3424 if (data < 0)
3425 continue;
3426
3427 idx.tag_seek[import_tags[i]] = data;
3428
3429 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3430 current_tcmh.serial = data + 1;
3431 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3432 current_tcmh.commitid = data + 1;
3433 }
3434
3435 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3436 }
3437
3438 bool tagcache_import_changelog(void)
3439 {
3440 struct master_header myhdr;
3441 struct tagcache_header tch;
3442 int clfd;
3443 long masterfd;
3444 char buf[2048];
3445
3446 if (!tc_stat.ready)
3447 return false;
3448
3449 while (read_lock)
3450 sleep(1);
3451
3452 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3453 if (clfd < 0)
3454 {
3455 logf("failure to open changelog");
3456 return false;
3457 }
3458
3459 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3460 {
3461 close(clfd);
3462 return false;
3463 }
3464
3465 write_lock++;
3466
3467 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3468
3469 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3470 parse_changelog_line);
3471
3472 close(clfd);
3473 close(masterfd);
3474
3475 if (filenametag_fd >= 0)
3476 {
3477 close(filenametag_fd);
3478 filenametag_fd = -1;
3479 }
3480
3481 write_lock--;
3482
3483 update_master_header();
3484
3485 return true;
3486 }
3487
3488 #endif /* !__PCTOOL__ */
3489
3490 bool tagcache_create_changelog(struct tagcache_search *tcs)
3491 {
3492 struct master_header myhdr;
3493 struct index_entry idx;
3494 char buf[TAG_MAXLEN+32];
3495 char temp[32];
3496 int clfd;
3497 int i, j;
3498
3499 if (!tc_stat.ready)
3500 return false;
3501
3502 if (!tagcache_search(tcs, tag_filename))
3503 return false;
3504
3505 /* Initialize the changelog */
3506 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3507 if (clfd < 0)
3508 {
3509 logf("failure to open changelog");
3510 return false;
3511 }
3512
3513 if (tcs->masterfd < 0)
3514 {
3515 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3516 return false;
3517 }
3518 else
3519 {
3520 lseek(tcs->masterfd, 0, SEEK_SET);
3521 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3522 }
3523
3524 write(clfd, "## Changelog version 1\n", 23);
3525
3526 for (i = 0; i < myhdr.tch.entry_count; i++)
3527 {
3528 if (ecread_index_entry(tcs->masterfd, &idx) != sizeof(struct index_entry))
3529 {
3530 logf("read error #9");
3531 tagcache_search_finish(tcs);
3532 close(clfd);
3533 return false;
3534 }
3535
3536 /* Skip until the entry found has been modified. */
3537 if (! (idx.flag & FLAG_DIRTYNUM) )
3538 continue;
3539
3540 /* Skip deleted entries too. */
3541 if (idx.flag & FLAG_DELETED)
3542 continue;
3543
3544 /* Now retrieve all tags. */
3545 for (j = 0; j < TAG_COUNT; j++)
3546 {
3547 if (TAGCACHE_IS_NUMERIC(j))
3548 {
3549 snprintf(temp, sizeof temp, "%d", (int)idx.tag_seek[j]);
3550 write_tag(clfd, tagcache_tag_to_str(j), temp);
3551 continue;
3552 }
3553
3554 tcs->type = j;
3555 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3556 write_tag(clfd, tagcache_tag_to_str(j), buf);
3557 }
3558
3559 write(clfd, "\n", 1);
3560 do_timed_yield();
3561 }
3562
3563 close(clfd);
3564
3565 tagcache_search_finish(tcs);
3566
3567 return true;
3568 }
3569
3570 static bool delete_entry(long idx_id)
3571 {
3572 int fd = -1;
3573 int masterfd = -1;
3574 int tag, i;
3575 struct index_entry idx, myidx;
3576 struct master_header myhdr;
3577 char buf[TAG_MAXLEN+32];
3578 int in_use[TAG_COUNT];
3579
3580 logf("delete_entry(): %ld", idx_id);
3581
3582 #ifdef HAVE_TC_RAMCACHE
3583 /* At first mark the entry removed from ram cache. */
3584 if (tc_stat.ramcache)
3585 ramcache_hdr->indices[idx_id].flag |= FLAG_DELETED;
3586 #endif
3587
3588 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3589 return false;
3590
3591 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3592 if (ecread_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3593 {
3594 logf("delete_entry(): read error");
3595 goto cleanup;
3596 }
3597
3598 if (myidx.flag & FLAG_DELETED)
3599 {
3600 logf("delete_entry(): already deleted!");
3601 goto cleanup;
3602 }
3603
3604 myidx.flag |= FLAG_DELETED;
3605 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3606 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3607 {
3608 logf("delete_entry(): write_error #1");
3609 goto cleanup;
3610 }
3611
3612 /* Now check which tags are no longer in use (if any) */
3613 for (tag = 0; tag < TAG_COUNT; tag++)
3614 in_use[tag] = 0;
3615
3616 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3617 for (i = 0; i < myhdr.tch.entry_count; i++)
3618 {
3619 struct index_entry *idxp;
3620
3621 #ifdef HAVE_TC_RAMCACHE
3622 /* Use RAM DB if available for greater speed */
3623 if (tc_stat.ramcache)
3624 idxp = &ramcache_hdr->indices[i];
3625 else
3626 #endif
3627 {
3628 if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry))
3629 {
3630 logf("delete_entry(): read error #2");
3631 goto cleanup;
3632 }
3633 idxp = &idx;
3634 }
3635
3636 if (idxp->flag & FLAG_DELETED)
3637 continue;
3638
3639 for (tag = 0; tag < TAG_COUNT; tag++)
3640 {
3641 if (TAGCACHE_IS_NUMERIC(tag))
3642 continue;
3643
3644 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3645 in_use[tag]++;
3646 }
3647 }
3648
3649 /* Now delete all tags no longer in use. */
3650 for (tag = 0; tag < TAG_COUNT; tag++)
3651 {
3652 struct tagcache_header tch;
3653 int oldseek = myidx.tag_seek[tag];
3654
3655 if (TAGCACHE_IS_NUMERIC(tag))
3656 continue;
3657
3658 /**
3659 * Replace tag seek with a hash value of the field string data.
3660 * That way runtime statistics of moved or altered files can be
3661 * resurrected.
3662 */
3663 #ifdef HAVE_TC_RAMCACHE
3664 if (tc_stat.ramcache && tag != tag_filename)
3665 {
3666 struct tagfile_entry *tfe;
3667 int32_t *seek = &ramcache_hdr->indices[idx_id].tag_seek[tag];
3668
3669 tfe = (struct tagfile_entry *)&ramcache_hdr->tags[tag][*seek];
3670 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3671 myidx.tag_seek[tag] = *seek;
3672 }
3673 else
3674 #endif
3675 {
3676 struct tagfile_entry tfe;
3677
3678 /* Open the index file, which contains the tag names. */
3679 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3680 goto cleanup;
3681
3682 /* Skip the header block */
3683 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3684 if (ecread_tagfile_entry(fd, &tfe) != sizeof(struct tagfile_entry))
3685 {
3686 logf("delete_entry(): read error #3");
3687 goto cleanup;
3688 }
3689
3690 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3691 {
3692 logf("delete_entry(): read error #4");
3693 goto cleanup;
3694 }
3695
3696 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3697 }
3698
3699 if (in_use[tag])
3700 {
3701 logf("in use: %d/%d", tag, in_use[tag]);
3702 if (fd >= 0)
3703 {
3704 close(fd);
3705 fd = -1;
3706 }
3707 continue;
3708 }
3709
3710 #ifdef HAVE_TC_RAMCACHE
3711 /* Delete from ram. */
3712 if (tc_stat.ramcache && tag != tag_filename)
3713 {
3714 struct tagfile_entry *tagentry =
3715 (struct tagfile_entry *)&ramcache_hdr->tags[tag][oldseek];
3716 tagentry->tag_data[0] = '\0';
3717 }
3718 #endif
3719
3720 /* Open the index file, which contains the tag names. */
3721 if (fd < 0)
3722 {
3723 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3724 goto cleanup;
3725 }
3726
3727 /* Skip the header block */
3728 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3729
3730 /* Debug, print 10 first characters of the tag
3731 read(fd, buf, 10);
3732 buf[10]='\0';
3733 logf("TAG:%s", buf);
3734 lseek(fd, -10, SEEK_CUR);
3735 */
3736
3737 /* Write first data byte in tag as \0 */
3738 write(fd, "", 1);
3739
3740 /* Now tag data has been removed */
3741 close(fd);
3742 fd = -1;
3743 }
3744
3745 /* Write index entry back into master index. */
3746 lseek(masterfd, sizeof(struct master_header) +
3747 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3748 if (ecwrite_index_entry(masterfd, &myidx) != sizeof(struct index_entry))
3749 {
3750 logf("delete_entry(): write_error #2");
3751 goto cleanup;
3752 }
3753
3754 close(masterfd);
3755
3756 return true;
3757
3758 cleanup:
3759 if (fd >= 0)
3760 close(fd);
3761 if (masterfd >= 0)
3762 close(masterfd);
3763
3764 return false;
3765 }
3766
3767 #ifndef __PCTOOL__
3768 /**
3769 * Returns true if there is an event waiting in the queue
3770 * that requires the current operation to be aborted.
3771 */
3772 static bool check_event_queue(void)
3773 {
3774 struct queue_event ev;
3775
3776 if(!queue_peek(&tagcache_queue, &ev))
3777 return false;
3778
3779 switch (ev.id)
3780 {
3781 case Q_STOP_SCAN:
3782 case SYS_POWEROFF:
3783 case SYS_USB_CONNECTED:
3784 return true;
3785 }
3786
3787 return false;
3788 }
3789 #endif
3790
3791 #ifdef HAVE_TC_RAMCACHE
3792 static bool allocate_tagcache(void)
3793 {
3794 struct master_header tcmh;
3795 int fd;
3796
3797 /* Load the header. */
3798 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3799 {
3800 ramcache_hdr = NULL;
3801 return false;
3802 }
3803
3804 close(fd);
3805
3806 /**
3807 * Now calculate the required cache size plus
3808 * some extra space for alignment fixes.
3809 */
3810 tc_stat.ramcache_allocated = tcmh.tch.datasize + 256 + TAGCACHE_RESERVE +
3811 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3812 int handle = core_alloc("tc ramcache", tc_stat.ramcache_allocated);
3813 ramcache_hdr = core_get_data(handle);
3814 memset(ramcache_hdr, 0, sizeof(struct ramcache_header));
3815 memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3816 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3817
3818 return true;
3819 }
3820
3821 # ifdef HAVE_EEPROM_SETTINGS
3822 static bool tagcache_dumpload(void)
3823 {
3824 struct statefile_header shdr;
3825 int fd, rc;
3826 long offpos;
3827 int i, handle;
3828
3829 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3830 if (fd < 0)
3831 {
3832 logf("no tagcache statedump");
3833 return false;
3834 }
3835
3836 /* Check the statefile memory placement */
3837 rc = read(fd, &shdr, sizeof(struct statefile_header));
3838 if (rc != sizeof(struct statefile_header)
3839 || shdr.magic != TAGCACHE_STATEFILE_MAGIC
3840 || shdr.mh.tch.magic != TAGCACHE_MAGIC)
3841 {
3842 logf("incorrect statefile");
3843 ramcache_hdr = NULL;
3844 close(fd);
3845 return false;
3846 }
3847
3848
3849 /* Lets allocate real memory and load it */
3850 handle = core_alloc("tc ramcache", shdr.tc_stat.ramcache_allocated);
3851 ramcache_hdr = core_get_data(handle);
3852 rc = read(fd, ramcache_hdr, shdr.tc_stat.ramcache_allocated);
3853 close(fd);
3854
3855 offpos = (long)ramcache_hdr - (long)shdr.hdr;
3856 if (rc != shdr.tc_stat.ramcache_allocated)
3857 {
3858 logf("read failure!");
3859 ramcache_hdr = NULL;
3860 return false;
3861 }
3862
3863 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3864
3865 /* Now fix the pointers */
3866 for (i = 0; i < TAG_COUNT; i++)
3867 ramcache_hdr->tags[i] += offpos;
3868
3869 /* Load the tagcache master header (should match the actual DB file header). */
3870 memcpy(&current_tcmh, &shdr.mh, sizeof current_tcmh);
3871
3872 return true;
3873 }
3874
3875 static bool tagcache_dumpsave(void)
3876 {
3877 struct statefile_header shdr;
3878 int fd;
3879
3880 if (!tc_stat.ramcache)
3881 return false;
3882
3883 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC, 0666);
3884 if (fd < 0)
3885 {
3886 logf("failed to create a statedump");
3887 return false;
3888 }
3889
3890 /* Create the header */
3891 shdr.magic = TAGCACHE_STATEFILE_MAGIC;
3892 shdr.hdr = ramcache_hdr;
3893 memcpy(&shdr.mh, &current_tcmh, sizeof current_tcmh);
3894 memcpy(&shdr.tc_stat, &tc_stat, sizeof tc_stat);
3895 write(fd, &shdr, sizeof shdr);
3896
3897 /* And dump the data too */
3898 write(fd, ramcache_hdr, tc_stat.ramcache_allocated);
3899 close(fd);
3900
3901 return true;
3902 }
3903 # endif
3904
3905 static bool load_tagcache(void)
3906 {
3907 struct tagcache_header *tch;
3908 struct master_header tcmh;
3909 long bytesleft = tc_stat.ramcache_allocated - sizeof(struct ramcache_header);
3910 struct index_entry *idx;
3911 int rc, fd;
3912 char *p;
3913 int i, tag;
3914
3915 # ifdef HAVE_DIRCACHE
3916 while (dircache_is_initializing())
3917 sleep(1);
3918
3919 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3920 # endif
3921
3922 logf("loading tagcache to ram...");
3923
3924 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3925 if (fd < 0)
3926 {
3927 logf("tagcache open failed");
3928 return false;
3929 }
3930
3931 if (ecread(fd, &tcmh, 1, master_header_ec, tc_stat.econ)
3932 != sizeof(struct master_header)
3933 || tcmh.tch.magic != TAGCACHE_MAGIC)
3934 {
3935 logf("incorrect header");
3936 return false;
3937 }
3938
3939 /* Master header copy should already match, this can be redundant to do. */
3940 memcpy(&current_tcmh, &tcmh, sizeof current_tcmh);
3941
3942 idx = ramcache_hdr->indices;
3943
3944 /* Load the master index table. */
3945 for (i = 0; i < tcmh.tch.entry_count; i++)
3946 {
3947 bytesleft -= sizeof(struct index_entry);
3948 if (bytesleft < 0)
3949 {
3950 logf("too big tagcache.");
3951 close(fd);
3952 return false;
3953 }
3954
3955 /* DEBUG: After tagcache commit and dircache rebuild, hdr-sturcture
3956 * may become corrupt. */
3957 rc = ecread_index_entry(fd, idx);
3958 if (rc != sizeof(struct index_entry))
3959 {
3960 logf("read error #10");
3961 close(fd);
3962 return false;
3963 }
3964
3965 idx++;
3966 }
3967
3968 close(fd);
3969
3970 /* Load the tags. */
3971 p = (char *)idx;
3972 for (tag = 0; tag < TAG_COUNT; tag++)
3973 {
3974 struct tagfile_entry *fe;
3975 char buf[TAG_MAXLEN+32];
3976
3977 if (TAGCACHE_IS_NUMERIC(tag))
3978 continue ;
3979
3980 //p = ((void *)p+1);
3981 p = (char *)((long)p & ~0x03) + 0x04;
3982 ramcache_hdr->tags[tag] = p;
3983
3984 /* Check the header. */
3985 tch = (struct tagcache_header *)p;
3986 p += sizeof(struct tagcache_header);
3987
3988 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3989 return false;
3990
3991 for (ramcache_hdr->entry_count[tag] = 0;
3992 ramcache_hdr->entry_count[tag] < tch->entry_count;
3993 ramcache_hdr->entry_count[tag]++)
3994 {
3995 long pos;
3996
3997 if (do_timed_yield())
3998 {
3999 /* Abort if we got a critical event in queue */
4000 if (check_event_queue())
4001 return false;
4002 }
4003
4004 fe = (struct tagfile_entry *)p;
4005 pos = lseek(fd, 0, SEEK_CUR);
4006 rc = ecread_tagfile_entry(fd, fe);
4007 if (rc != sizeof(struct tagfile_entry))
4008 {
4009 /* End of lookup table. */
4010 logf("read error #11");
4011 close(fd);
4012 return false;
4013 }
4014
4015 /* We have a special handling for the filename tags. */
4016 if (tag == tag_filename)
4017 {
4018 # ifdef HAVE_DIRCACHE
4019 int dc;
4020 # endif
4021
4022 idx = &ramcache_hdr->indices[fe->idx_id];
4023
4024 if (fe->tag_length >= (long)sizeof(buf)-1)
4025 {
4026 read(fd, buf, 10);
4027 buf[10] = '\0';
4028 logf("TAG:%s", buf);
4029 logf("too long filename");
4030 close(fd);
4031 return false;
4032 }
4033
4034 rc = read(fd, buf, fe->tag_length);
4035 if (rc != fe->tag_length)
4036 {
4037 logf("read error #12");
4038 close(fd);
4039 return false;
4040 }
4041
4042 /* Check if the entry has already been removed */
4043 if (idx->flag & FLAG_DELETED)
4044 continue;
4045
4046 /* This flag must not be used yet. */
4047 if (idx->flag & FLAG_DIRCACHE)
4048 {
4049 logf("internal error!");
4050 close(fd);
4051 return false;
4052 }
4053
4054 if (idx->tag_seek[tag] != pos)
4055 {
4056 logf("corrupt data structures!");
4057 close(fd);
4058 return false;
4059 }
4060
4061 # ifdef HAVE_DIRCACHE
4062 if (dircache_is_enabled())
4063 {
4064 dc = dircache_get_entry_id(buf);
4065 if (dc < 0)
4066 {
4067 logf("Entry no longer valid.");
4068 logf("-> %s", buf);
4069 if (global_settings.tagcache_autoupdate)
4070 delete_entry(fe->idx_id);
4071 continue ;
4072 }
4073
4074 idx->flag |= FLAG_DIRCACHE;
4075 idx->tag_seek[tag_filename] = dc;
4076 }
4077 else
4078 # endif
4079 {
4080 /* This will be very slow unless dircache is enabled
4081 or target is flash based, but do it anyway for
4082 consistency. */
4083 /* Check if entry has been removed. */
4084 if (global_settings.tagcache_autoupdate)
4085 {
4086 if (!file_exists(buf))
4087 {
4088 logf("Entry no longer valid.");
4089 logf("-> %s", buf);
4090 delete_entry(fe->idx_id);
4091 continue;
4092 }
4093 }
4094 }
4095
4096 continue ;
4097 }
4098
4099 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
4100 if (bytesleft < 0)
4101 {
4102 logf("too big tagcache #2");
4103 logf("tl: %ld", fe->tag_length);
4104 logf("bl: %ld", bytesleft);
4105 close(fd);
4106 return false;
4107 }
4108
4109 p = fe->tag_data;
4110 rc = read(fd, fe->tag_data, fe->tag_length);
4111 p += rc;
4112
4113 if (rc != fe->tag_length)
4114 {
4115 logf("read error #13");
4116 logf("rc=0x%04x", rc); // 0x431
4117 logf("len=0x%04lx", fe->tag_length); // 0x4000
4118 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4119 logf("tag=0x%02x", tag); // 0x00
4120 close(fd);
4121 return false;
4122 }
4123 }
4124 close(fd);
4125 }
4126
4127 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4128 logf("tagcache loaded into ram!");
4129
4130 return true;
4131 }
4132 #endif /* HAVE_TC_RAMCACHE */
4133
4134 static bool check_deleted_files(void)
4135 {
4136 int fd;
4137 char buf[TAG_MAXLEN+32];
4138 struct tagfile_entry tfe;
4139
4140 logf("reverse scan...");
4141 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4142 fd = open(buf, O_RDONLY);
4143
4144 if (fd < 0)
4145 {
4146 logf("%s open fail", buf);
4147 return false;
4148 }
4149
4150 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4151 while (ecread_tagfile_entry(fd, &tfe) == sizeof(struct tagfile_entry)
4152 #ifndef __PCTOOL__
4153 && !check_event_queue()
4154 #endif
4155 )
4156 {
4157 if (tfe.tag_length >= (long)sizeof(buf)-1)
4158 {
4159 logf("too long tag");
4160 close(fd);
4161 return false;
4162 }
4163
4164 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4165 {
4166 logf("read error #14");
4167 close(fd);
4168 return false;
4169 }
4170
4171 /* Check if the file has already deleted from the db. */
4172 if (*buf == '\0')
4173 continue;
4174
4175 /* Now check if the file exists. */
4176 if (!file_exists(buf))
4177 {
4178 logf("Entry no longer valid.");
4179 logf("-> %s / %ld", buf, tfe.tag_length);
4180 delete_entry(tfe.idx_id);
4181 }
4182 }
4183
4184 close(fd);
4185
4186 logf("done");
4187
4188 return true;
4189 }
4190
4191
4192 /* Note that this function must not be inlined, otherwise the whole point
4193 * of having the code in a separate function is lost.
4194 */
4195 static void __attribute__ ((noinline)) check_ignore(const char *dirname,
4196 int *ignore, int *unignore)
4197 {
4198 char newpath[MAX_PATH];
4199
4200 /* check for a database.ignore file */
4201 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4202 *ignore = file_exists(newpath);
4203 /* check for a database.unignore file */
4204 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4205 *unignore = file_exists(newpath);
4206 }
4207
4208 static struct search_roots_ll {
4209 const char *path;
4210 struct search_roots_ll * next;
4211 } roots_ll;
4212
4213 #ifdef APPLICATION
4214 /*
4215 * This adds a path to the search roots, possibly during traveling through
4216 * the filesystem. It only adds if the path is not inside an already existing
4217 * search root.
4218 *
4219 * Returns true if it added the path to the search roots
4220 *
4221 * Windows 2000 and greater supports symlinks, but they don't provide
4222 * realpath() or readlink(), and symlinks are rarely used on them so
4223 * ignore this for windows for now
4224 **/
4225 static bool add_search_root(const char *name)
4226 {
4227 (void)name;
4228 #ifndef WIN32
4229 struct search_roots_ll *this, *prev = NULL;
4230 char target[MAX_PATH];
4231 /* Okay, realpath() is almost completely broken on android
4232 *
4233 * It doesn't accept NULL for resolved_name to dynamically allocate
4234 * the resulting path; and it assumes resolved_name to be PATH_MAX
4235 * (actually MAXPATHLEN, but it's the same [as of 2.3]) long
4236 * and blindly writes to the end if it
4237 *
4238 * therefore use sufficiently large static storage here
4239 * Note that PATH_MAX != MAX_PATH
4240 **/
4241 static char abs_target[PATH_MAX];
4242 ssize_t len;
4243
4244 len = readlink(name, target, sizeof(target));
4245 if (len < 0)
4246 return false;
4247
4248 target[len] = '\0';
4249 if (realpath(target, abs_target) == NULL)
4250 return false;
4251
4252 for(this = &roots_ll; this; prev = this, this = this->next)
4253 {
4254 size_t root_len = strlen(this->path);
4255 /* check if the link target is inside of an existing search root
4256 * don't add if target is inside, we'll scan it later */
4257 if (!strncmp(this->path, abs_target, root_len))
4258 return false;
4259 }
4260
4261 if (prev)
4262 {
4263 size_t len = strlen(abs_target) + 1; /* count \0 */
4264 this = malloc(sizeof(struct search_roots_ll) + len );
4265 if (!this || len > MAX_PATH)
4266 {
4267 logf("Error at adding a search root: %s", this ? "path too long":"OOM");
4268 free(this);
4269 prev->next = NULL;
4270 return false;
4271 }
4272 this->path = ((char*)this) + sizeof(struct search_roots_ll);
4273 strcpy((char*)this->path, abs_target); /* ok to cast const away here */
4274 this->next = NULL;
4275 prev->next = this;
4276 logf("Added %s to the search roots\n", abs_target);
4277 return true;
4278 }
4279 #endif
4280 return false;
4281 }
4282
4283 static int free_search_roots(struct search_roots_ll * start)
4284 {
4285 int ret = 0;
4286 if (start->next)
4287 {
4288 ret += free_search_roots(start->next);
4289 ret += sizeof(struct search_roots_ll);
4290 free(start->next);
4291 }
4292 return ret;
4293 }
4294 #else /* native, simulator */
4295 #define add_search_root(a) do {} while(0)
4296 #define free_search_roots(a) do {} while(0)
4297 #endif
4298
4299 static bool check_dir(const char *dirname, int add_files)
4300 {
4301 DIR *dir;
4302 int len;
4303 int success = false;
4304 int ignore, unignore;
4305
4306 dir = opendir(dirname);
4307 if (!dir)
4308 {
4309 logf("tagcache: opendir(%s) failed", dirname);
4310 return false;
4311 }
4312 /* check for a database.ignore and database.unignore */
4313 check_ignore(dirname, &ignore, &unignore);
4314
4315 /* don't do anything if both ignore and unignore are there */
4316 if (ignore != unignore)
4317 add_files = unignore;
4318
4319 /* Recursively scan the dir. */
4320 #ifdef __PCTOOL__
4321 while (1)
4322 #else
4323 while (!check_event_queue())
4324 #endif
4325 {
4326 struct dirent *entry = readdir(dir);
4327 if (entry == NULL)
4328 {
4329 success = true;
4330 break;
4331 }
4332
4333 if (!strcmp((char *)entry->d_name, ".") ||
4334 !strcmp((char *)entry->d_name, ".."))
4335 continue;
4336
4337 struct dirinfo info = dir_get_info(dir, entry);
4338
4339 yield();
4340
4341 len = strlen(curpath);
4342 /* don't add an extra / for curpath == / */
4343 if (len <= 1) len = 0;
4344 snprintf(&curpath[len], sizeof(curpath) - len, "/%s", entry->d_name);
4345
4346 processed_dir_count++;
4347 if (info.attribute & ATTR_DIRECTORY)
4348 #ifndef SIMULATOR
4349 { /* don't follow symlinks to dirs, but try to add it as a search root
4350 * this makes able to avoid looping in recursive symlinks */
4351 if (info.attribute & ATTR_LINK)
4352 add_search_root(curpath);
4353 else
4354 check_dir(curpath, add_files);
4355 }
4356 #else
4357 check_dir(curpath, add_files);
4358 #endif
4359 else if (add_files)
4360 {
4361 tc_stat.curentry = curpath;
4362
4363 /* Add a new entry to the temporary db file. */
4364 add_tagcache(curpath, (info.wrtdate << 16) | info.wrttime
4365 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4366 , dir->internal_entry
4367 #endif
4368 );
4369
4370 /* Wait until current path for debug screen is read and unset. */
4371 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4372 yield();
4373
4374 tc_stat.curentry = NULL;
4375 }
4376
4377 curpath[len] = '\0';
4378 }
4379
4380 closedir(dir);
4381
4382 return success;
4383 }
4384
4385 void tagcache_screensync_event(void)
4386 {
4387 tc_stat.curentry = NULL;
4388 }
4389
4390 void tagcache_screensync_enable(bool state)
4391 {
4392 tc_stat.syncscreen = state;
4393 }
4394
4395 void tagcache_build(const char *path)
4396 {
4397 struct tagcache_header header;
4398 bool ret;
4399
4400 curpath[0] = '\0';
4401 data_size = 0;
4402 total_entry_count = 0;
4403 processed_dir_count = 0;
4404
4405 #ifdef HAVE_DIRCACHE
4406 while (dircache_is_initializing())
4407 sleep(1);
4408 #endif
4409
4410 logf("updating tagcache");
4411
4412 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4413 if (cachefd >= 0)
4414 {
4415 logf("skipping, cache already waiting for commit");
4416 close(cachefd);
4417 return ;
4418 }
4419
4420 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC, 0666);
4421 if (cachefd < 0)
4422 {
4423 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4424 return ;
4425 }
4426
4427 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4428
4429 cpu_boost(true);
4430
4431 logf("Scanning files...");
4432 /* Scan for new files. */
4433 memset(&header, 0, sizeof(struct tagcache_header));
4434 write(cachefd, &header, sizeof(struct tagcache_header));
4435
4436 ret = true;
4437 roots_ll.path = path;
4438 roots_ll.next = NULL;
4439 struct search_roots_ll * this;
4440 /* check_dir might add new roots */
4441 for(this = &roots_ll; this; this = this->next)
4442 {
4443 strcpy(curpath, this->path);
4444 ret = ret && check_dir(this->path, true);
4445 }
4446 if (roots_ll.next)
4447 free_search_roots(roots_ll.next);
4448
4449 /* Write the header. */
4450 header.magic = TAGCACHE_MAGIC;
4451 header.datasize = data_size;
4452 header.entry_count = total_entry_count;
4453 lseek(cachefd, 0, SEEK_SET);
4454 write(cachefd, &header, sizeof(struct tagcache_header));
4455 close(cachefd);
4456
4457 if (filenametag_fd >= 0)
4458 {
4459 close(filenametag_fd);
4460 filenametag_fd = -1;
4461 }
4462
4463 if (!ret)
4464 {
4465 logf("Aborted.");
4466 cpu_boost(false);
4467 return ;
4468 }
4469
4470 /* Commit changes to the database. */
4471 #ifdef __PCTOOL__
4472 allocate_tempbuf();
4473 #endif
4474 if (commit())
4475 {
4476 logf("tagcache built!");
4477 }
4478 #ifdef __PCTOOL__
4479 free_tempbuf();
4480 #endif
4481
4482 #ifdef HAVE_TC_RAMCACHE
4483 if (ramcache_hdr)
4484 {
4485 /* Import runtime statistics if we just initialized the db. */
4486 if (current_tcmh.serial == 0)
4487 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4488 }
4489 #endif
4490
4491 cpu_boost(false);
4492 }
4493
4494 #ifdef HAVE_TC_RAMCACHE
4495 static void load_ramcache(void)
4496 {
4497 if (!ramcache_hdr)
4498 return ;
4499
4500 cpu_boost(true);
4501
4502 /* At first we should load the cache (if exists). */
4503 tc_stat.ramcache = load_tagcache();
4504
4505 if (!tc_stat.ramcache)
4506 {
4507 /* If loading failed, it must indicate some problem with the db
4508 * so disable it entirely to prevent further issues. */
4509 tc_stat.ready = false;
4510 ramcache_hdr = NULL;
4511 }
4512
4513 cpu_boost(false);
4514 }
4515
4516 void tagcache_unload_ramcache(void)
4517 {
4518 tc_stat.ramcache = false;
4519 /* Just to make sure there is no statefile present. */
4520 // remove(TAGCACHE_STATEFILE);
4521 }
4522 #endif
4523
4524 #ifndef __PCTOOL__
4525 static void tagcache_thread(void)
4526 {
4527 struct queue_event ev;
4528 bool check_done = false;
4529
4530 /* If the previous cache build/update was interrupted, commit
4531 * the changes first in foreground. */
4532 cpu_boost(true);
4533 allocate_tempbuf();
4534 commit();
4535 free_tempbuf();
4536
4537 #ifdef HAVE_TC_RAMCACHE
4538 # ifdef HAVE_EEPROM_SETTINGS
4539 if (firmware_settings.initialized && firmware_settings.disk_clean
4540 && global_settings.tagcache_ram)
4541 {
4542 check_done = tagcache_dumpload();
4543 }
4544
4545 remove(TAGCACHE_STATEFILE);
4546 # endif
4547
4548 /* Allocate space for the tagcache if found on disk. */
4549 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4550 allocate_tagcache();
4551 #endif
4552
4553 cpu_boost(false);
4554 tc_stat.initialized = true;
4555
4556 /* Don't delay bootup with the header check but do it on background. */
4557 if (!tc_stat.ready)
4558 {
4559 sleep(HZ);
4560 tc_stat.ready = check_all_headers();
4561 tc_stat.readyvalid = true;
4562 }
4563
4564 while (1)
4565 {
4566 run_command_queue(false);
4567
4568 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4569
4570 switch (ev.id)
4571 {
4572 case Q_IMPORT_CHANGELOG:
4573 tagcache_import_changelog();
4574 break;
4575
4576 case Q_REBUILD:
4577 remove_files();
4578 remove(TAGCACHE_FILE_TEMP);
4579 tagcache_build("/");
4580 break;
4581
4582 case Q_UPDATE:
4583 tagcache_build("/");
4584 #ifdef HAVE_TC_RAMCACHE
4585 load_ramcache();
4586 #endif
4587 check_deleted_files();
4588 break ;
4589
4590 case Q_START_SCAN:
4591 check_done = false;
4592 case SYS_TIMEOUT:
4593 if (check_done || !tc_stat.ready)
4594 break ;
4595
4596 #ifdef HAVE_TC_RAMCACHE
4597 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4598 {
4599 load_ramcache();
4600 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4601 tagcache_build("/");
4602 }
4603 else
4604 #endif
4605 if (global_settings.tagcache_autoupdate)
4606 {
4607 tagcache_build("/");
4608
4609 /* This will be very slow unless dircache is enabled
4610 or target is flash based, but do it anyway for
4611 consistency. */
4612 check_deleted_files();
4613 }
4614
4615 logf("tagcache check done");
4616
4617 check_done = true;
4618 break ;
4619
4620 case Q_STOP_SCAN:
4621 break ;
4622
4623 case SYS_POWEROFF:
4624 break ;
4625
4626 #if (CONFIG_PLATFORM & PLATFORM_NATIVE)
4627 case SYS_USB_CONNECTED:
4628 logf("USB: TagCache");
4629 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4630 usb_wait_for_disconnect(&tagcache_queue);
4631 break ;
4632 #endif
4633 }
4634 }
4635 }
4636
4637 bool tagcache_prepare_shutdown(void)
4638 {
4639 if (tagcache_get_commit_step() > 0)
4640 return false;
4641
4642 tagcache_stop_scan();
4643 while (read_lock || write_lock)
4644 sleep(1);
4645
4646 return true;
4647 }
4648
4649 void tagcache_shutdown(void)
4650 {
4651 /* Flush the command queue. */
4652 run_command_queue(true);
4653
4654 #ifdef HAVE_EEPROM_SETTINGS
4655 if (tc_stat.ramcache)
4656 tagcache_dumpsave();
4657 #endif
4658 }
4659
4660 static int get_progress(void)
4661 {
4662 int total_count = -1;
4663
4664 #ifdef HAVE_DIRCACHE
4665 if (dircache_is_enabled())
4666 {
4667 total_count = dircache_get_entry_count();
4668 }
4669 else
4670 #endif
4671 #ifdef HAVE_TC_RAMCACHE
4672 {
4673 if (ramcache_hdr && tc_stat.ramcache)
4674 total_count = current_tcmh.tch.entry_count;
4675 }
4676 #endif
4677
4678 if (total_count < 0)
4679 return -1;
4680
4681 return processed_dir_count * 100 / total_count;
4682 }
4683
4684 struct tagcache_stat* tagcache_get_stat(void)
4685 {
4686 tc_stat.progress = get_progress();
4687 tc_stat.processed_entries = processed_dir_count;
4688
4689 return &tc_stat;
4690 }
4691
4692 void tagcache_start_scan(void)
4693 {
4694 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4695 }
4696
4697 bool tagcache_update(void)
4698 {
4699 if (!tc_stat.ready)
4700 return false;
4701
4702 queue_post(&tagcache_queue, Q_UPDATE, 0);
4703 return false;
4704 }
4705
4706 bool tagcache_rebuild()
4707 {
4708 queue_post(&tagcache_queue, Q_REBUILD, 0);
4709 return false;
4710 }
4711
4712 void tagcache_stop_scan(void)
4713 {
4714 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4715 }
4716
4717 #ifdef HAVE_TC_RAMCACHE
4718 bool tagcache_is_ramcache(void)
4719 {
4720 return tc_stat.ramcache;
4721 }
4722 #endif
4723
4724 #endif /* !__PCTOOL__ */
4725
4726
4727 void tagcache_init(void)
4728 {
4729 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4730 memset(&current_tcmh, 0, sizeof(struct master_header));
4731 filenametag_fd = -1;
4732 write_lock = read_lock = 0;
4733
4734 #ifndef __PCTOOL__
4735 mutex_init(&command_queue_mutex);
4736 queue_init(&tagcache_queue, true);
4737 create_thread(tagcache_thread, tagcache_stack,
4738 sizeof(tagcache_stack), 0, tagcache_thread_name
4739 IF_PRIO(, PRIORITY_BACKGROUND)
4740 IF_COP(, CPU));
4741 #else
4742 tc_stat.initialized = true;
4743 allocate_tempbuf();
4744 commit();
4745 free_tempbuf();
4746 tc_stat.ready = check_all_headers();
4747 #endif
4748 }
4749
4750 #ifdef __PCTOOL__
4751 void tagcache_reverse_scan(void)
4752 {
4753 logf("Checking for deleted files");
4754 check_deleted_files();
4755 }
4756 #endif
4757
4758 bool tagcache_is_initialized(void)
4759 {
4760 return tc_stat.initialized;
4761 }
4762 bool tagcache_is_fully_initialized(void)
4763 {
4764 return tc_stat.readyvalid;
4765 }
4766 bool tagcache_is_usable(void)
4767 {
4768 return tc_stat.initialized && tc_stat.ready;
4769 }
4770 int tagcache_get_commit_step(void)
4771 {
4772 return tc_stat.commit_step;
4773 }
4774 int tagcache_get_max_commit_step(void)
4775 {
4776 return (int)(SORTED_TAGS_COUNT)+1;
4777 }
4778
My public Rockbox git repo