?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add Bob Jenkins' lookup3 32-bit and 64-bit hashes to pluginlib.
[kugel-rb.git] / apps / tagcache.c
blob055f43944661807390bf31fccecbf8ffd639599f
1 /***************************************************************************
2 * __________ __ ___.
3 * Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 * \/ \/ \/ \/ \/
8 * $Id$
9 *
10 * Copyright (C) 2005 by Miika Pekkarinen
11 *
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License
14 * as published by the Free Software Foundation; either version 2
15 * of the License, or (at your option) any later version.
16 *
17 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
18 * KIND, either express or implied.
19 *
20 ****************************************************************************/
21
22 /*
23 * TagCache API
24 *
25 * ----------x---------x------------------x-----
26 * | | | External
27 * +---------------x-------+ | TagCache | Libraries
28 * | Modification routines | | Core |
29 * +-x---------x-----------+ | |
30 * | (R/W) | | | |
31 * | +------x-------------x-+ +-------------x-----+ |
32 * | | x==x Filters & clauses | |
33 * | | Search routines | +-------------------+ |
34 * | | x============================x DirCache
35 * | +-x--------------------+ | (optional)
36 * | | (R) |
37 * | | +-------------------------------+ +---------+ |
38 * | | | DB Commit (sort,unique,index) | | | |
39 * | | +-x--------------------------x--+ | Control | |
40 * | | | (R/W) | (R) | Thread | |
41 * | | | +----------------------+ | | | |
42 * | | | | TagCache DB Builder | | +---------+ |
43 * | | | +-x-------------x------+ | |
44 * | | | | (R) | (W) | |
45 * | | | | +--x--------x---------+ |
46 * | | | | | Temporary Commit DB | |
47 * | | | | +---------------------+ |
48 * +-x----x-------x--+ |
49 * | TagCache RAM DB x==\(W) +-----------------+ |
50 * +-----------------+ \===x | |
51 * | | | | (R) | Ram DB Loader x============x DirCache
52 * +-x----x---x---x---+ /==x | | (optional)
53 * | Tagcache Disk DB x==/ +-----------------+ |
54 * +------------------+ |
55 *
56 */
57
58 /* #define LOGF_ENABLE */
59
60 #include <stdio.h>
61 #include <stdlib.h>
62 #include <ctype.h>
63 #include "config.h"
64 #include "ata_idle_notify.h"
65 #include "thread.h"
66 #include "kernel.h"
67 #include "system.h"
68 #include "logf.h"
69 #include "string.h"
70 #include "usb.h"
71 #include "metadata.h"
72 #include "tagcache.h"
73 #include "buffer.h"
74 #include "crc32.h"
75 #include "misc.h"
76 #include "settings.h"
77 #include "dir.h"
78 #include "structec.h"
79
80 #ifndef __PCTOOL__
81 #include "lang.h"
82 #include "eeprom_settings.h"
83 #endif
84
85 #ifdef __PCTOOL__
86 #define yield() do { } while(0)
87 #define sim_sleep(timeout) do { } while(0)
88 #define do_timed_yield() do { } while(0)
89 #endif
90
91 #ifndef __PCTOOL__
92 /* Tag Cache thread. */
93 static struct event_queue tagcache_queue;
94 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
95 static const char tagcache_thread_name[] = "tagcache";
96 #endif
97
98 #define UNTAGGED "<Untagged>"
99
100 /* Previous path when scanning directory tree recursively. */
101 static char curpath[TAG_MAXLEN+32];
102
103 /* Used when removing duplicates. */
104 static char *tempbuf; /* Allocated when needed. */
105 static long tempbufidx; /* Current location in buffer. */
106 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
107 static long tempbuf_left; /* Buffer space left. */
108 static long tempbuf_pos;
109
110 /* Tags we want to get sorted (loaded to the tempbuf). */
111 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
112 tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
113
114 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
115 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
116 tag_composer, tag_comment, tag_albumartist, tag_grouping };
117
118 /* Numeric tags (we can use these tags with conditional clauses). */
119 static const int numeric_tags[] = { tag_year, tag_discnumber,
120 tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
121 tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
122 tag_virt_length_min, tag_virt_length_sec,
123 tag_virt_playtime_min, tag_virt_playtime_sec,
124 tag_virt_entryage, tag_virt_autoscore };
125
126 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
127 static const char *tags_str[] = { "artist", "album", "genre", "title",
128 "filename", "composer", "comment", "albumartist", "grouping", "year",
129 "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
130 "playtime", "lastplayed", "commitid", "mtime" };
131
132 /* Status information of the tagcache. */
133 static struct tagcache_stat tc_stat;
134
135 /* Queue commands. */
136 enum tagcache_queue {
137 Q_STOP_SCAN = 0,
138 Q_START_SCAN,
139 Q_IMPORT_CHANGELOG,
140 Q_UPDATE,
141 Q_REBUILD,
142
143 /* Internal tagcache command queue. */
144 CMD_UPDATE_MASTER_HEADER,
145 CMD_UPDATE_NUMERIC,
146 };
147
148 struct tagcache_command_entry {
149 int32_t command;
150 int32_t idx_id;
151 int32_t tag;
152 int32_t data;
153 };
154
155 #ifndef __PCTOOL__
156 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
157 static volatile int command_queue_widx = 0;
158 static volatile int command_queue_ridx = 0;
159 static struct mutex command_queue_mutex;
160 #endif
161
162 /* Tag database structures. */
163
164 /* Variable-length tag entry in tag files. */
165 struct tagfile_entry {
166 int32_t tag_length; /* Length of the data in bytes including '\0' */
167 int32_t idx_id; /* Corresponding entry location in index file of not unique tags */
168 char tag_data[0]; /* Begin of the tag data */
169 };
170
171 /* Fixed-size tag entry in master db index. */
172 struct index_entry {
173 int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
174 int32_t flag; /* Status flags */
175 };
176
177 /* Header is the same in every file. */
178 struct tagcache_header {
179 int32_t magic; /* Header version number */
180 int32_t datasize; /* Data size in bytes */
181 int32_t entry_count; /* Number of entries in this file */
182 };
183
184 struct master_header {
185 struct tagcache_header tch;
186 int32_t serial; /* Increasing counting number */
187 int32_t commitid; /* Number of commits so far */
188 int32_t dirty;
189 };
190
191 /* For the endianess correction */
192 static const char *tagfile_entry_ec = "ll";
193 /**
194 Note: This should be (1 + TAG_COUNT) amount of l's.
195 */
196 static const char *index_entry_ec = "lllllllllllllllllllll";
197
198 static const char *tagcache_header_ec = "lll";
199 static const char *master_header_ec = "llllll";
200
201 static struct master_header current_tcmh;
202
203 #ifdef HAVE_TC_RAMCACHE
204 /* Header is created when loading database to ram. */
205 struct ramcache_header {
206 struct master_header h; /* Header from the master index */
207 struct index_entry *indices; /* Master index file content */
208 char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */
209 int entry_count[TAG_COUNT]; /* Number of entries in the indices. */
210 };
211
212 # ifdef HAVE_EEPROM_SETTINGS
213 struct statefile_header {
214 struct ramcache_header *hdr;
215 struct tagcache_stat tc_stat;
216 };
217 # endif
218
219 /* Pointer to allocated ramcache_header */
220 static struct ramcache_header *hdr;
221 #endif
222
223 /**
224 * Full tag entries stored in a temporary file waiting
225 * for commit to the cache. */
226 struct temp_file_entry {
227 long tag_offset[TAG_COUNT];
228 short tag_length[TAG_COUNT];
229 long flag;
230
231 long data_length;
232 };
233
234 struct tempbuf_id_list {
235 long id;
236 struct tempbuf_id_list *next;
237 };
238
239 struct tempbuf_searchidx {
240 long idx_id;
241 char *str;
242 int seek;
243 struct tempbuf_id_list idlist;
244 };
245
246 /* Lookup buffer for fixing messed up index while after sorting. */
247 static long commit_entry_count;
248 static long lookup_buffer_depth;
249 static struct tempbuf_searchidx **lookup;
250
251 /* Used when building the temporary file. */
252 static int cachefd = -1, filenametag_fd;
253 static int total_entry_count = 0;
254 static int data_size = 0;
255 static int processed_dir_count;
256
257 /* Thread safe locking */
258 static volatile int write_lock;
259 static volatile int read_lock;
260
261 static bool delete_entry(long idx_id);
262
263 const char* tagcache_tag_to_str(int tag)
264 {
265 return tags_str[tag];
266 }
267
268 bool tagcache_is_numeric_tag(int type)
269 {
270 int i;
271
272 for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
273 {
274 if (type == numeric_tags[i])
275 return true;
276 }
277
278 return false;
279 }
280
281 bool tagcache_is_unique_tag(int type)
282 {
283 int i;
284
285 for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
286 {
287 if (type == unique_tags[i])
288 return true;
289 }
290
291 return false;
292 }
293
294 bool tagcache_is_sorted_tag(int type)
295 {
296 int i;
297
298 for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
299 {
300 if (type == sorted_tags[i])
301 return true;
302 }
303
304 return false;
305 }
306
307 #ifdef HAVE_DIRCACHE
308 /**
309 * Returns true if specified flag is still present, i.e., dircache
310 * has not been reloaded.
311 */
312 static bool is_dircache_intact(void)
313 {
314 return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
315 }
316 #endif
317
318 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
319 {
320 int fd;
321 char buf[MAX_PATH];
322 int rc;
323
324 if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
325 return -1;
326
327 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
328
329 fd = open(buf, write ? O_RDWR : O_RDONLY);
330 if (fd < 0)
331 {
332 logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
333 tc_stat.ready = false;
334 return fd;
335 }
336
337 /* Check the header. */
338 rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
339 if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
340 {
341 logf("header error");
342 tc_stat.ready = false;
343 close(fd);
344 return -2;
345 }
346
347 return fd;
348 }
349
350 static int open_master_fd(struct master_header *hdr, bool write)
351 {
352 int fd;
353 int rc;
354
355 fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
356 if (fd < 0)
357 {
358 logf("master file open failed for R/W");
359 tc_stat.ready = false;
360 return fd;
361 }
362
363 tc_stat.econ = false;
364
365 /* Check the header. */
366 rc = read(fd, hdr, sizeof(struct master_header));
367 if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
368 {
369 /* Success. */
370 return fd;
371 }
372
373 /* Trying to read again, this time with endianess correction enabled. */
374 lseek(fd, 0, SEEK_SET);
375
376 rc = ecread(fd, hdr, 1, master_header_ec, true);
377 if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
378 {
379 logf("header error");
380 tc_stat.ready = false;
381 close(fd);
382 return -2;
383 }
384
385 tc_stat.econ = true;
386
387 return fd;
388 }
389
390 #ifndef __PCTOOL__
391 static bool do_timed_yield(void)
392 {
393 /* Sorting can lock up for quite a while, so yield occasionally */
394 static long wakeup_tick = 0;
395 if (current_tick >= wakeup_tick)
396 {
397 wakeup_tick = current_tick + (HZ/4);
398 yield();
399 return true;
400 }
401 return false;
402 }
403 #endif
404
405 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
406 static long find_entry_ram(const char *filename,
407 const struct dirent *dc)
408 {
409 static long last_pos = 0;
410 int i;
411
412 /* Check if we tagcache is loaded into ram. */
413 if (!tc_stat.ramcache)
414 return -1;
415
416 if (dc == NULL)
417 dc = dircache_get_entry_ptr(filename);
418
419 if (dc == NULL)
420 {
421 logf("tagcache: file not found.");
422 return -1;
423 }
424
425 try_again:
426
427 if (last_pos > 0)
428 i = last_pos;
429 else
430 i = 0;
431
432 for (; i < hdr->h.tch.entry_count; i++)
433 {
434 if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
435 {
436 last_pos = MAX(0, i - 3);
437 return i;
438 }
439
440 do_timed_yield();
441 }
442
443 if (last_pos > 0)
444 {
445 last_pos = 0;
446 goto try_again;
447 }
448
449 return -1;
450 }
451 #endif
452
453 static long find_entry_disk(const char *filename)
454 {
455 struct tagcache_header tch;
456 static long last_pos = -1;
457 long pos_history[POS_HISTORY_COUNT];
458 long pos_history_idx = 0;
459 bool found = false;
460 struct tagfile_entry tfe;
461 int fd;
462 char buf[TAG_MAXLEN+32];
463 int i;
464 int pos = -1;
465
466 if (!tc_stat.ready)
467 return -2;
468
469 fd = filenametag_fd;
470 if (fd < 0)
471 {
472 last_pos = -1;
473 if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
474 return -1;
475 }
476
477 check_again:
478
479 if (last_pos > 0)
480 lseek(fd, last_pos, SEEK_SET);
481 else
482 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
483
484 while (true)
485 {
486 pos = lseek(fd, 0, SEEK_CUR);
487 for (i = pos_history_idx-1; i >= 0; i--)
488 pos_history[i+1] = pos_history[i];
489 pos_history[0] = pos;
490
491 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
492 != sizeof(struct tagfile_entry))
493 {
494 break ;
495 }
496
497 if (tfe.tag_length >= (long)sizeof(buf))
498 {
499 logf("too long tag #1");
500 close(fd);
501 last_pos = -1;
502 return -2;
503 }
504
505 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
506 {
507 logf("read error #2");
508 close(fd);
509 last_pos = -1;
510 return -3;
511 }
512
513 if (!strcasecmp(filename, buf))
514 {
515 last_pos = pos_history[pos_history_idx];
516 found = true;
517 break ;
518 }
519
520 if (pos_history_idx < POS_HISTORY_COUNT - 1)
521 pos_history_idx++;
522 }
523
524 /* Not found? */
525 if (!found)
526 {
527 if (last_pos > 0)
528 {
529 last_pos = -1;
530 logf("seek again");
531 goto check_again;
532 }
533
534 if (fd != filenametag_fd)
535 close(fd);
536 return -4;
537 }
538
539 if (fd != filenametag_fd)
540 close(fd);
541
542 return tfe.idx_id;
543 }
544
545 static int find_index(const char *filename)
546 {
547 long idx_id = -1;
548
549 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
550 if (tc_stat.ramcache && is_dircache_intact())
551 idx_id = find_entry_ram(filename, NULL);
552 #endif
553
554 if (idx_id < 0)
555 idx_id = find_entry_disk(filename);
556
557 return idx_id;
558 }
559
560 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
561 {
562 int idx_id;
563
564 if (!tc_stat.ready)
565 return false;
566
567 idx_id = find_index(filename);
568 if (idx_id < 0)
569 return false;
570
571 if (!tagcache_search(tcs, tag_filename))
572 return false;
573
574 tcs->entry_count = 0;
575 tcs->idx_id = idx_id;
576
577 return true;
578 }
579
580 static bool get_index(int masterfd, int idxid,
581 struct index_entry *idx, bool use_ram)
582 {
583 bool localfd = false;
584
585 if (idxid < 0)
586 {
587 logf("Incorrect idxid: %d", idxid);
588 return false;
589 }
590
591 #ifdef HAVE_TC_RAMCACHE
592 if (tc_stat.ramcache && use_ram)
593 {
594 if (hdr->indices[idxid].flag & FLAG_DELETED)
595 return false;
596
597 memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
598 return true;
599 }
600 #else
601 (void)use_ram;
602 #endif
603
604 if (masterfd < 0)
605 {
606 struct master_header tcmh;
607
608 localfd = true;
609 masterfd = open_master_fd(&tcmh, false);
610 if (masterfd < 0)
611 return false;
612 }
613
614 lseek(masterfd, idxid * sizeof(struct index_entry)
615 + sizeof(struct master_header), SEEK_SET);
616 if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
617 != sizeof(struct index_entry))
618 {
619 logf("read error #3");
620 if (localfd)
621 close(masterfd);
622
623 return false;
624 }
625
626 if (localfd)
627 close(masterfd);
628
629 if (idx->flag & FLAG_DELETED)
630 return false;
631
632 return true;
633 }
634
635 #ifndef __PCTOOL__
636
637 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
638 {
639 /* We need to exclude all memory only flags & tags when writing to disk. */
640 if (idx->flag & FLAG_DIRCACHE)
641 {
642 logf("memory only flags!");
643 return false;
644 }
645
646 #ifdef HAVE_TC_RAMCACHE
647 /* Only update numeric data. Writing the whole index to RAM by memcpy
648 * destroys dircache pointers!
649 */
650 if (tc_stat.ramcache)
651 {
652 int tag;
653 struct index_entry *idx_ram = &hdr->indices[idxid];
654
655 for (tag = 0; tag < TAG_COUNT; tag++)
656 {
657 if (tagcache_is_numeric_tag(tag))
658 {
659 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
660 }
661 }
662
663 /* Don't touch the dircache flag or attributes. */
664 idx_ram->flag = (idx->flag & 0x0000ffff)
665 | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
666 }
667 #endif
668
669 lseek(masterfd, idxid * sizeof(struct index_entry)
670 + sizeof(struct master_header), SEEK_SET);
671 if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
672 != sizeof(struct index_entry))
673 {
674 logf("write error #3");
675 logf("idxid: %d", idxid);
676 return false;
677 }
678
679 return true;
680 }
681
682 #endif /* !__PCTOOL__ */
683
684 static bool open_files(struct tagcache_search *tcs, int tag)
685 {
686 if (tcs->idxfd[tag] < 0)
687 {
688 char fn[MAX_PATH];
689
690 snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
691 tcs->idxfd[tag] = open(fn, O_RDONLY);
692 }
693
694 if (tcs->idxfd[tag] < 0)
695 {
696 logf("File not open!");
697 return false;
698 }
699
700 return true;
701 }
702
703 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
704 int tag, char *buf, long size)
705 {
706 struct tagfile_entry tfe;
707 long seek;
708
709 *buf = '\0';
710
711 if (tagcache_is_numeric_tag(tag))
712 return false;
713
714 seek = idx->tag_seek[tag];
715 if (seek < 0)
716 {
717 logf("Retrieve failed");
718 return false;
719 }
720
721 #ifdef HAVE_TC_RAMCACHE
722 if (tcs->ramsearch)
723 {
724 struct tagfile_entry *ep;
725
726 # ifdef HAVE_DIRCACHE
727 if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
728 && is_dircache_intact())
729 {
730 dircache_copy_path((struct dirent *)seek,
731 buf, size);
732 return true;
733 }
734 else
735 # endif
736 if (tag != tag_filename)
737 {
738 ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
739 strncpy(buf, ep->tag_data, size-1);
740
741 return true;
742 }
743 }
744 #endif
745
746 if (!open_files(tcs, tag))
747 return false;
748
749 lseek(tcs->idxfd[tag], seek, SEEK_SET);
750 if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
751 != sizeof(struct tagfile_entry))
752 {
753 logf("read error #5");
754 return false;
755 }
756
757 if (tfe.tag_length >= size)
758 {
759 logf("too small buffer");
760 return false;
761 }
762
763 if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
764 tfe.tag_length)
765 {
766 logf("read error #6");
767 return false;
768 }
769
770 buf[tfe.tag_length] = '\0';
771
772 return true;
773 }
774
775 static long check_virtual_tags(int tag, const struct index_entry *idx)
776 {
777 long data = 0;
778
779 switch (tag)
780 {
781 case tag_virt_length_sec:
782 data = (idx->tag_seek[tag_length]/1000) % 60;
783 break;
784
785 case tag_virt_length_min:
786 data = (idx->tag_seek[tag_length]/1000) / 60;
787 break;
788
789 case tag_virt_playtime_sec:
790 data = (idx->tag_seek[tag_playtime]/1000) % 60;
791 break;
792
793 case tag_virt_playtime_min:
794 data = (idx->tag_seek[tag_playtime]/1000) / 60;
795 break;
796
797 case tag_virt_autoscore:
798 if (idx->tag_seek[tag_length] == 0
799 || idx->tag_seek[tag_playcount] == 0)
800 {
801 data = 0;
802 }
803 else
804 {
805 data = 100 * idx->tag_seek[tag_playtime]
806 / idx->tag_seek[tag_length]
807 / idx->tag_seek[tag_playcount];
808 }
809 break;
810
811 /* How many commits before the file has been added to the DB. */
812 case tag_virt_entryage:
813 data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
814 break;
815
816 default:
817 data = idx->tag_seek[tag];
818 }
819
820 return data;
821 }
822
823 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
824 {
825 struct index_entry idx;
826
827 if (!tc_stat.ready)
828 return false;
829
830 if (!tagcache_is_numeric_tag(tag))
831 return -1;
832
833 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
834 return -2;
835
836 return check_virtual_tags(tag, &idx);
837 }
838
839 inline static bool str_ends_with(const char *str1, const char *str2)
840 {
841 int str_len = strlen(str1);
842 int clause_len = strlen(str2);
843
844 if (clause_len > str_len)
845 return false;
846
847 return !strcasecmp(&str1[str_len - clause_len], str2);
848 }
849
850 inline static bool str_oneof(const char *str, const char *list)
851 {
852 const char *sep;
853 int l, len = strlen(str);
854
855 while (*list)
856 {
857 sep = strchr(list, '|');
858 l = sep ? (long)sep - (long)list : (int)strlen(list);
859 if ((l==len) && !strncasecmp(str, list, len))
860 return true;
861 list += sep ? l + 1 : l;
862 }
863
864 return false;
865 }
866
867 static bool check_against_clause(long numeric, const char *str,
868 const struct tagcache_search_clause *clause)
869 {
870 if (clause->numeric)
871 {
872 switch (clause->type)
873 {
874 case clause_is:
875 return numeric == clause->numeric_data;
876 case clause_is_not:
877 return numeric != clause->numeric_data;
878 case clause_gt:
879 return numeric > clause->numeric_data;
880 case clause_gteq:
881 return numeric >= clause->numeric_data;
882 case clause_lt:
883 return numeric < clause->numeric_data;
884 case clause_lteq:
885 return numeric <= clause->numeric_data;
886 default:
887 logf("Incorrect numeric tag: %d", clause->type);
888 }
889 }
890 else
891 {
892 switch (clause->type)
893 {
894 case clause_is:
895 return !strcasecmp(clause->str, str);
896 case clause_is_not:
897 return strcasecmp(clause->str, str);
898 case clause_gt:
899 return 0>strcasecmp(clause->str, str);
900 case clause_gteq:
901 return 0>=strcasecmp(clause->str, str);
902 case clause_lt:
903 return 0<strcasecmp(clause->str, str);
904 case clause_lteq:
905 return 0<=strcasecmp(clause->str, str);
906 case clause_contains:
907 return (strcasestr(str, clause->str) != NULL);
908 case clause_not_contains:
909 return (strcasestr(str, clause->str) == NULL);
910 case clause_begins_with:
911 return (strcasestr(str, clause->str) == str);
912 case clause_not_begins_with:
913 return (strcasestr(str, clause->str) != str);
914 case clause_ends_with:
915 return str_ends_with(str, clause->str);
916 case clause_not_ends_with:
917 return !str_ends_with(str, clause->str);
918 case clause_oneof:
919 return str_oneof(str, clause->str);
920
921 default:
922 logf("Incorrect tag: %d", clause->type);
923 }
924 }
925
926 return false;
927 }
928
929 static bool check_clauses(struct tagcache_search *tcs,
930 struct index_entry *idx,
931 struct tagcache_search_clause **clause, int count)
932 {
933 int i;
934
935 #ifdef HAVE_TC_RAMCACHE
936 if (tcs->ramsearch)
937 {
938 /* Go through all conditional clauses. */
939 for (i = 0; i < count; i++)
940 {
941 struct tagfile_entry *tfe;
942 int seek;
943 char buf[256];
944 char *str = NULL;
945
946 seek = check_virtual_tags(clause[i]->tag, idx);
947
948 if (!tagcache_is_numeric_tag(clause[i]->tag))
949 {
950 if (clause[i]->tag == tag_filename)
951 {
952 retrieve(tcs, idx, tag_filename, buf, sizeof buf);
953 str = buf;
954 }
955 else
956 {
957 tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
958 str = tfe->tag_data;
959 }
960 }
961
962 if (!check_against_clause(seek, str, clause[i]))
963 return false;
964 }
965 }
966 else
967 #endif
968 {
969 /* Check for conditions. */
970 for (i = 0; i < count; i++)
971 {
972 struct tagfile_entry tfe;
973 int seek;
974 char str[256];
975
976 seek = check_virtual_tags(clause[i]->tag, idx);
977
978 memset(str, 0, sizeof str);
979 if (!tagcache_is_numeric_tag(clause[i]->tag))
980 {
981 int fd = tcs->idxfd[clause[i]->tag];
982 lseek(fd, seek, SEEK_SET);
983 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
984 if (tfe.tag_length >= (int)sizeof(str))
985 {
986 logf("Too long tag read!");
987 break ;
988 }
989
990 read(fd, str, tfe.tag_length);
991
992 /* Check if entry has been deleted. */
993 if (str[0] == '\0')
994 break;
995 }
996
997 if (!check_against_clause(seek, str, clause[i]))
998 return false;
999 }
1000 }
1001
1002 return true;
1003 }
1004
1005 bool tagcache_check_clauses(struct tagcache_search *tcs,
1006 struct tagcache_search_clause **clause, int count)
1007 {
1008 struct index_entry idx;
1009
1010 if (count == 0)
1011 return true;
1012
1013 if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1014 return false;
1015
1016 return check_clauses(tcs, &idx, clause, count);
1017 }
1018
1019 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1020 {
1021 int i;
1022
1023 /* If uniq buffer is not defined we must return true for search to work. */
1024 if (tcs->unique_list == NULL
1025 || (!tagcache_is_unique_tag(tcs->type)
1026 && !tagcache_is_numeric_tag(tcs->type)))
1027 {
1028 return true;
1029 }
1030
1031 for (i = 0; i < tcs->unique_list_count; i++)
1032 {
1033 /* Return false if entry is found. */
1034 if (tcs->unique_list[i] == id)
1035 return false;
1036 }
1037
1038 if (tcs->unique_list_count < tcs->unique_list_capacity)
1039 {
1040 tcs->unique_list[i] = id;
1041 tcs->unique_list_count++;
1042 }
1043
1044 return true;
1045 }
1046
1047 static bool build_lookup_list(struct tagcache_search *tcs)
1048 {
1049 struct index_entry entry;
1050 int i;
1051
1052 tcs->seek_list_count = 0;
1053
1054 #ifdef HAVE_TC_RAMCACHE
1055 if (tcs->ramsearch)
1056 {
1057 int j;
1058
1059 for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1060 {
1061 struct index_entry *idx = &hdr->indices[i];
1062 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1063 break ;
1064
1065 /* Skip deleted files. */
1066 if (idx->flag & FLAG_DELETED)
1067 continue;
1068
1069 /* Go through all filters.. */
1070 for (j = 0; j < tcs->filter_count; j++)
1071 {
1072 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1073 {
1074 break ;
1075 }
1076 }
1077
1078 if (j < tcs->filter_count)
1079 continue ;
1080
1081 /* Check for conditions. */
1082 if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1083 continue;
1084
1085 /* Add to the seek list if not already in uniq buffer. */
1086 if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1087 continue;
1088
1089 /* Lets add it. */
1090 tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1091 tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1092 tcs->seek_list_count++;
1093 }
1094
1095 tcs->seek_pos = i;
1096
1097 return tcs->seek_list_count > 0;
1098 }
1099 #endif
1100
1101 lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1102 sizeof(struct master_header), SEEK_SET);
1103
1104 while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1105 == sizeof(struct index_entry))
1106 {
1107 if (tcs->seek_list_count == SEEK_LIST_SIZE)
1108 break ;
1109
1110 tcs->seek_pos++;
1111
1112 /* Check if entry has been deleted. */
1113 if (entry.flag & FLAG_DELETED)
1114 continue;
1115
1116 /* Go through all filters.. */
1117 for (i = 0; i < tcs->filter_count; i++)
1118 {
1119 if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1120 break ;
1121 }
1122
1123 if (i < tcs->filter_count)
1124 continue ;
1125
1126 /* Check for conditions. */
1127 if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1128 continue;
1129
1130 /* Add to the seek list if not already in uniq buffer. */
1131 if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1132 continue;
1133
1134 /* Lets add it. */
1135 tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1136 tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1137 tcs->seek_list_count++;
1138
1139 yield();
1140 }
1141
1142 return tcs->seek_list_count > 0;
1143 }
1144
1145
1146 static void remove_files(void)
1147 {
1148 int i;
1149 char buf[MAX_PATH];
1150
1151 tc_stat.ready = false;
1152 tc_stat.ramcache = false;
1153 tc_stat.econ = false;
1154 remove(TAGCACHE_FILE_MASTER);
1155 for (i = 0; i < TAG_COUNT; i++)
1156 {
1157 if (tagcache_is_numeric_tag(i))
1158 continue;
1159
1160 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1161 remove(buf);
1162 }
1163 }
1164
1165
1166 static bool check_all_headers(void)
1167 {
1168 struct master_header myhdr;
1169 struct tagcache_header tch;
1170 int tag;
1171 int fd;
1172
1173 if ( (fd = open_master_fd(&myhdr, false)) < 0)
1174 return false;
1175
1176 close(fd);
1177 if (myhdr.dirty)
1178 {
1179 logf("tagcache is dirty!");
1180 return false;
1181 }
1182
1183 memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1184
1185 for (tag = 0; tag < TAG_COUNT; tag++)
1186 {
1187 if (tagcache_is_numeric_tag(tag))
1188 continue;
1189
1190 if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1191 return false;
1192
1193 close(fd);
1194 }
1195
1196 return true;
1197 }
1198
1199 bool tagcache_search(struct tagcache_search *tcs, int tag)
1200 {
1201 struct tagcache_header tag_hdr;
1202 struct master_header master_hdr;
1203 int i;
1204
1205 if (tcs->initialized)
1206 tagcache_search_finish(tcs);
1207
1208 while (read_lock)
1209 sleep(1);
1210
1211 memset(tcs, 0, sizeof(struct tagcache_search));
1212 if (tc_stat.commit_step > 0 || !tc_stat.ready)
1213 return false;
1214
1215 tcs->position = sizeof(struct tagcache_header);
1216 tcs->type = tag;
1217 tcs->seek_pos = 0;
1218 tcs->seek_list_count = 0;
1219 tcs->filter_count = 0;
1220 tcs->masterfd = -1;
1221
1222 for (i = 0; i < TAG_COUNT; i++)
1223 tcs->idxfd[i] = -1;
1224
1225 #ifndef HAVE_TC_RAMCACHE
1226 tcs->ramsearch = false;
1227 #else
1228 tcs->ramsearch = tc_stat.ramcache;
1229 if (tcs->ramsearch)
1230 {
1231 tcs->entry_count = hdr->entry_count[tcs->type];
1232 }
1233 else
1234 #endif
1235 {
1236 if (!tagcache_is_numeric_tag(tcs->type))
1237 {
1238 tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1239 if (tcs->idxfd[tcs->type] < 0)
1240 return false;
1241 }
1242
1243 /* Always open as R/W so we can pass tcs to functions that modify data also
1244 * without failing. */
1245 tcs->masterfd = open_master_fd(&master_hdr, true);
1246
1247 if (tcs->masterfd < 0)
1248 return false;
1249 }
1250
1251 tcs->valid = true;
1252 tcs->initialized = true;
1253 write_lock++;
1254
1255 return true;
1256 }
1257
1258 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1259 void *buffer, long length)
1260 {
1261 tcs->unique_list = (unsigned long *)buffer;
1262 tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1263 tcs->unique_list_count = 0;
1264 }
1265
1266 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1267 int tag, int seek)
1268 {
1269 if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1270 return false;
1271
1272 if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1273 return false;
1274
1275 tcs->filter_tag[tcs->filter_count] = tag;
1276 tcs->filter_seek[tcs->filter_count] = seek;
1277 tcs->filter_count++;
1278
1279 return true;
1280 }
1281
1282 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1283 struct tagcache_search_clause *clause)
1284 {
1285 int i;
1286
1287 if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1288 {
1289 logf("Too many clauses");
1290 return false;
1291 }
1292
1293 /* Check if there is already a similar filter in present (filters are
1294 * much faster than clauses).
1295 */
1296 for (i = 0; i < tcs->filter_count; i++)
1297 {
1298 if (tcs->filter_tag[i] == clause->tag)
1299 return true;
1300 }
1301
1302 if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1303 {
1304 char buf[MAX_PATH];
1305
1306 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1307 tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1308 }
1309
1310 tcs->clause[tcs->clause_count] = clause;
1311 tcs->clause_count++;
1312
1313 return true;
1314 }
1315
1316 /* TODO: Remove this mess. */
1317 #ifdef HAVE_DIRCACHE
1318 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1319 ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1320 #else
1321 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1322 #endif
1323
1324 static bool get_next(struct tagcache_search *tcs)
1325 {
1326 static char buf[TAG_MAXLEN+32];
1327 struct tagfile_entry entry;
1328 long flag = 0;
1329
1330 if (!tcs->valid || !tc_stat.ready)
1331 return false;
1332
1333 if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1334 #ifdef HAVE_TC_RAMCACHE
1335 && !tcs->ramsearch
1336 #endif
1337 )
1338 return false;
1339
1340 /* Relative fetch. */
1341 if (tcs->filter_count > 0 || tcs->clause_count > 0
1342 || tagcache_is_numeric_tag(tcs->type))
1343 {
1344 /* Check for end of list. */
1345 if (tcs->seek_list_count == 0)
1346 {
1347 /* Try to fetch more. */
1348 if (!build_lookup_list(tcs))
1349 {
1350 tcs->valid = false;
1351 return false;
1352 }
1353 }
1354
1355 tcs->seek_list_count--;
1356 flag = tcs->seek_flags[tcs->seek_list_count];
1357
1358 /* Seek stream to the correct position and continue to direct fetch. */
1359 if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1360 && !tagcache_is_numeric_tag(tcs->type))
1361 {
1362 if (!open_files(tcs, tcs->type))
1363 return false;
1364
1365 lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1366 }
1367 else
1368 tcs->position = tcs->seek_list[tcs->seek_list_count];
1369 }
1370
1371 if (tagcache_is_numeric_tag(tcs->type))
1372 {
1373 snprintf(buf, sizeof(buf), "%d", tcs->position);
1374 tcs->result_seek = tcs->position;
1375 tcs->result = buf;
1376 tcs->result_len = strlen(buf) + 1;
1377 return true;
1378 }
1379
1380 /* Direct fetch. */
1381 #ifdef HAVE_TC_RAMCACHE
1382 if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1383 {
1384 struct tagfile_entry *ep;
1385
1386 if (tcs->entry_count == 0)
1387 {
1388 tcs->valid = false;
1389 return false;
1390 }
1391 tcs->entry_count--;
1392
1393 tcs->result_seek = tcs->position;
1394
1395 # ifdef HAVE_DIRCACHE
1396 if (tcs->type == tag_filename)
1397 {
1398 dircache_copy_path((struct dirent *)tcs->position,
1399 buf, sizeof buf);
1400 tcs->result = buf;
1401 tcs->result_len = strlen(buf) + 1;
1402 tcs->idx_id = FLAG_GET_ATTR(flag);
1403 tcs->ramresult = false;
1404
1405 return true;
1406 }
1407 # endif
1408
1409 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1410 tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1411 tcs->result = ep->tag_data;
1412 tcs->result_len = strlen(tcs->result) + 1;
1413 tcs->idx_id = ep->idx_id;
1414 tcs->ramresult = true;
1415
1416 return true;
1417 }
1418 else
1419 #endif
1420 {
1421 if (!open_files(tcs, tcs->type))
1422 return false;
1423
1424 tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1425 if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1426 tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1427 {
1428 /* End of data. */
1429 tcs->valid = false;
1430 return false;
1431 }
1432 }
1433
1434 if (entry.tag_length > (long)sizeof(buf))
1435 {
1436 tcs->valid = false;
1437 logf("too long tag #2");
1438 return false;
1439 }
1440
1441 if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1442 {
1443 tcs->valid = false;
1444 logf("read error #4");
1445 return false;
1446 }
1447
1448 tcs->result = buf;
1449 tcs->result_len = strlen(tcs->result) + 1;
1450 tcs->idx_id = entry.idx_id;
1451 tcs->ramresult = false;
1452
1453 return true;
1454 }
1455
1456 bool tagcache_get_next(struct tagcache_search *tcs)
1457 {
1458 while (get_next(tcs))
1459 {
1460 if (tcs->result_len > 1)
1461 return true;
1462 }
1463
1464 return false;
1465 }
1466
1467 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1468 int tag, char *buf, long size)
1469 {
1470 struct index_entry idx;
1471
1472 *buf = '\0';
1473 if (!get_index(tcs->masterfd, idxid, &idx, true))
1474 return false;
1475
1476 return retrieve(tcs, &idx, tag, buf, size);
1477 }
1478
1479 static bool update_master_header(void)
1480 {
1481 struct master_header myhdr;
1482 int fd;
1483
1484 if (!tc_stat.ready)
1485 return false;
1486
1487 if ( (fd = open_master_fd(&myhdr, true)) < 0)
1488 return false;
1489
1490 myhdr.serial = current_tcmh.serial;
1491 myhdr.commitid = current_tcmh.commitid;
1492 myhdr.dirty = current_tcmh.dirty;
1493
1494 /* Write it back */
1495 lseek(fd, 0, SEEK_SET);
1496 ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1497 close(fd);
1498
1499 #ifdef HAVE_TC_RAMCACHE
1500 if (hdr)
1501 {
1502 hdr->h.serial = current_tcmh.serial;
1503 hdr->h.commitid = current_tcmh.commitid;
1504 hdr->h.dirty = current_tcmh.dirty;
1505 }
1506 #endif
1507
1508 return true;
1509 }
1510
1511 #if 0
1512
1513 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1514 {
1515 struct tagentry *entry;
1516
1517 if (tcs->type != tag_title)
1518 return ;
1519
1520 /* We will need reserve buffer for this. */
1521 if (tcs->ramcache)
1522 {
1523 struct tagfile_entry *ep;
1524
1525 ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1526 tcs->seek_list[tcs->seek_list_count];
1527 }
1528
1529 entry = find_entry_ram();
1530
1531 }
1532 #endif
1533
1534 void tagcache_search_finish(struct tagcache_search *tcs)
1535 {
1536 int i;
1537
1538 if (!tcs->initialized)
1539 return;
1540
1541 if (tcs->masterfd >= 0)
1542 {
1543 close(tcs->masterfd);
1544 tcs->masterfd = -1;
1545 }
1546
1547 for (i = 0; i < TAG_COUNT; i++)
1548 {
1549 if (tcs->idxfd[i] >= 0)
1550 {
1551 close(tcs->idxfd[i]);
1552 tcs->idxfd[i] = -1;
1553 }
1554 }
1555
1556 tcs->ramsearch = false;
1557 tcs->valid = false;
1558 tcs->initialized = 0;
1559 if (write_lock > 0)
1560 write_lock--;
1561 }
1562
1563 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1564 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1565 {
1566 return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1567 }
1568
1569 static long get_tag_numeric(const struct index_entry *entry, int tag)
1570 {
1571 return check_virtual_tags(tag, entry);
1572 }
1573
1574 static char* get_tag_string(const struct index_entry *entry, int tag)
1575 {
1576 char* s = get_tag(entry, tag)->tag_data;
1577 return strcmp(s, UNTAGGED) ? s : NULL;
1578 }
1579
1580 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1581 {
1582 struct index_entry *entry;
1583 int idx_id;
1584
1585 if (!tc_stat.ready)
1586 return false;
1587
1588 /* Find the corresponding entry in tagcache. */
1589 idx_id = find_entry_ram(filename, NULL);
1590 if (idx_id < 0 || !tc_stat.ramcache)
1591 return false;
1592
1593 entry = &hdr->indices[idx_id];
1594
1595 id3->title = get_tag_string(entry, tag_title);
1596 id3->artist = get_tag_string(entry, tag_artist);
1597 id3->album = get_tag_string(entry, tag_album);
1598 id3->genre_string = get_tag_string(entry, tag_genre);
1599 id3->composer = get_tag_string(entry, tag_composer);
1600 id3->comment = get_tag_string(entry, tag_comment);
1601 id3->albumartist = get_tag_string(entry, tag_albumartist);
1602 id3->grouping = get_tag_string(entry, tag_grouping);
1603
1604 id3->playcount = get_tag_numeric(entry, tag_playcount);
1605 id3->rating = get_tag_numeric(entry, tag_rating);
1606 id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1607 id3->score = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1608 id3->year = get_tag_numeric(entry, tag_year);
1609
1610 id3->discnum = get_tag_numeric(entry, tag_discnumber);
1611 id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1612 id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1613 if (id3->bitrate == 0)
1614 id3->bitrate = 1;
1615
1616 return true;
1617 }
1618 #endif
1619
1620 static inline void write_item(const char *item)
1621 {
1622 int len = strlen(item) + 1;
1623
1624 data_size += len;
1625 write(cachefd, item, len);
1626 }
1627
1628 static int check_if_empty(char **tag)
1629 {
1630 int length;
1631
1632 if (*tag == NULL || **tag == '\0')
1633 {
1634 *tag = UNTAGGED;
1635 return sizeof(UNTAGGED); /* Tag length */
1636 }
1637
1638 length = strlen(*tag);
1639 if (length > TAG_MAXLEN)
1640 {
1641 logf("over length tag: %s", *tag);
1642 length = TAG_MAXLEN;
1643 (*tag)[length] = '\0';
1644 }
1645
1646 return length + 1;
1647 }
1648
1649 #define ADD_TAG(entry,tag,data) \
1650 /* Adding tag */ \
1651 entry.tag_offset[tag] = offset; \
1652 entry.tag_length[tag] = check_if_empty(data); \
1653 offset += entry.tag_length[tag]
1654
1655 static void add_tagcache(char *path, unsigned long mtime
1656 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1657 ,const struct dirent *dc
1658 #endif
1659 )
1660 {
1661 struct mp3entry id3;
1662 struct temp_file_entry entry;
1663 bool ret;
1664 int fd;
1665 int idx_id = -1;
1666 char tracknumfix[3];
1667 int offset = 0;
1668 int path_length = strlen(path);
1669 bool has_albumartist;
1670 bool has_grouping;
1671
1672 if (cachefd < 0)
1673 return ;
1674
1675 /* Check for overlength file path. */
1676 if (path_length > TAG_MAXLEN)
1677 {
1678 /* Path can't be shortened. */
1679 logf("Too long path: %s", path);
1680 return ;
1681 }
1682
1683 /* Check if the file is supported. */
1684 if (probe_file_format(path) == AFMT_UNKNOWN)
1685 return ;
1686
1687 /* Check if the file is already cached. */
1688 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1689 if (tc_stat.ramcache && is_dircache_intact())
1690 {
1691 idx_id = find_entry_ram(path, dc);
1692 }
1693 else
1694 #endif
1695 {
1696 if (filenametag_fd >= 0)
1697 {
1698 idx_id = find_entry_disk(path);
1699 }
1700 }
1701
1702 /* Check if file has been modified. */
1703 if (idx_id >= 0)
1704 {
1705 struct index_entry idx;
1706
1707 /* TODO: Mark that the index exists (for fast reverse scan) */
1708 //found_idx[idx_id/8] |= idx_id%8;
1709
1710 if (!get_index(-1, idx_id, &idx, true))
1711 {
1712 logf("failed to retrieve index entry");
1713 return ;
1714 }
1715
1716 if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1717 {
1718 /* No changes to file. */
1719 return ;
1720 }
1721
1722 /* Metadata might have been changed. Delete the entry. */
1723 logf("Re-adding: %s", path);
1724 if (!delete_entry(idx_id))
1725 {
1726 logf("delete_entry failed: %d", idx_id);
1727 return ;
1728 }
1729 }
1730
1731 fd = open(path, O_RDONLY);
1732 if (fd < 0)
1733 {
1734 logf("open fail: %s", path);
1735 return ;
1736 }
1737
1738 memset(&id3, 0, sizeof(struct mp3entry));
1739 memset(&entry, 0, sizeof(struct temp_file_entry));
1740 memset(&tracknumfix, 0, sizeof(tracknumfix));
1741 ret = get_metadata(&id3, fd, path);
1742 close(fd);
1743
1744 if (!ret)
1745 return ;
1746
1747 logf("-> %s", path);
1748
1749 /* Generate track number if missing. */
1750 if (id3.tracknum <= 0)
1751 {
1752 const char *p = strrchr(path, '.');
1753
1754 if (p == NULL)
1755 p = &path[strlen(path)-1];
1756
1757 while (*p != '/')
1758 {
1759 if (isdigit(*p) && isdigit(*(p-1)))
1760 {
1761 tracknumfix[1] = *p--;
1762 tracknumfix[0] = *p;
1763 break;
1764 }
1765 p--;
1766 }
1767
1768 if (tracknumfix[0] != '\0')
1769 {
1770 id3.tracknum = atoi(tracknumfix);
1771 /* Set a flag to indicate track number has been generated. */
1772 entry.flag |= FLAG_TRKNUMGEN;
1773 }
1774 else
1775 {
1776 /* Unable to generate track number. */
1777 id3.tracknum = -1;
1778 }
1779 }
1780
1781 /* Numeric tags */
1782 entry.tag_offset[tag_year] = id3.year;
1783 entry.tag_offset[tag_discnumber] = id3.discnum;
1784 entry.tag_offset[tag_tracknumber] = id3.tracknum;
1785 entry.tag_offset[tag_length] = id3.length;
1786 entry.tag_offset[tag_bitrate] = id3.bitrate;
1787 entry.tag_offset[tag_mtime] = mtime;
1788
1789 /* String tags. */
1790 has_albumartist = id3.albumartist != NULL
1791 && strlen(id3.albumartist) > 0;
1792 has_grouping = id3.grouping != NULL
1793 && strlen(id3.grouping) > 0;
1794
1795 ADD_TAG(entry, tag_filename, &path);
1796 ADD_TAG(entry, tag_title, &id3.title);
1797 ADD_TAG(entry, tag_artist, &id3.artist);
1798 ADD_TAG(entry, tag_album, &id3.album);
1799 ADD_TAG(entry, tag_genre, &id3.genre_string);
1800 ADD_TAG(entry, tag_composer, &id3.composer);
1801 ADD_TAG(entry, tag_comment, &id3.comment);
1802 if (has_albumartist)
1803 {
1804 ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1805 }
1806 else
1807 {
1808 ADD_TAG(entry, tag_albumartist, &id3.artist);
1809 }
1810 if (has_grouping)
1811 {
1812 ADD_TAG(entry, tag_grouping, &id3.grouping);
1813 }
1814 else
1815 {
1816 ADD_TAG(entry, tag_grouping, &id3.title);
1817 }
1818 entry.data_length = offset;
1819
1820 /* Write the header */
1821 write(cachefd, &entry, sizeof(struct temp_file_entry));
1822
1823 /* And tags also... Correct order is critical */
1824 write_item(path);
1825 write_item(id3.title);
1826 write_item(id3.artist);
1827 write_item(id3.album);
1828 write_item(id3.genre_string);
1829 write_item(id3.composer);
1830 write_item(id3.comment);
1831 if (has_albumartist)
1832 {
1833 write_item(id3.albumartist);
1834 }
1835 else
1836 {
1837 write_item(id3.artist);
1838 }
1839 if (has_grouping)
1840 {
1841 write_item(id3.grouping);
1842 }
1843 else
1844 {
1845 write_item(id3.title);
1846 }
1847 total_entry_count++;
1848 }
1849
1850 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1851 {
1852 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1853 int len = strlen(str)+1;
1854 int i;
1855 unsigned crc32;
1856 unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1857 char buf[TAG_MAXLEN+32];
1858
1859 for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1860 buf[i] = tolower(str[i]);
1861 buf[i] = '\0';
1862
1863 crc32 = crc_32(buf, i, 0xffffffff);
1864
1865 if (unique)
1866 {
1867 /* Check if the crc does not exist -> entry does not exist for sure. */
1868 for (i = 0; i < tempbufidx; i++)
1869 {
1870 if (crcbuf[-i] != crc32)
1871 continue;
1872
1873 if (!strcasecmp(str, index[i].str))
1874 {
1875 if (id < 0 || id >= lookup_buffer_depth)
1876 {
1877 logf("lookup buf overf.: %d", id);
1878 return false;
1879 }
1880
1881 lookup[id] = &index[i];
1882 return true;
1883 }
1884 }
1885 }
1886
1887 /* Insert to CRC buffer. */
1888 crcbuf[-tempbufidx] = crc32;
1889 tempbuf_left -= 4;
1890
1891 /* Insert it to the buffer. */
1892 tempbuf_left -= len;
1893 if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1894 return false;
1895
1896 if (id >= lookup_buffer_depth)
1897 {
1898 logf("lookup buf overf. #2: %d", id);
1899 return false;
1900 }
1901
1902 if (id >= 0)
1903 {
1904 lookup[id] = &index[tempbufidx];
1905 index[tempbufidx].idlist.id = id;
1906 }
1907 else
1908 index[tempbufidx].idlist.id = -1;
1909
1910 index[tempbufidx].idlist.next = NULL;
1911 index[tempbufidx].idx_id = idx_id;
1912 index[tempbufidx].seek = -1;
1913 index[tempbufidx].str = &tempbuf[tempbuf_pos];
1914 memcpy(index[tempbufidx].str, str, len);
1915 tempbuf_pos += len;
1916 tempbufidx++;
1917
1918 return true;
1919 }
1920
1921 static int compare(const void *p1, const void *p2)
1922 {
1923 do_timed_yield();
1924
1925 struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1926 struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1927
1928 if (strcmp(e1->str, UNTAGGED) == 0)
1929 {
1930 if (strcmp(e2->str, UNTAGGED) == 0)
1931 return 0;
1932 return -1;
1933 }
1934 else if (strcmp(e2->str, UNTAGGED) == 0)
1935 return 1;
1936
1937 return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1938 }
1939
1940 static int tempbuf_sort(int fd)
1941 {
1942 struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1943 struct tagfile_entry fe;
1944 int i;
1945 int length;
1946
1947 /* Generate reverse lookup entries. */
1948 for (i = 0; i < lookup_buffer_depth; i++)
1949 {
1950 struct tempbuf_id_list *idlist;
1951
1952 if (!lookup[i])
1953 continue;
1954
1955 if (lookup[i]->idlist.id == i)
1956 continue;
1957
1958 idlist = &lookup[i]->idlist;
1959 while (idlist->next != NULL)
1960 idlist = idlist->next;
1961
1962 tempbuf_left -= sizeof(struct tempbuf_id_list);
1963 if (tempbuf_left - 4 < 0)
1964 return -1;
1965
1966 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1967 if (tempbuf_pos & 0x03)
1968 {
1969 tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1970 tempbuf_left -= 3;
1971 idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1972 }
1973 tempbuf_pos += sizeof(struct tempbuf_id_list);
1974
1975 idlist = idlist->next;
1976 idlist->id = i;
1977 idlist->next = NULL;
1978
1979 do_timed_yield();
1980 }
1981
1982 qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1983 memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1984
1985 for (i = 0; i < tempbufidx; i++)
1986 {
1987 struct tempbuf_id_list *idlist = &index[i].idlist;
1988
1989 /* Fix the lookup list. */
1990 while (idlist != NULL)
1991 {
1992 if (idlist->id >= 0)
1993 lookup[idlist->id] = &index[i];
1994 idlist = idlist->next;
1995 }
1996
1997 index[i].seek = lseek(fd, 0, SEEK_CUR);
1998 length = strlen(index[i].str) + 1;
1999 fe.tag_length = length;
2000 fe.idx_id = index[i].idx_id;
2001
2002 /* Check the chunk alignment. */
2003 if ((fe.tag_length + sizeof(struct tagfile_entry))
2004 % TAGFILE_ENTRY_CHUNK_LENGTH)
2005 {
2006 fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2007 ((fe.tag_length + sizeof(struct tagfile_entry))
2008 % TAGFILE_ENTRY_CHUNK_LENGTH);
2009 }
2010
2011 #ifdef TAGCACHE_STRICT_ALIGN
2012 /* Make sure the entry is long aligned. */
2013 if (index[i].seek & 0x03)
2014 {
2015 logf("tempbuf_sort: alignment error!");
2016 return -3;
2017 }
2018 #endif
2019
2020 if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2021 sizeof(struct tagfile_entry))
2022 {
2023 logf("tempbuf_sort: write error #1");
2024 return -1;
2025 }
2026
2027 if (write(fd, index[i].str, length) != length)
2028 {
2029 logf("tempbuf_sort: write error #2");
2030 return -2;
2031 }
2032
2033 /* Write some padding. */
2034 if (fe.tag_length - length > 0)
2035 write(fd, "XXXXXXXX", fe.tag_length - length);
2036 }
2037
2038 return i;
2039 }
2040
2041 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2042 {
2043 if (id < 0 || id >= lookup_buffer_depth)
2044 return NULL;
2045
2046 return lookup[id];
2047 }
2048
2049
2050 inline static int tempbuf_find_location(int id)
2051 {
2052 struct tempbuf_searchidx *entry;
2053
2054 entry = tempbuf_locate(id);
2055 if (entry == NULL)
2056 return -1;
2057
2058 return entry->seek;
2059 }
2060
2061 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2062 {
2063 struct master_header tcmh;
2064 struct index_entry idx;
2065 int masterfd;
2066 int masterfd_pos;
2067 struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2068 int max_entries;
2069 int entries_processed = 0;
2070 int i, j;
2071 char buf[TAG_MAXLEN];
2072
2073 max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2074
2075 logf("Building numeric indices...");
2076 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2077
2078 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2079 return false;
2080
2081 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2082 SEEK_CUR);
2083 if (masterfd_pos == filesize(masterfd))
2084 {
2085 logf("we can't append!");
2086 close(masterfd);
2087 return false;
2088 }
2089
2090 while (entries_processed < h->entry_count)
2091 {
2092 int count = MIN(h->entry_count - entries_processed, max_entries);
2093
2094 /* Read in as many entries as possible. */
2095 for (i = 0; i < count; i++)
2096 {
2097 struct temp_file_entry *tfe = &entrybuf[i];
2098 int datastart;
2099
2100 /* Read in numeric data. */
2101 if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2102 sizeof(struct temp_file_entry))
2103 {
2104 logf("read fail #1");
2105 close(masterfd);
2106 return false;
2107 }
2108
2109 datastart = lseek(tmpfd, 0, SEEK_CUR);
2110
2111 /**
2112 * Read string data from the following tags:
2113 * - tag_filename
2114 * - tag_artist
2115 * - tag_album
2116 * - tag_title
2117 *
2118 * A crc32 hash is calculated from the read data
2119 * and stored back to the data offset field kept in memory.
2120 */
2121 #define tmpdb_read_string_tag(tag) \
2122 lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2123 if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2124 { \
2125 logf("read fail: buffer overflow"); \
2126 close(masterfd); \
2127 return false; \
2128 } \
2129 \
2130 if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2131 tfe->tag_length[tag]) \
2132 { \
2133 logf("read fail #2"); \
2134 close(masterfd); \
2135 return false; \
2136 } \
2137 \
2138 tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2139 lseek(tmpfd, datastart, SEEK_SET)
2140
2141 tmpdb_read_string_tag(tag_filename);
2142 tmpdb_read_string_tag(tag_artist);
2143 tmpdb_read_string_tag(tag_album);
2144 tmpdb_read_string_tag(tag_title);
2145
2146 /* Seek to the end of the string data. */
2147 lseek(tmpfd, tfe->data_length, SEEK_CUR);
2148 }
2149
2150 /* Backup the master index position. */
2151 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2152 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2153
2154 /* Check if we can resurrect some deleted runtime statistics data. */
2155 for (i = 0; i < tcmh.tch.entry_count; i++)
2156 {
2157 /* Read the index entry. */
2158 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2159 != sizeof(struct index_entry))
2160 {
2161 logf("read fail #3");
2162 close(masterfd);
2163 return false;
2164 }
2165
2166 /**
2167 * Skip unless the entry is marked as being deleted
2168 * or the data has already been resurrected.
2169 */
2170 if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2171 continue;
2172
2173 /* Now try to match the entry. */
2174 /**
2175 * To succesfully match a song, the following conditions
2176 * must apply:
2177 *
2178 * For numeric fields: tag_length
2179 * - Full identical match is required
2180 *
2181 * If tag_filename matches, no further checking necessary.
2182 *
2183 * For string hashes: tag_artist, tag_album, tag_title
2184 * - Two of these must match
2185 */
2186 for (j = 0; j < count; j++)
2187 {
2188 struct temp_file_entry *tfe = &entrybuf[j];
2189
2190 /* Try to match numeric fields first. */
2191 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2192 continue;
2193
2194 /* Now it's time to do the hash matching. */
2195 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2196 {
2197 int match_count = 0;
2198
2199 /* No filename match, check if we can match two other tags. */
2200 #define tmpdb_match(tag) \
2201 if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2202 match_count++
2203
2204 tmpdb_match(tag_artist);
2205 tmpdb_match(tag_album);
2206 tmpdb_match(tag_title);
2207
2208 if (match_count < 2)
2209 {
2210 /* Still no match found, give up. */
2211 continue;
2212 }
2213 }
2214
2215 /* A match found, now copy & resurrect the statistical data. */
2216 #define tmpdb_copy_tag(tag) \
2217 tfe->tag_offset[tag] = idx.tag_seek[tag]
2218
2219 tmpdb_copy_tag(tag_playcount);
2220 tmpdb_copy_tag(tag_rating);
2221 tmpdb_copy_tag(tag_playtime);
2222 tmpdb_copy_tag(tag_lastplayed);
2223 tmpdb_copy_tag(tag_commitid);
2224
2225 /* Avoid processing this entry again. */
2226 idx.flag |= FLAG_RESURRECTED;
2227
2228 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2229 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2230 != sizeof(struct index_entry))
2231 {
2232 logf("masterfd writeback fail #1");
2233 close(masterfd);
2234 return false;
2235 }
2236
2237 logf("Entry resurrected");
2238 }
2239 }
2240
2241
2242 /* Restore the master index position. */
2243 lseek(masterfd, masterfd_pos, SEEK_SET);
2244
2245 /* Commit the data to the index. */
2246 for (i = 0; i < count; i++)
2247 {
2248 int loc = lseek(masterfd, 0, SEEK_CUR);
2249
2250 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2251 != sizeof(struct index_entry))
2252 {
2253 logf("read fail #3");
2254 close(masterfd);
2255 return false;
2256 }
2257
2258 for (j = 0; j < TAG_COUNT; j++)
2259 {
2260 if (!tagcache_is_numeric_tag(j))
2261 continue;
2262
2263 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2264 }
2265 idx.flag = entrybuf[i].flag;
2266
2267 if (idx.tag_seek[tag_commitid])
2268 {
2269 /* Data has been resurrected. */
2270 idx.flag |= FLAG_DIRTYNUM;
2271 }
2272 else if (tc_stat.ready && current_tcmh.commitid > 0)
2273 {
2274 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2275 idx.flag |= FLAG_DIRTYNUM;
2276 }
2277
2278 /* Write back the updated index. */
2279 lseek(masterfd, loc, SEEK_SET);
2280 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2281 != sizeof(struct index_entry))
2282 {
2283 logf("write fail");
2284 close(masterfd);
2285 return false;
2286 }
2287 }
2288
2289 entries_processed += count;
2290 logf("%d/%ld entries processed", entries_processed, h->entry_count);
2291 }
2292
2293 close(masterfd);
2294
2295 return true;
2296 }
2297
2298 /**
2299 * Return values:
2300 * > 0 success
2301 * == 0 temporary failure
2302 * < 0 fatal error
2303 */
2304 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2305 {
2306 int i;
2307 struct tagcache_header tch;
2308 struct master_header tcmh;
2309 struct index_entry idxbuf[IDX_BUF_DEPTH];
2310 int idxbuf_pos;
2311 char buf[TAG_MAXLEN+32];
2312 int fd = -1, masterfd;
2313 bool error = false;
2314 int init;
2315 int masterfd_pos;
2316
2317 logf("Building index: %d", index_type);
2318
2319 /* Check the number of entries we need to allocate ram for. */
2320 commit_entry_count = h->entry_count + 1;
2321
2322 masterfd = open_master_fd(&tcmh, false);
2323 if (masterfd >= 0)
2324 {
2325 commit_entry_count += tcmh.tch.entry_count;
2326 close(masterfd);
2327 }
2328 else
2329 remove_files(); /* Just to be sure we are clean. */
2330
2331 /* Open the index file, which contains the tag names. */
2332 fd = open_tag_fd(&tch, index_type, true);
2333 if (fd >= 0)
2334 {
2335 logf("tch.datasize=%ld", tch.datasize);
2336 lookup_buffer_depth = 1 +
2337 /* First part */ commit_entry_count +
2338 /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2339 }
2340 else
2341 {
2342 lookup_buffer_depth = 1 +
2343 /* First part */ commit_entry_count +
2344 /* Second part */ 0;
2345 }
2346
2347 logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2348 logf("commit_entry_count=%ld", commit_entry_count);
2349
2350 /* Allocate buffer for all index entries from both old and new
2351 * tag files. */
2352 tempbufidx = 0;
2353 tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2354
2355 /* Allocate lookup buffer. The first portion of commit_entry_count
2356 * contains the new tags in the temporary file and the second
2357 * part for locating entries already in the db.
2358 *
2359 * New tags Old tags
2360 * +---------+---------------------------+
2361 * | index | position/ENTRY_CHUNK_SIZE | lookup buffer
2362 * +---------+---------------------------+
2363 *
2364 * Old tags are inserted to a temporary buffer with position:
2365 * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2366 * And new tags with index:
2367 * tempbuf_insert(idx, ...);
2368 *
2369 * The buffer is sorted and written into tag file:
2370 * tempbuf_sort(...);
2371 * leaving master index locations messed up.
2372 *
2373 * That is fixed using the lookup buffer for old tags:
2374 * new_seek = tempbuf_find_location(old_seek, ...);
2375 * and for new tags:
2376 * new_seek = tempbuf_find_location(idx);
2377 */
2378 lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2379 tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2380 memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2381
2382 /* And calculate the remaining data space used mainly for storing
2383 * tag data (strings). */
2384 tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2385 if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2386 {
2387 logf("Buffer way too small!");
2388 return 0;
2389 }
2390
2391 if (fd >= 0)
2392 {
2393 /**
2394 * If tag file contains unique tags (sorted index), we will load
2395 * it entirely into memory so we can resort it later for use with
2396 * chunked browsing.
2397 */
2398 if (tagcache_is_sorted_tag(index_type))
2399 {
2400 logf("loading tags...");
2401 for (i = 0; i < tch.entry_count; i++)
2402 {
2403 struct tagfile_entry entry;
2404 int loc = lseek(fd, 0, SEEK_CUR);
2405 bool ret;
2406
2407 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2408 != sizeof(struct tagfile_entry))
2409 {
2410 logf("read error #7");
2411 close(fd);
2412 return -2;
2413 }
2414
2415 if (entry.tag_length >= (int)sizeof(buf))
2416 {
2417 logf("too long tag #3");
2418 close(fd);
2419 return -2;
2420 }
2421
2422 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2423 {
2424 logf("read error #8");
2425 close(fd);
2426 return -2;
2427 }
2428
2429 /* Skip deleted entries. */
2430 if (buf[0] == '\0')
2431 continue;
2432
2433 /**
2434 * Save the tag and tag id in the memory buffer. Tag id
2435 * is saved so we can later reindex the master lookup
2436 * table when the index gets resorted.
2437 */
2438 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2439 + commit_entry_count, entry.idx_id,
2440 tagcache_is_unique_tag(index_type));
2441 if (!ret)
2442 {
2443 close(fd);
2444 return -3;
2445 }
2446 do_timed_yield();
2447 }
2448 logf("done");
2449 }
2450 else
2451 tempbufidx = tch.entry_count;
2452 }
2453 else
2454 {
2455 /**
2456 * Creating new index file to store the tags. No need to preload
2457 * anything whether the index type is sorted or not.
2458 */
2459 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2460 fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2461 if (fd < 0)
2462 {
2463 logf("%s open fail", buf);
2464 return -2;
2465 }
2466
2467 tch.magic = TAGCACHE_MAGIC;
2468 tch.entry_count = 0;
2469 tch.datasize = 0;
2470
2471 if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2472 != sizeof(struct tagcache_header))
2473 {
2474 logf("header write failed");
2475 close(fd);
2476 return -2;
2477 }
2478 }
2479
2480 /* Loading the tag lookup file as "master file". */
2481 logf("Loading index file");
2482 masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2483
2484 if (masterfd < 0)
2485 {
2486 logf("Creating new DB");
2487 masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2488
2489 if (masterfd < 0)
2490 {
2491 logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2492 close(fd);
2493 return -2;
2494 }
2495
2496 /* Write the header (write real values later). */
2497 memset(&tcmh, 0, sizeof(struct master_header));
2498 tcmh.tch = *h;
2499 tcmh.tch.entry_count = 0;
2500 tcmh.tch.datasize = 0;
2501 tcmh.dirty = true;
2502 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2503 init = true;
2504 masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2505 }
2506 else
2507 {
2508 /**
2509 * Master file already exists so we need to process the current
2510 * file first.
2511 */
2512 init = false;
2513
2514 if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2515 sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2516 {
2517 logf("header error");
2518 close(fd);
2519 close(masterfd);
2520 return -2;
2521 }
2522
2523 /**
2524 * If we reach end of the master file, we need to expand it to
2525 * hold new tags. If the current index is not sorted, we can
2526 * simply append new data to end of the file.
2527 * However, if the index is sorted, we need to update all tag
2528 * pointers in the master file for the current index.
2529 */
2530 masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2531 SEEK_CUR);
2532 if (masterfd_pos == filesize(masterfd))
2533 {
2534 logf("appending...");
2535 init = true;
2536 }
2537 }
2538
2539 /**
2540 * Load new unique tags in memory to be sorted later and added
2541 * to the master lookup file.
2542 */
2543 if (tagcache_is_sorted_tag(index_type))
2544 {
2545 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2546 /* h is the header of the temporary file containing new tags. */
2547 logf("inserting new tags...");
2548 for (i = 0; i < h->entry_count; i++)
2549 {
2550 struct temp_file_entry entry;
2551
2552 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2553 sizeof(struct temp_file_entry))
2554 {
2555 logf("read fail #3");
2556 error = true;
2557 goto error_exit;
2558 }
2559
2560 /* Read data. */
2561 if (entry.tag_length[index_type] >= (long)sizeof(buf))
2562 {
2563 logf("too long entry!");
2564 error = true;
2565 goto error_exit;
2566 }
2567
2568 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2569 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2570 entry.tag_length[index_type])
2571 {
2572 logf("read fail #4");
2573 error = true;
2574 goto error_exit;
2575 }
2576
2577 if (tagcache_is_unique_tag(index_type))
2578 error = !tempbuf_insert(buf, i, -1, true);
2579 else
2580 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2581
2582 if (error)
2583 {
2584 logf("insert error");
2585 goto error_exit;
2586 }
2587
2588 /* Skip to next. */
2589 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2590 entry.tag_length[index_type], SEEK_CUR);
2591 do_timed_yield();
2592 }
2593 logf("done");
2594
2595 /* Sort the buffer data and write it to the index file. */
2596 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2597 i = tempbuf_sort(fd);
2598 if (i < 0)
2599 goto error_exit;
2600 logf("sorted %d tags", i);
2601
2602 /**
2603 * Now update all indexes in the master lookup file.
2604 */
2605 logf("updating indices...");
2606 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2607 for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2608 {
2609 int j;
2610 int loc = lseek(masterfd, 0, SEEK_CUR);
2611
2612 idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2613
2614 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2615 != (int)sizeof(struct index_entry)*idxbuf_pos)
2616 {
2617 logf("read fail #5");
2618 error = true;
2619 goto error_exit ;
2620 }
2621 lseek(masterfd, loc, SEEK_SET);
2622
2623 for (j = 0; j < idxbuf_pos; j++)
2624 {
2625 if (idxbuf[j].flag & FLAG_DELETED)
2626 {
2627 /* We can just ignore deleted entries. */
2628 // idxbuf[j].tag_seek[index_type] = 0;
2629 continue;
2630 }
2631
2632 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2633 idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2634 + commit_entry_count);
2635
2636 if (idxbuf[j].tag_seek[index_type] < 0)
2637 {
2638 logf("update error: %d/%d/%d",
2639 idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2640 error = true;
2641 goto error_exit;
2642 }
2643
2644 do_timed_yield();
2645 }
2646
2647 /* Write back the updated index. */
2648 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2649 index_entry_ec, tc_stat.econ) !=
2650 (int)sizeof(struct index_entry)*idxbuf_pos)
2651 {
2652 logf("write fail");
2653 error = true;
2654 goto error_exit;
2655 }
2656 }
2657 logf("done");
2658 }
2659
2660 /**
2661 * Walk through the temporary file containing the new tags.
2662 */
2663 // build_normal_index(h, tmpfd, masterfd, idx);
2664 logf("updating new indices...");
2665 lseek(masterfd, masterfd_pos, SEEK_SET);
2666 lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2667 lseek(fd, 0, SEEK_END);
2668 for (i = 0; i < h->entry_count; i += idxbuf_pos)
2669 {
2670 int j;
2671
2672 idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2673 if (init)
2674 {
2675 memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2676 }
2677 else
2678 {
2679 int loc = lseek(masterfd, 0, SEEK_CUR);
2680
2681 if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2682 != (int)sizeof(struct index_entry)*idxbuf_pos)
2683 {
2684 logf("read fail #6");
2685 error = true;
2686 break ;
2687 }
2688 lseek(masterfd, loc, SEEK_SET);
2689 }
2690
2691 /* Read entry headers. */
2692 for (j = 0; j < idxbuf_pos; j++)
2693 {
2694 if (!tagcache_is_sorted_tag(index_type))
2695 {
2696 struct temp_file_entry entry;
2697 struct tagfile_entry fe;
2698
2699 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2700 sizeof(struct temp_file_entry))
2701 {
2702 logf("read fail #7");
2703 error = true;
2704 break ;
2705 }
2706
2707 /* Read data. */
2708 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2709 {
2710 logf("too long entry!");
2711 logf("length=%d", entry.tag_length[index_type]);
2712 logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2713 error = true;
2714 break ;
2715 }
2716
2717 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2718 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2719 entry.tag_length[index_type])
2720 {
2721 logf("read fail #8");
2722 logf("offset=0x%02lx", entry.tag_offset[index_type]);
2723 logf("length=0x%02x", entry.tag_length[index_type]);
2724 error = true;
2725 break ;
2726 }
2727
2728 /* Write to index file. */
2729 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2730 fe.tag_length = entry.tag_length[index_type];
2731 fe.idx_id = tcmh.tch.entry_count + i + j;
2732 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2733 write(fd, buf, fe.tag_length);
2734 tempbufidx++;
2735
2736 /* Skip to next. */
2737 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2738 entry.tag_length[index_type], SEEK_CUR);
2739 }
2740 else
2741 {
2742 /* Locate the correct entry from the sorted array. */
2743 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2744 if (idxbuf[j].tag_seek[index_type] < 0)
2745 {
2746 logf("entry not found (%d)", j);
2747 error = true;
2748 break ;
2749 }
2750 }
2751 }
2752
2753 /* Write index. */
2754 if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2755 index_entry_ec, tc_stat.econ) !=
2756 (int)sizeof(struct index_entry)*idxbuf_pos)
2757 {
2758 logf("tagcache: write fail #4");
2759 error = true;
2760 break ;
2761 }
2762
2763 do_timed_yield();
2764 }
2765 logf("done");
2766
2767 /* Finally write the header. */
2768 tch.magic = TAGCACHE_MAGIC;
2769 tch.entry_count = tempbufidx;
2770 tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2771 lseek(fd, 0, SEEK_SET);
2772 ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2773
2774 if (index_type != tag_filename)
2775 h->datasize += tch.datasize;
2776 logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2777 error_exit:
2778
2779 close(fd);
2780 close(masterfd);
2781
2782 if (error)
2783 return -2;
2784
2785 return 1;
2786 }
2787
2788 static bool commit(void)
2789 {
2790 struct tagcache_header tch;
2791 struct master_header tcmh;
2792 int i, len, rc;
2793 int tmpfd;
2794 int masterfd;
2795 #ifdef HAVE_DIRCACHE
2796 bool dircache_buffer_stolen = false;
2797 #endif
2798 bool local_allocation = false;
2799
2800 logf("committing tagcache");
2801
2802 while (write_lock)
2803 sleep(1);
2804
2805 tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2806 if (tmpfd < 0)
2807 {
2808 logf("nothing to commit");
2809 return true;
2810 }
2811
2812
2813 /* Load the header. */
2814 len = sizeof(struct tagcache_header);
2815 rc = read(tmpfd, &tch, len);
2816
2817 if (tch.magic != TAGCACHE_MAGIC || rc != len)
2818 {
2819 logf("incorrect header");
2820 close(tmpfd);
2821 remove(TAGCACHE_FILE_TEMP);
2822 return false;
2823 }
2824
2825 if (tch.entry_count == 0)
2826 {
2827 logf("nothing to commit");
2828 close(tmpfd);
2829 remove(TAGCACHE_FILE_TEMP);
2830 return true;
2831 }
2832
2833 #ifdef HAVE_EEPROM_SETTINGS
2834 remove(TAGCACHE_STATEFILE);
2835 #endif
2836
2837 /* At first be sure to unload the ramcache! */
2838 #ifdef HAVE_TC_RAMCACHE
2839 tc_stat.ramcache = false;
2840 #endif
2841
2842 read_lock++;
2843
2844 /* Try to steal every buffer we can :) */
2845 if (tempbuf_size == 0)
2846 local_allocation = true;
2847
2848 #ifdef HAVE_DIRCACHE
2849 if (tempbuf_size == 0)
2850 {
2851 /* Try to steal the dircache buffer. */
2852 tempbuf = dircache_steal_buffer(&tempbuf_size);
2853 tempbuf_size &= ~0x03;
2854
2855 if (tempbuf_size > 0)
2856 {
2857 dircache_buffer_stolen = true;
2858 }
2859 }
2860 #endif
2861
2862 #ifdef HAVE_TC_RAMCACHE
2863 if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2864 {
2865 tempbuf = (char *)(hdr + 1);
2866 tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2867 tempbuf_size &= ~0x03;
2868 }
2869 #endif
2870
2871 /* And finally fail if there are no buffers available. */
2872 if (tempbuf_size == 0)
2873 {
2874 logf("delaying commit until next boot");
2875 tc_stat.commit_delayed = true;
2876 close(tmpfd);
2877 read_lock--;
2878 return false;
2879 }
2880
2881 logf("commit %ld entries...", tch.entry_count);
2882
2883 /* Mark DB dirty so it will stay disabled if commit fails. */
2884 current_tcmh.dirty = true;
2885 update_master_header();
2886
2887 /* Now create the index files. */
2888 tc_stat.commit_step = 0;
2889 tch.datasize = 0;
2890 tc_stat.commit_delayed = false;
2891
2892 for (i = 0; i < TAG_COUNT; i++)
2893 {
2894 int ret;
2895
2896 if (tagcache_is_numeric_tag(i))
2897 continue;
2898
2899 tc_stat.commit_step++;
2900 ret = build_index(i, &tch, tmpfd);
2901 if (ret <= 0)
2902 {
2903 close(tmpfd);
2904 logf("tagcache failed init");
2905 if (ret == 0)
2906 tc_stat.commit_delayed = true;
2907
2908 tc_stat.commit_step = 0;
2909 read_lock--;
2910 return false;
2911 }
2912 }
2913
2914 if (!build_numeric_indices(&tch, tmpfd))
2915 {
2916 logf("Failure to commit numeric indices");
2917 close(tmpfd);
2918 tc_stat.commit_step = 0;
2919 read_lock--;
2920 return false;
2921 }
2922
2923 close(tmpfd);
2924 remove(TAGCACHE_FILE_TEMP);
2925
2926 tc_stat.commit_step = 0;
2927
2928 /* Update the master index headers. */
2929 if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2930 {
2931 read_lock--;
2932 return false;
2933 }
2934
2935 tcmh.tch.entry_count += tch.entry_count;
2936 tcmh.tch.datasize = sizeof(struct master_header)
2937 + sizeof(struct index_entry) * tcmh.tch.entry_count
2938 + tch.datasize;
2939 tcmh.dirty = false;
2940 tcmh.commitid++;
2941
2942 lseek(masterfd, 0, SEEK_SET);
2943 ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2944 close(masterfd);
2945
2946 logf("tagcache committed");
2947 tc_stat.ready = check_all_headers();
2948 tc_stat.readyvalid = true;
2949
2950 if (local_allocation)
2951 {
2952 tempbuf = NULL;
2953 tempbuf_size = 0;
2954 }
2955
2956 #ifdef HAVE_DIRCACHE
2957 /* Rebuild the dircache, if we stole the buffer. */
2958 if (dircache_buffer_stolen)
2959 dircache_build(0);
2960 #endif
2961
2962 #ifdef HAVE_TC_RAMCACHE
2963 /* Reload tagcache. */
2964 if (tc_stat.ramcache_allocated > 0)
2965 tagcache_start_scan();
2966 #endif
2967
2968 read_lock--;
2969
2970 return true;
2971 }
2972
2973 static void allocate_tempbuf(void)
2974 {
2975 /* Yeah, malloc would be really nice now :) */
2976 #ifdef __PCTOOL__
2977 tempbuf_size = 32*1024*1024;
2978 tempbuf = malloc(tempbuf_size);
2979 #else
2980 tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2981 tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2982 audiobuf += tempbuf_size;
2983 #endif
2984 }
2985
2986 static void free_tempbuf(void)
2987 {
2988 if (tempbuf_size == 0)
2989 return ;
2990
2991 #ifdef __PCTOOL__
2992 free(tempbuf);
2993 #else
2994 audiobuf -= tempbuf_size;
2995 #endif
2996 tempbuf = NULL;
2997 tempbuf_size = 0;
2998 }
2999
3000 #ifndef __PCTOOL__
3001
3002 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3003 {
3004 struct index_entry idx;
3005
3006 if (!tc_stat.ready)
3007 return false;
3008
3009 if (!tagcache_is_numeric_tag(tag))
3010 return false;
3011
3012 if (!get_index(masterfd, idx_id, &idx, false))
3013 return false;
3014
3015 idx.tag_seek[tag] = data;
3016 idx.flag |= FLAG_DIRTYNUM;
3017
3018 return write_index(masterfd, idx_id, &idx);
3019 }
3020
3021 #if 0
3022 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3023 int tag, long data)
3024 {
3025 struct master_header myhdr;
3026
3027 if (tcs->masterfd < 0)
3028 {
3029 if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3030 return false;
3031 }
3032
3033 return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3034 }
3035 #endif
3036
3037 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3038
3039 static bool command_queue_is_full(void)
3040 {
3041 int next;
3042
3043 next = command_queue_widx + 1;
3044 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3045 next = 0;
3046
3047 return (next == command_queue_ridx);
3048 }
3049
3050 static bool command_queue_sync_callback(void)
3051 {
3052
3053 struct master_header myhdr;
3054 int masterfd;
3055
3056 mutex_lock(&command_queue_mutex);
3057
3058 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3059 return false;
3060
3061 while (command_queue_ridx != command_queue_widx)
3062 {
3063 struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3064
3065 switch (ce->command)
3066 {
3067 case CMD_UPDATE_MASTER_HEADER:
3068 {
3069 close(masterfd);
3070 update_master_header();
3071
3072 /* Re-open the masterfd. */
3073 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3074 return true;
3075
3076 break;
3077 }
3078 case CMD_UPDATE_NUMERIC:
3079 {
3080 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3081 break;
3082 }
3083 }
3084
3085 if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3086 command_queue_ridx = 0;
3087 }
3088
3089 close(masterfd);
3090
3091 tc_stat.queue_length = 0;
3092 mutex_unlock(&command_queue_mutex);
3093 return true;
3094 }
3095
3096 static void run_command_queue(bool force)
3097 {
3098 if (COMMAND_QUEUE_IS_EMPTY)
3099 return;
3100
3101 if (force || command_queue_is_full())
3102 command_queue_sync_callback();
3103 else
3104 register_storage_idle_func(command_queue_sync_callback);
3105 }
3106
3107 static void queue_command(int cmd, long idx_id, int tag, long data)
3108 {
3109 while (1)
3110 {
3111 int next;
3112
3113 mutex_lock(&command_queue_mutex);
3114 next = command_queue_widx + 1;
3115 if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3116 next = 0;
3117
3118 /* Make sure queue is not full. */
3119 if (next != command_queue_ridx)
3120 {
3121 struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3122
3123 ce->command = cmd;
3124 ce->idx_id = idx_id;
3125 ce->tag = tag;
3126 ce->data = data;
3127
3128 command_queue_widx = next;
3129
3130 tc_stat.queue_length++;
3131
3132 mutex_unlock(&command_queue_mutex);
3133 break;
3134 }
3135
3136 /* Queue is full, try again later... */
3137 mutex_unlock(&command_queue_mutex);
3138 sleep(1);
3139 }
3140 }
3141
3142 long tagcache_increase_serial(void)
3143 {
3144 long old;
3145
3146 if (!tc_stat.ready)
3147 return -2;
3148
3149 while (read_lock)
3150 sleep(1);
3151
3152 old = current_tcmh.serial++;
3153 queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3154
3155 return old;
3156 }
3157
3158 void tagcache_update_numeric(int idx_id, int tag, long data)
3159 {
3160 queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3161 }
3162 #endif /* !__PCTOOL__ */
3163
3164 long tagcache_get_serial(void)
3165 {
3166 return current_tcmh.serial;
3167 }
3168
3169 long tagcache_get_commitid(void)
3170 {
3171 return current_tcmh.commitid;
3172 }
3173
3174 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3175 {
3176 char buf[512];
3177 int i;
3178
3179 snprintf(buf, sizeof buf, "%s=\"", tagstr);
3180 for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3181 {
3182 if (*datastr == '\0')
3183 break;
3184
3185 if (*datastr == '"' || *datastr == '\\')
3186 buf[i++] = '\\';
3187
3188 buf[i] = *(datastr++);
3189 }
3190
3191 strcpy(&buf[i], "\" ");
3192
3193 write(fd, buf, i + 2);
3194
3195 return true;
3196 }
3197
3198 #ifndef __PCTOOL__
3199
3200 static bool read_tag(char *dest, long size,
3201 const char *src, const char *tagstr)
3202 {
3203 int pos;
3204 char current_tag[32];
3205
3206 while (*src != '\0')
3207 {
3208 /* Skip all whitespace */
3209 while (*src == ' ')
3210 src++;
3211
3212 if (*src == '\0')
3213 break;
3214
3215 pos = 0;
3216 /* Read in tag name */
3217 while (*src != '=' && *src != ' ')
3218 {
3219 current_tag[pos] = *src;
3220 src++;
3221 pos++;
3222
3223 if (*src == '\0' || pos >= (long)sizeof(current_tag))
3224 return false;
3225 }
3226 current_tag[pos] = '\0';
3227
3228 /* Read in tag data */
3229
3230 /* Find the start. */
3231 while (*src != '"' && *src != '\0')
3232 src++;
3233
3234 if (*src == '\0' || *(++src) == '\0')
3235 return false;
3236
3237 /* Read the data. */
3238 for (pos = 0; pos < size; pos++)
3239 {
3240 if (*src == '\0')
3241 break;
3242
3243 if (*src == '\\')
3244 {
3245 dest[pos] = *(src+1);
3246 src += 2;
3247 continue;
3248 }
3249
3250 dest[pos] = *src;
3251
3252 if (*src == '"')
3253 {
3254 src++;
3255 break;
3256 }
3257
3258 if (*src == '\0')
3259 break;
3260
3261 src++;
3262 }
3263 dest[pos] = '\0';
3264
3265 if (!strcasecmp(tagstr, current_tag))
3266 return true;
3267 }
3268
3269 return false;
3270 }
3271
3272 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3273 {
3274 struct index_entry idx;
3275 char tag_data[TAG_MAXLEN+32];
3276 int idx_id;
3277 long masterfd = (long)parameters;
3278 const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3279 tag_commitid };
3280 int i;
3281 (void)line_n;
3282
3283 if (*buf == '#')
3284 return 0;
3285
3286 logf("%d/%s", line_n, buf);
3287 if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3288 {
3289 logf("filename missing");
3290 logf("-> %s", buf);
3291 return 0;
3292 }
3293
3294 idx_id = find_index(tag_data);
3295 if (idx_id < 0)
3296 {
3297 logf("entry not found");
3298 return 0;
3299 }
3300
3301 if (!get_index(masterfd, idx_id, &idx, false))
3302 {
3303 logf("failed to retrieve index entry");
3304 return 0;
3305 }
3306
3307 /* Stop if tag has already been modified. */
3308 if (idx.flag & FLAG_DIRTYNUM)
3309 return 0;
3310
3311 logf("import: %s", tag_data);
3312
3313 idx.flag |= FLAG_DIRTYNUM;
3314 for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3315 {
3316 int data;
3317
3318 if (!read_tag(tag_data, sizeof tag_data, buf,
3319 tagcache_tag_to_str(import_tags[i])))
3320 {
3321 continue;
3322 }
3323
3324 data = atoi(tag_data);
3325 if (data < 0)
3326 continue;
3327
3328 idx.tag_seek[import_tags[i]] = data;
3329
3330 if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3331 current_tcmh.serial = data + 1;
3332 else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3333 current_tcmh.commitid = data + 1;
3334 }
3335
3336 return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3337 }
3338
3339 bool tagcache_import_changelog(void)
3340 {
3341 struct master_header myhdr;
3342 struct tagcache_header tch;
3343 int clfd;
3344 long masterfd;
3345 char buf[2048];
3346
3347 if (!tc_stat.ready)
3348 return false;
3349
3350 while (read_lock)
3351 sleep(1);
3352
3353 clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3354 if (clfd < 0)
3355 {
3356 logf("failure to open changelog");
3357 return false;
3358 }
3359
3360 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3361 {
3362 close(clfd);
3363 return false;
3364 }
3365
3366 write_lock++;
3367
3368 filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3369
3370 fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3371 parse_changelog_line);
3372
3373 close(clfd);
3374 close(masterfd);
3375
3376 if (filenametag_fd >= 0)
3377 close(filenametag_fd);
3378
3379 write_lock--;
3380
3381 update_master_header();
3382
3383 return true;
3384 }
3385
3386 #endif /* !__PCTOOL__ */
3387
3388 bool tagcache_create_changelog(struct tagcache_search *tcs)
3389 {
3390 struct master_header myhdr;
3391 struct index_entry idx;
3392 char buf[TAG_MAXLEN+32];
3393 char temp[32];
3394 int clfd;
3395 int i, j;
3396
3397 if (!tc_stat.ready)
3398 return false;
3399
3400 if (!tagcache_search(tcs, tag_filename))
3401 return false;
3402
3403 /* Initialize the changelog */
3404 clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3405 if (clfd < 0)
3406 {
3407 logf("failure to open changelog");
3408 return false;
3409 }
3410
3411 if (tcs->masterfd < 0)
3412 {
3413 if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3414 return false;
3415 }
3416 else
3417 {
3418 lseek(tcs->masterfd, 0, SEEK_SET);
3419 ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3420 }
3421
3422 write(clfd, "## Changelog version 1\n", 23);
3423
3424 for (i = 0; i < myhdr.tch.entry_count; i++)
3425 {
3426 if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3427 != sizeof(struct index_entry))
3428 {
3429 logf("read error #9");
3430 tagcache_search_finish(tcs);
3431 close(clfd);
3432 return false;
3433 }
3434
3435 /* Skip until the entry found has been modified. */
3436 if (! (idx.flag & FLAG_DIRTYNUM) )
3437 continue;
3438
3439 /* Skip deleted entries too. */
3440 if (idx.flag & FLAG_DELETED)
3441 continue;
3442
3443 /* Now retrieve all tags. */
3444 for (j = 0; j < TAG_COUNT; j++)
3445 {
3446 if (tagcache_is_numeric_tag(j))
3447 {
3448 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3449 write_tag(clfd, tagcache_tag_to_str(j), temp);
3450 continue;
3451 }
3452
3453 tcs->type = j;
3454 tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3455 write_tag(clfd, tagcache_tag_to_str(j), buf);
3456 }
3457
3458 write(clfd, "\n", 1);
3459 do_timed_yield();
3460 }
3461
3462 close(clfd);
3463
3464 tagcache_search_finish(tcs);
3465
3466 return true;
3467 }
3468
3469 static bool delete_entry(long idx_id)
3470 {
3471 int fd = -1;
3472 int masterfd = -1;
3473 int tag, i;
3474 struct index_entry idx, myidx;
3475 struct master_header myhdr;
3476 char buf[TAG_MAXLEN+32];
3477 int in_use[TAG_COUNT];
3478
3479 logf("delete_entry(): %ld", idx_id);
3480
3481 #ifdef HAVE_TC_RAMCACHE
3482 /* At first mark the entry removed from ram cache. */
3483 if (tc_stat.ramcache)
3484 hdr->indices[idx_id].flag |= FLAG_DELETED;
3485 #endif
3486
3487 if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3488 return false;
3489
3490 lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3491 if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3492 != sizeof(struct index_entry))
3493 {
3494 logf("delete_entry(): read error");
3495 goto cleanup;
3496 }
3497
3498 if (myidx.flag & FLAG_DELETED)
3499 {
3500 logf("delete_entry(): already deleted!");
3501 goto cleanup;
3502 }
3503
3504 myidx.flag |= FLAG_DELETED;
3505 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3506 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3507 != sizeof(struct index_entry))
3508 {
3509 logf("delete_entry(): write_error #1");
3510 goto cleanup;
3511 }
3512
3513 /* Now check which tags are no longer in use (if any) */
3514 for (tag = 0; tag < TAG_COUNT; tag++)
3515 in_use[tag] = 0;
3516
3517 lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3518 for (i = 0; i < myhdr.tch.entry_count; i++)
3519 {
3520 struct index_entry *idxp;
3521
3522 #ifdef HAVE_TC_RAMCACHE
3523 /* Use RAM DB if available for greater speed */
3524 if (tc_stat.ramcache)
3525 idxp = &hdr->indices[i];
3526 else
3527 #endif
3528 {
3529 if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3530 != sizeof(struct index_entry))
3531 {
3532 logf("delete_entry(): read error #2");
3533 goto cleanup;
3534 }
3535 idxp = &idx;
3536 }
3537
3538 if (idxp->flag & FLAG_DELETED)
3539 continue;
3540
3541 for (tag = 0; tag < TAG_COUNT; tag++)
3542 {
3543 if (tagcache_is_numeric_tag(tag))
3544 continue;
3545
3546 if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3547 in_use[tag]++;
3548 }
3549 }
3550
3551 /* Now delete all tags no longer in use. */
3552 for (tag = 0; tag < TAG_COUNT; tag++)
3553 {
3554 struct tagcache_header tch;
3555 int oldseek = myidx.tag_seek[tag];
3556
3557 if (tagcache_is_numeric_tag(tag))
3558 continue;
3559
3560 /**
3561 * Replace tag seek with a hash value of the field string data.
3562 * That way runtime statistics of moved or altered files can be
3563 * resurrected.
3564 */
3565 #ifdef HAVE_TC_RAMCACHE
3566 if (tc_stat.ramcache && tag != tag_filename)
3567 {
3568 struct tagfile_entry *tfe;
3569 int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3570
3571 tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3572 *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3573 myidx.tag_seek[tag] = *seek;
3574 }
3575 else
3576 #endif
3577 {
3578 struct tagfile_entry tfe;
3579
3580 /* Open the index file, which contains the tag names. */
3581 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3582 goto cleanup;
3583
3584 /* Skip the header block */
3585 lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3586 if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3587 != sizeof(struct tagfile_entry))
3588 {
3589 logf("delete_entry(): read error #3");
3590 goto cleanup;
3591 }
3592
3593 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3594 {
3595 logf("delete_entry(): read error #4");
3596 goto cleanup;
3597 }
3598
3599 myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3600 }
3601
3602 if (in_use[tag])
3603 {
3604 logf("in use: %d/%d", tag, in_use[tag]);
3605 if (fd >= 0)
3606 {
3607 close(fd);
3608 fd = -1;
3609 }
3610 continue;
3611 }
3612
3613 #ifdef HAVE_TC_RAMCACHE
3614 /* Delete from ram. */
3615 if (tc_stat.ramcache && tag != tag_filename)
3616 {
3617 struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3618 tagentry->tag_data[0] = '\0';
3619 }
3620 #endif
3621
3622 /* Open the index file, which contains the tag names. */
3623 if (fd < 0)
3624 {
3625 if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3626 goto cleanup;
3627 }
3628
3629 /* Skip the header block */
3630 lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3631
3632 /* Debug, print 10 first characters of the tag
3633 read(fd, buf, 10);
3634 buf[10]='\0';
3635 logf("TAG:%s", buf);
3636 lseek(fd, -10, SEEK_CUR);
3637 */
3638
3639 /* Write first data byte in tag as \0 */
3640 write(fd, "", 1);
3641
3642 /* Now tag data has been removed */
3643 close(fd);
3644 fd = -1;
3645 }
3646
3647 /* Write index entry back into master index. */
3648 lseek(masterfd, sizeof(struct master_header) +
3649 (idx_id * sizeof(struct index_entry)), SEEK_SET);
3650 if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3651 != sizeof(struct index_entry))
3652 {
3653 logf("delete_entry(): write_error #2");
3654 goto cleanup;
3655 }
3656
3657 close(masterfd);
3658
3659 return true;
3660
3661 cleanup:
3662 if (fd >= 0)
3663 close(fd);
3664 if (masterfd >= 0)
3665 close(masterfd);
3666
3667 return false;
3668 }
3669
3670 #ifndef __PCTOOL__
3671 /**
3672 * Returns true if there is an event waiting in the queue
3673 * that requires the current operation to be aborted.
3674 */
3675 static bool check_event_queue(void)
3676 {
3677 struct queue_event ev;
3678
3679 queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3680 switch (ev.id)
3681 {
3682 case Q_STOP_SCAN:
3683 case SYS_POWEROFF:
3684 case SYS_USB_CONNECTED:
3685 /* Put the event back into the queue. */
3686 queue_post(&tagcache_queue, ev.id, ev.data);
3687 return true;
3688 }
3689
3690 return false;
3691 }
3692 #endif
3693
3694 #ifdef HAVE_TC_RAMCACHE
3695 static bool allocate_tagcache(void)
3696 {
3697 struct master_header tcmh;
3698 int fd;
3699
3700 /* Load the header. */
3701 if ( (fd = open_master_fd(&tcmh, false)) < 0)
3702 {
3703 hdr = NULL;
3704 return false;
3705 }
3706
3707 close(fd);
3708
3709 /**
3710 * Now calculate the required cache size plus
3711 * some extra space for alignment fixes.
3712 */
3713 tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3714 sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3715 hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3716 memset(hdr, 0, sizeof(struct ramcache_header));
3717 memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3718 hdr->indices = (struct index_entry *)(hdr + 1);
3719 logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3720
3721 return true;
3722 }
3723
3724 # ifdef HAVE_EEPROM_SETTINGS
3725 static bool tagcache_dumpload(void)
3726 {
3727 struct statefile_header shdr;
3728 int fd, rc;
3729 long offpos;
3730 int i;
3731
3732 fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3733 if (fd < 0)
3734 {
3735 logf("no tagcache statedump");
3736 return false;
3737 }
3738
3739 /* Check the statefile memory placement */
3740 hdr = buffer_alloc(0);
3741 rc = read(fd, &shdr, sizeof(struct statefile_header));
3742 if (rc != sizeof(struct statefile_header)
3743 /* || (long)hdr != (long)shdr.hdr */)
3744 {
3745 logf("incorrect statefile");
3746 hdr = NULL;
3747 close(fd);
3748 return false;
3749 }
3750
3751 offpos = (long)hdr - (long)shdr.hdr;
3752
3753 /* Lets allocate real memory and load it */
3754 hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3755 rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3756 close(fd);
3757
3758 if (rc != shdr.tc_stat.ramcache_allocated)
3759 {
3760 logf("read failure!");
3761 hdr = NULL;
3762 return false;
3763 }
3764
3765 memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3766
3767 /* Now fix the pointers */
3768 hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3769 for (i = 0; i < TAG_COUNT; i++)
3770 hdr->tags[i] += offpos;
3771
3772 return true;
3773 }
3774
3775 static bool tagcache_dumpsave(void)
3776 {
3777 struct statefile_header shdr;
3778 int fd;
3779
3780 if (!tc_stat.ramcache)
3781 return false;
3782
3783 fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3784 if (fd < 0)
3785 {
3786 logf("failed to create a statedump");
3787 return false;
3788 }
3789
3790 /* Create the header */
3791 shdr.hdr = hdr;
3792 memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3793 write(fd, &shdr, sizeof(struct statefile_header));
3794
3795 /* And dump the data too */
3796 write(fd, hdr, tc_stat.ramcache_allocated);
3797 close(fd);
3798
3799 return true;
3800 }
3801 # endif
3802
3803 static bool load_tagcache(void)
3804 {
3805 struct tagcache_header *tch;
3806 long bytesleft = tc_stat.ramcache_allocated;
3807 struct index_entry *idx;
3808 int rc, fd;
3809 char *p;
3810 int i, tag;
3811
3812 # ifdef HAVE_DIRCACHE
3813 while (dircache_is_initializing())
3814 sleep(1);
3815
3816 dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3817 # endif
3818
3819 logf("loading tagcache to ram...");
3820
3821 fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3822 if (fd < 0)
3823 {
3824 logf("tagcache open failed");
3825 return false;
3826 }
3827
3828 if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3829 != sizeof(struct master_header)
3830 || hdr->h.tch.magic != TAGCACHE_MAGIC)
3831 {
3832 logf("incorrect header");
3833 return false;
3834 }
3835
3836 idx = hdr->indices;
3837
3838 /* Load the master index table. */
3839 for (i = 0; i < hdr->h.tch.entry_count; i++)
3840 {
3841 rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3842 if (rc != sizeof(struct index_entry))
3843 {
3844 logf("read error #10");
3845 close(fd);
3846 return false;
3847 }
3848
3849 bytesleft -= sizeof(struct index_entry);
3850 if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3851 {
3852 logf("too big tagcache.");
3853 close(fd);
3854 return false;
3855 }
3856
3857 idx++;
3858 }
3859
3860 close(fd);
3861
3862 /* Load the tags. */
3863 p = (char *)idx;
3864 for (tag = 0; tag < TAG_COUNT; tag++)
3865 {
3866 struct tagfile_entry *fe;
3867 char buf[TAG_MAXLEN+32];
3868
3869 if (tagcache_is_numeric_tag(tag))
3870 continue ;
3871
3872 //p = ((void *)p+1);
3873 p = (char *)((long)p & ~0x03) + 0x04;
3874 hdr->tags[tag] = p;
3875
3876 /* Check the header. */
3877 tch = (struct tagcache_header *)p;
3878 p += sizeof(struct tagcache_header);
3879
3880 if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3881 return false;
3882
3883 for (hdr->entry_count[tag] = 0;
3884 hdr->entry_count[tag] < tch->entry_count;
3885 hdr->entry_count[tag]++)
3886 {
3887 long pos;
3888
3889 if (do_timed_yield())
3890 {
3891 /* Abort if we got a critical event in queue */
3892 if (check_event_queue())
3893 return false;
3894 }
3895
3896 fe = (struct tagfile_entry *)p;
3897 pos = lseek(fd, 0, SEEK_CUR);
3898 rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3899 if (rc != sizeof(struct tagfile_entry))
3900 {
3901 /* End of lookup table. */
3902 logf("read error #11");
3903 close(fd);
3904 return false;
3905 }
3906
3907 /* We have a special handling for the filename tags. */
3908 if (tag == tag_filename)
3909 {
3910 # ifdef HAVE_DIRCACHE
3911 const struct dirent *dc;
3912 # endif
3913
3914 // FIXME: This is wrong!
3915 // idx = &hdr->indices[hdr->entry_count[i]];
3916 idx = &hdr->indices[fe->idx_id];
3917
3918 if (fe->tag_length >= (long)sizeof(buf)-1)
3919 {
3920 read(fd, buf, 10);
3921 buf[10] = '\0';
3922 logf("TAG:%s", buf);
3923 logf("too long filename");
3924 close(fd);
3925 return false;
3926 }
3927
3928 rc = read(fd, buf, fe->tag_length);
3929 if (rc != fe->tag_length)
3930 {
3931 logf("read error #12");
3932 close(fd);
3933 return false;
3934 }
3935
3936 /* Check if the entry has already been removed */
3937 if (idx->flag & FLAG_DELETED)
3938 continue;
3939
3940 /* This flag must not be used yet. */
3941 if (idx->flag & FLAG_DIRCACHE)
3942 {
3943 logf("internal error!");
3944 close(fd);
3945 return false;
3946 }
3947
3948 if (idx->tag_seek[tag] != pos)
3949 {
3950 logf("corrupt data structures!");
3951 close(fd);
3952 return false;
3953 }
3954
3955 # ifdef HAVE_DIRCACHE
3956 if (dircache_is_enabled())
3957 {
3958 dc = dircache_get_entry_ptr(buf);
3959 if (dc == NULL)
3960 {
3961 logf("Entry no longer valid.");
3962 logf("-> %s", buf);
3963 delete_entry(fe->idx_id);
3964 continue ;
3965 }
3966
3967 idx->flag |= FLAG_DIRCACHE;
3968 FLAG_SET_ATTR(idx->flag, fe->idx_id);
3969 idx->tag_seek[tag_filename] = (long)dc;
3970 }
3971 else
3972 # endif
3973 {
3974 /* This will be very slow unless dircache is enabled
3975 or target is flash based, but do it anyway for
3976 consistency. */
3977 /* Check if entry has been removed. */
3978 if (global_settings.tagcache_autoupdate)
3979 {
3980 if (!file_exists(buf))
3981 {
3982 logf("Entry no longer valid.");
3983 logf("-> %s", buf);
3984 delete_entry(fe->idx_id);
3985 continue;
3986 }
3987 }
3988 }
3989
3990 continue ;
3991 }
3992
3993 bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3994 if (bytesleft < 0)
3995 {
3996 logf("too big tagcache #2");
3997 logf("tl: %d", fe->tag_length);
3998 logf("bl: %ld", bytesleft);
3999 close(fd);
4000 return false;
4001 }
4002
4003 p = fe->tag_data;
4004 rc = read(fd, fe->tag_data, fe->tag_length);
4005 p += rc;
4006
4007 if (rc != fe->tag_length)
4008 {
4009 logf("read error #13");
4010 logf("rc=0x%04x", rc); // 0x431
4011 logf("len=0x%04x", fe->tag_length); // 0x4000
4012 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4013 logf("tag=0x%02x", tag); // 0x00
4014 close(fd);
4015 return false;
4016 }
4017 }
4018 close(fd);
4019 }
4020
4021 tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4022 logf("tagcache loaded into ram!");
4023
4024 return true;
4025 }
4026 #endif /* HAVE_TC_RAMCACHE */
4027
4028 static bool check_deleted_files(void)
4029 {
4030 int fd;
4031 char buf[TAG_MAXLEN+32];
4032 struct tagfile_entry tfe;
4033
4034 logf("reverse scan...");
4035 snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4036 fd = open(buf, O_RDONLY);
4037
4038 if (fd < 0)
4039 {
4040 logf("%s open fail", buf);
4041 return false;
4042 }
4043
4044 lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4045 while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4046 == sizeof(struct tagfile_entry)
4047 #ifndef __PCTOOL__
4048 && !check_event_queue()
4049 #endif
4050 )
4051 {
4052 if (tfe.tag_length >= (long)sizeof(buf)-1)
4053 {
4054 logf("too long tag");
4055 close(fd);
4056 return false;
4057 }
4058
4059 if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4060 {
4061 logf("read error #14");
4062 close(fd);
4063 return false;
4064 }
4065
4066 /* Check if the file has already deleted from the db. */
4067 if (*buf == '\0')
4068 continue;
4069
4070 /* Now check if the file exists. */
4071 if (!file_exists(buf))
4072 {
4073 logf("Entry no longer valid.");
4074 logf("-> %s / %d", buf, tfe.tag_length);
4075 delete_entry(tfe.idx_id);
4076 }
4077 }
4078
4079 close(fd);
4080
4081 logf("done");
4082
4083 return true;
4084 }
4085
4086 static bool check_dir(const char *dirname, int add_files)
4087 {
4088 DIR *dir;
4089 int len;
4090 int success = false;
4091 int ignore, unignore;
4092 char newpath[MAX_PATH];
4093
4094 dir = opendir(dirname);
4095 if (!dir)
4096 {
4097 logf("tagcache: opendir() failed");
4098 return false;
4099 }
4100
4101 /* check for a database.ignore file */
4102 snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4103 ignore = file_exists(newpath);
4104 /* check for a database.unignore file */
4105 snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4106 unignore = file_exists(newpath);
4107
4108 /* don't do anything if both ignore and unignore are there */
4109 if (ignore != unignore)
4110 add_files = unignore;
4111
4112 /* Recursively scan the dir. */
4113 #ifdef __PCTOOL__
4114 while (1)
4115 #else
4116 while (!check_event_queue())
4117 #endif
4118 {
4119 struct dirent *entry;
4120
4121 entry = readdir(dir);
4122
4123 if (entry == NULL)
4124 {
4125 success = true;
4126 break ;
4127 }
4128
4129 if (!strcmp((char *)entry->d_name, ".") ||
4130 !strcmp((char *)entry->d_name, ".."))
4131 continue;
4132
4133 yield();
4134
4135 len = strlen(curpath);
4136 snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4137 entry->d_name);
4138
4139 processed_dir_count++;
4140 if (entry->attribute & ATTR_DIRECTORY)
4141 check_dir(curpath, add_files);
4142 else if (add_files)
4143 {
4144 tc_stat.curentry = curpath;
4145
4146 /* Add a new entry to the temporary db file. */
4147 add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4148 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4149 , dir->internal_entry
4150 #endif
4151 );
4152
4153 /* Wait until current path for debug screen is read and unset. */
4154 while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4155 yield();
4156
4157 tc_stat.curentry = NULL;
4158 }
4159
4160 curpath[len] = '\0';
4161 }
4162
4163 closedir(dir);
4164
4165 return success;
4166 }
4167
4168 void tagcache_screensync_event(void)
4169 {
4170 tc_stat.curentry = NULL;
4171 }
4172
4173 void tagcache_screensync_enable(bool state)
4174 {
4175 tc_stat.syncscreen = state;
4176 }
4177
4178 void tagcache_build(const char *path)
4179 {
4180 struct tagcache_header header;
4181 bool ret;
4182
4183 curpath[0] = '\0';
4184 data_size = 0;
4185 total_entry_count = 0;
4186 processed_dir_count = 0;
4187
4188 #ifdef HAVE_DIRCACHE
4189 while (dircache_is_initializing())
4190 sleep(1);
4191 #endif
4192
4193 logf("updating tagcache");
4194
4195 cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4196 if (cachefd >= 0)
4197 {
4198 logf("skipping, cache already waiting for commit");
4199 close(cachefd);
4200 return ;
4201 }
4202
4203 cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4204 if (cachefd < 0)
4205 {
4206 logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4207 return ;
4208 }
4209
4210 filenametag_fd = open_tag_fd(&header, tag_filename, false);
4211
4212 cpu_boost(true);
4213
4214 logf("Scanning files...");
4215 /* Scan for new files. */
4216 memset(&header, 0, sizeof(struct tagcache_header));
4217 write(cachefd, &header, sizeof(struct tagcache_header));
4218
4219 if (strcmp("/", path) != 0)
4220 strcpy(curpath, path);
4221 ret = check_dir(path, true);
4222
4223 /* Write the header. */
4224 header.magic = TAGCACHE_MAGIC;
4225 header.datasize = data_size;
4226 header.entry_count = total_entry_count;
4227 lseek(cachefd, 0, SEEK_SET);
4228 write(cachefd, &header, sizeof(struct tagcache_header));
4229 close(cachefd);
4230
4231 if (filenametag_fd >= 0)
4232 {
4233 close(filenametag_fd);
4234 filenametag_fd = -1;
4235 }
4236
4237 if (!ret)
4238 {
4239 logf("Aborted.");
4240 cpu_boost(false);
4241 return ;
4242 }
4243
4244 /* Commit changes to the database. */
4245 #ifdef __PCTOOL__
4246 allocate_tempbuf();
4247 #endif
4248 if (commit())
4249 {
4250 remove(TAGCACHE_FILE_TEMP);
4251 logf("tagcache built!");
4252 }
4253 #ifdef __PCTOOL__
4254 free_tempbuf();
4255 #endif
4256
4257 #ifdef HAVE_TC_RAMCACHE
4258 if (hdr)
4259 {
4260 /* Import runtime statistics if we just initialized the db. */
4261 if (hdr->h.serial == 0)
4262 queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4263 }
4264 #endif
4265
4266 cpu_boost(false);
4267 }
4268
4269 #ifdef HAVE_TC_RAMCACHE
4270 static void load_ramcache(void)
4271 {
4272 if (!hdr)
4273 return ;
4274
4275 cpu_boost(true);
4276
4277 /* At first we should load the cache (if exists). */
4278 tc_stat.ramcache = load_tagcache();
4279
4280 if (!tc_stat.ramcache)
4281 {
4282 /* If loading failed, it must indicate some problem with the db
4283 * so disable it entirely to prevent further issues. */
4284 tc_stat.ready = false;
4285 hdr = NULL;
4286 }
4287
4288 cpu_boost(false);
4289 }
4290
4291 void tagcache_unload_ramcache(void)
4292 {
4293 tc_stat.ramcache = false;
4294 /* Just to make sure there is no statefile present. */
4295 // remove(TAGCACHE_STATEFILE);
4296 }
4297 #endif
4298
4299 #ifndef __PCTOOL__
4300 static void tagcache_thread(void)
4301 {
4302 struct queue_event ev;
4303 bool check_done = false;
4304
4305 /* If the previous cache build/update was interrupted, commit
4306 * the changes first in foreground. */
4307 cpu_boost(true);
4308 allocate_tempbuf();
4309 commit();
4310 free_tempbuf();
4311
4312 #ifdef HAVE_TC_RAMCACHE
4313 # ifdef HAVE_EEPROM_SETTINGS
4314 if (firmware_settings.initialized && firmware_settings.disk_clean)
4315 check_done = tagcache_dumpload();
4316
4317 remove(TAGCACHE_STATEFILE);
4318 # endif
4319
4320 /* Allocate space for the tagcache if found on disk. */
4321 if (global_settings.tagcache_ram && !tc_stat.ramcache)
4322 allocate_tagcache();
4323 #endif
4324
4325 cpu_boost(false);
4326 tc_stat.initialized = true;
4327
4328 /* Don't delay bootup with the header check but do it on background. */
4329 sleep(HZ);
4330 tc_stat.ready = check_all_headers();
4331 tc_stat.readyvalid = true;
4332
4333 while (1)
4334 {
4335 run_command_queue(false);
4336
4337 queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4338
4339 switch (ev.id)
4340 {
4341 case Q_IMPORT_CHANGELOG:
4342 tagcache_import_changelog();
4343 break;
4344
4345 case Q_REBUILD:
4346 remove_files();
4347 remove(TAGCACHE_FILE_TEMP);
4348 tagcache_build("/");
4349 break;
4350
4351 case Q_UPDATE:
4352 tagcache_build("/");
4353 #ifdef HAVE_TC_RAMCACHE
4354 load_ramcache();
4355 #endif
4356 check_deleted_files();
4357 break ;
4358
4359 case Q_START_SCAN:
4360 check_done = false;
4361 case SYS_TIMEOUT:
4362 if (check_done || !tc_stat.ready)
4363 break ;
4364
4365 #ifdef HAVE_TC_RAMCACHE
4366 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4367 {
4368 load_ramcache();
4369 if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4370 tagcache_build("/");
4371 }
4372 else
4373 #endif
4374 if (global_settings.tagcache_autoupdate)
4375 {
4376 tagcache_build("/");
4377
4378 /* This will be very slow unless dircache is enabled
4379 or target is flash based, but do it anyway for
4380 consistency. */
4381 check_deleted_files();
4382 }
4383
4384 logf("tagcache check done");
4385
4386 check_done = true;
4387 break ;
4388
4389 case Q_STOP_SCAN:
4390 break ;
4391
4392 case SYS_POWEROFF:
4393 break ;
4394
4395 #ifndef SIMULATOR
4396 case SYS_USB_CONNECTED:
4397 logf("USB: TagCache");
4398 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4399 usb_wait_for_disconnect(&tagcache_queue);
4400 break ;
4401 #endif
4402 }
4403 }
4404 }
4405
4406 bool tagcache_prepare_shutdown(void)
4407 {
4408 if (tagcache_get_commit_step() > 0)
4409 return false;
4410
4411 tagcache_stop_scan();
4412 while (read_lock || write_lock)
4413 sleep(1);
4414
4415 return true;
4416 }
4417
4418 void tagcache_shutdown(void)
4419 {
4420 /* Flush the command queue. */
4421 run_command_queue(true);
4422
4423 #ifdef HAVE_EEPROM_SETTINGS
4424 if (tc_stat.ramcache)
4425 tagcache_dumpsave();
4426 #endif
4427 }
4428
4429 static int get_progress(void)
4430 {
4431 int total_count = -1;
4432
4433 #ifdef HAVE_DIRCACHE
4434 if (dircache_is_enabled())
4435 {
4436 total_count = dircache_get_entry_count();
4437 }
4438 else
4439 #endif
4440 #ifdef HAVE_TC_RAMCACHE
4441 {
4442 if (hdr && tc_stat.ramcache)
4443 total_count = hdr->h.tch.entry_count;
4444 }
4445 #endif
4446
4447 if (total_count < 0)
4448 return -1;
4449
4450 return processed_dir_count * 100 / total_count;
4451 }
4452
4453 struct tagcache_stat* tagcache_get_stat(void)
4454 {
4455 tc_stat.progress = get_progress();
4456 tc_stat.processed_entries = processed_dir_count;
4457
4458 return &tc_stat;
4459 }
4460
4461 void tagcache_start_scan(void)
4462 {
4463 queue_post(&tagcache_queue, Q_START_SCAN, 0);
4464 }
4465
4466 bool tagcache_update(void)
4467 {
4468 if (!tc_stat.ready)
4469 return false;
4470
4471 queue_post(&tagcache_queue, Q_UPDATE, 0);
4472 return false;
4473 }
4474
4475 bool tagcache_rebuild()
4476 {
4477 queue_post(&tagcache_queue, Q_REBUILD, 0);
4478 return false;
4479 }
4480
4481 void tagcache_stop_scan(void)
4482 {
4483 queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4484 }
4485
4486 #ifdef HAVE_TC_RAMCACHE
4487 bool tagcache_is_ramcache(void)
4488 {
4489 return tc_stat.ramcache;
4490 }
4491 #endif
4492
4493 #endif /* !__PCTOOL__ */
4494
4495
4496 void tagcache_init(void)
4497 {
4498 memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4499 memset(&current_tcmh, 0, sizeof(struct master_header));
4500 filenametag_fd = -1;
4501 write_lock = read_lock = 0;
4502
4503 #ifndef __PCTOOL__
4504 mutex_init(&command_queue_mutex);
4505 queue_init(&tagcache_queue, true);
4506 create_thread(tagcache_thread, tagcache_stack,
4507 sizeof(tagcache_stack), 0, tagcache_thread_name
4508 IF_PRIO(, PRIORITY_BACKGROUND)
4509 IF_COP(, CPU));
4510 #else
4511 tc_stat.initialized = true;
4512 allocate_tempbuf();
4513 commit();
4514 free_tempbuf();
4515 tc_stat.ready = check_all_headers();
4516 #endif
4517 }
4518
4519 #ifdef __PCTOOL__
4520 void tagcache_reverse_scan(void)
4521 {
4522 logf("Checking for deleted files");
4523 check_deleted_files();
4524 }
4525 #endif
4526
4527 bool tagcache_is_initialized(void)
4528 {
4529 return tc_stat.initialized;
4530 }
4531 bool tagcache_is_usable(void)
4532 {
4533 return tc_stat.initialized && tc_stat.ready;
4534 }
4535 int tagcache_get_commit_step(void)
4536 {
4537 return tc_stat.commit_step;
4538 }
4539 int tagcache_get_max_commit_step(void)
4540 {
4541 return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4542 }
4543
My public Rockbox git repo