| /*************************************************************************** |
| * __________ __ ___. |
| * Open \______ \ ____ ____ | | _\_ |__ _______ ___ |
| * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ / |
| * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < < |
| * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \ |
| * \/ \/ \/ \/ \/ |
| * $Id$ |
| * |
| * Copyright (C) 2005 by Miika Pekkarinen |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY |
| * KIND, either express or implied. |
| * |
| ****************************************************************************/ |
| |
| /* |
| * TagCache API |
| * |
| * ----------x---------x------------------x----- |
| * | | | External |
| * +---------------x-------+ | TagCache | Libraries |
| * | Modification routines | | Core | |
| * +-x---------x-----------+ | | |
| * | (R/W) | | | | |
| * | +------x-------------x-+ +-------------x-----+ | |
| * | | x==x Filters & clauses | | |
| * | | Search routines | +-------------------+ | |
| * | | x============================x DirCache |
| * | +-x--------------------+ | (optional) |
| * | | (R) | |
| * | | +-------------------------------+ +---------+ | |
| * | | | DB Commit (sort,unique,index) | | | | |
| * | | +-x--------------------------x--+ | Control | | |
| * | | | (R/W) | (R) | Thread | | |
| * | | | +----------------------+ | | | | |
| * | | | | TagCache DB Builder | | +---------+ | |
| * | | | +-x-------------x------+ | | |
| * | | | | (R) | (W) | | |
| * | | | | +--x--------x---------+ | |
| * | | | | | Temporary Commit DB | | |
| * | | | | +---------------------+ | |
| * +-x----x-------x--+ | |
| * | TagCache RAM DB x==\(W) +-----------------+ | |
| * +-----------------+ \===x | | |
| * | | | | (R) | Ram DB Loader x============x DirCache |
| * +-x----x---x---x---+ /==x | | (optional) |
| * | Tagcache Disk DB x==/ +-----------------+ | |
| * +------------------+ | |
| * |
| */ |
| |
| /*#define LOGF_ENABLE*/ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <ctype.h> |
| #ifdef APPLICATION |
| #include <unistd.h> /* readlink() */ |
| #include <limits.h> /* PATH_MAX */ |
| #endif |
| #include "config.h" |
| #include "ata_idle_notify.h" |
| #include "thread.h" |
| #include "kernel.h" |
| #include "system.h" |
| #include "logf.h" |
| #include "string-extra.h" |
| #include "usb.h" |
| #include "metadata.h" |
| #include "tagcache.h" |
| #include "core_alloc.h" |
| #include "crc32.h" |
| #include "misc.h" |
| #include "settings.h" |
| #include "dir.h" |
| #include "pathfuncs.h" |
| #include "structec.h" |
| #include "debug.h" |
| #include "dircache.h" |
| |
| #ifndef __PCTOOL__ |
| #include "lang.h" |
| #include "eeprom_settings.h" |
| #endif |
| |
| #ifdef __PCTOOL__ |
| #define yield() do { } while(0) |
| #define sim_sleep(timeout) do { } while(0) |
| #define do_timed_yield() do { } while(0) |
| #endif |
| |
| #ifndef __PCTOOL__ |
| /* Tag Cache thread. */ |
| static struct event_queue tagcache_queue SHAREDBSS_ATTR; |
| static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)]; |
| static const char tagcache_thread_name[] = "tagcache"; |
| #endif |
| |
| /* Previous path when scanning directory tree recursively. */ |
| static char curpath[TAG_MAXLEN+32]; |
| |
| /* Used when removing duplicates. */ |
| static char *tempbuf; /* Allocated when needed. */ |
| static long tempbufidx; /* Current location in buffer. */ |
| static size_t tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */ |
| static long tempbuf_left; /* Buffer space left. */ |
| static long tempbuf_pos; |
| #ifndef __PCTOOL__ |
| static int tempbuf_handle; |
| #endif |
| |
| #define SORTED_TAGS_COUNT 8 |
| #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS) |
| #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS) |
| #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \ |
| (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS)) |
| /* Tags we want to get sorted (loaded to the tempbuf). */ |
| #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ |
| (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ |
| (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title)) |
| |
| /* Uniqued tags (we can use these tags with filters and conditional clauses). */ |
| #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ |
| (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ |
| (1LU << tag_albumartist) | (1LU << tag_grouping)) |
| |
| /* String presentation of the tags defined in tagcache.h. Must be in correct order! */ |
| static const char *tags_str[] = { "artist", "album", "genre", "title", |
| "filename", "composer", "comment", "albumartist", "grouping", "year", |
| "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating", |
| "playtime", "lastplayed", "commitid", "mtime", "lastelapsed", "lastoffset" }; |
| |
| /* Status information of the tagcache. */ |
| static struct tagcache_stat tc_stat; |
| |
| /* Queue commands. */ |
| enum tagcache_queue { |
| Q_STOP_SCAN = 0, |
| Q_START_SCAN, |
| Q_IMPORT_CHANGELOG, |
| Q_UPDATE, |
| Q_REBUILD, |
| |
| /* Internal tagcache command queue. */ |
| CMD_UPDATE_MASTER_HEADER, |
| CMD_UPDATE_NUMERIC, |
| }; |
| |
| struct tagcache_command_entry { |
| int32_t command; |
| int32_t idx_id; |
| int32_t tag; |
| int32_t data; |
| }; |
| |
| #ifndef __PCTOOL__ |
| static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH]; |
| static volatile int command_queue_widx = 0; |
| static volatile int command_queue_ridx = 0; |
| static struct mutex command_queue_mutex SHAREDBSS_ATTR; |
| #endif |
| |
| /* Tag database structures. */ |
| |
| /* Variable-length tag entry in tag files. */ |
| struct tagfile_entry { |
| int32_t tag_length; /* Length of the data in bytes including '\0' */ |
| int32_t idx_id; /* Corresponding entry location in index file of not unique tags */ |
| char tag_data[0]; /* Begin of the tag data */ |
| }; |
| |
| /* Fixed-size tag entry in master db index. */ |
| struct index_entry { |
| int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */ |
| int32_t flag; /* Status flags */ |
| }; |
| |
| /* Header is the same in every file. */ |
| struct tagcache_header { |
| int32_t magic; /* Header version number */ |
| int32_t datasize; /* Data size in bytes */ |
| int32_t entry_count; /* Number of entries in this file */ |
| }; |
| |
| struct master_header { |
| struct tagcache_header tch; |
| int32_t serial; /* Increasing counting number */ |
| int32_t commitid; /* Number of commits so far */ |
| int32_t dirty; |
| }; |
| |
| /* For the endianess correction */ |
| static const char * const tagfile_entry_ec = "ll"; |
| /** |
| Note: This should be (1 + TAG_COUNT) amount of l's. |
| */ |
| static const char * const index_entry_ec = "lllllllllllllllllllllll"; |
| |
| static const char * const tagcache_header_ec = "lll"; |
| static const char * const master_header_ec = "llllll"; |
| |
| static struct master_header current_tcmh; |
| |
| #ifdef HAVE_TC_RAMCACHE |
| |
| #define TC_ALIGN_PTR(p, type, gap_out_p) \ |
| ({ typeof (p) __p = (p); \ |
| typeof (p) __palgn = ALIGN_UP(__p, __alignof__(type)); \ |
| *(gap_out_p) = (char *)__palgn - (char *)__p; \ |
| __palgn; }) |
| |
| #define IF_TCRCDC(...) IF_DIRCACHE(__VA_ARGS__) |
| |
| #ifdef HAVE_DIRCACHE |
| #define tcrc_dcfrefs \ |
| ((struct dircache_fileref *)(tcramcache.hdr->tags[tag_filename] + \ |
| sizeof (struct tagcache_header))) |
| #endif /* HAVE_DIRCACHE */ |
| |
| /* Header is created when loading database to ram. */ |
| struct ramcache_header { |
| char *tags[TAG_COUNT]; /* Tag file content (dcfrefs if tag_filename) */ |
| int entry_count[TAG_COUNT]; /* Number of entries in the indices. */ |
| struct index_entry indices[0]; /* Master index file content */ |
| }; |
| |
| #ifdef HAVE_EEPROM_SETTINGS |
| struct statefile_header { |
| int32_t magic; /* Statefile version number */ |
| struct master_header mh; /* Header from the master index */ |
| struct ramcache_header *hdr; /* Old load address of hdr for relocation */ |
| struct tagcache_stat tc_stat; |
| }; |
| #endif /* HAVE_EEPROM_SETTINGS */ |
| |
| /* In-RAM ramcache structure (not persisted) */ |
| static struct tcramcache |
| { |
| struct ramcache_header *hdr; /* allocated ramcache_header */ |
| int handle; /* buffer handle */ |
| int move_lock; |
| } tcramcache; |
| |
| static inline void tcrc_buffer_lock(void) |
| { |
| tcramcache.move_lock++; |
| } |
| |
| static inline void tcrc_buffer_unlock(void) |
| { |
| tcramcache.move_lock--; |
| } |
| |
| #else /* ndef HAVE_TC_RAMCACHE */ |
| |
| #define IF_TCRCDC(...) |
| |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| /** |
| * Full tag entries stored in a temporary file waiting |
| * for commit to the cache. */ |
| struct temp_file_entry { |
| long tag_offset[TAG_COUNT]; |
| short tag_length[TAG_COUNT]; |
| long flag; |
| |
| long data_length; |
| }; |
| |
| struct tempbuf_id_list { |
| long id; |
| struct tempbuf_id_list *next; |
| }; |
| |
| struct tempbuf_searchidx { |
| long idx_id; |
| char *str; |
| int seek; |
| struct tempbuf_id_list idlist; |
| }; |
| |
| /* Lookup buffer for fixing messed up index while after sorting. */ |
| static long commit_entry_count; |
| static long lookup_buffer_depth; |
| static struct tempbuf_searchidx **lookup; |
| |
| /* Used when building the temporary file. */ |
| static int cachefd = -1, filenametag_fd; |
| static int total_entry_count = 0; |
| static int data_size = 0; |
| static int processed_dir_count; |
| |
| /* Thread safe locking */ |
| static volatile int write_lock; |
| static volatile int read_lock; |
| |
| static bool delete_entry(long idx_id); |
| |
| static void allocate_tempbuf(void) |
| { |
| /* Yeah, malloc would be really nice now :) */ |
| size_t size; |
| tempbuf_size = 0; |
| |
| #ifdef __PCTOOL__ |
| size = 32*1024*1024; |
| tempbuf = malloc(size); |
| if (tempbuf) |
| tempbuf_size = size; |
| #else /* !__PCTOOL__ */ |
| tempbuf_handle = core_alloc_maximum("tc tempbuf", &size, NULL); |
| if (tempbuf_handle > 0) |
| { |
| tempbuf = core_get_data(tempbuf_handle); |
| tempbuf_size = size; |
| } |
| #endif /* __PCTOOL__ */ |
| } |
| |
| static void free_tempbuf(void) |
| { |
| if (tempbuf_size == 0) |
| return ; |
| |
| #ifdef __PCTOOL__ |
| free(tempbuf); |
| #else |
| tempbuf_handle = core_free(tempbuf_handle); |
| #endif |
| tempbuf = NULL; |
| tempbuf_size = 0; |
| } |
| |
| const char* tagcache_tag_to_str(int tag) |
| { |
| return tags_str[tag]; |
| } |
| |
| /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */ |
| static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf) |
| { |
| return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ); |
| } |
| |
| static ssize_t ecread_index_entry(int fd, struct index_entry *buf) |
| { |
| return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ); |
| } |
| |
| static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf) |
| { |
| return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ); |
| } |
| |
| static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) |
| { |
| int fd; |
| char buf[MAX_PATH]; |
| int rc; |
| |
| if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT) |
| return -1; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag); |
| |
| fd = open(buf, write ? O_RDWR : O_RDONLY); |
| if (fd < 0) |
| { |
| logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf); |
| tc_stat.ready = false; |
| return fd; |
| } |
| |
| /* Check the header. */ |
| rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ); |
| if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header)) |
| { |
| logf("header error"); |
| tc_stat.ready = false; |
| close(fd); |
| return -2; |
| } |
| |
| return fd; |
| } |
| |
| static int open_master_fd(struct master_header *hdr, bool write) |
| { |
| int fd; |
| int rc; |
| |
| fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY); |
| if (fd < 0) |
| { |
| logf("master file open failed for R/W"); |
| tc_stat.ready = false; |
| return fd; |
| } |
| |
| tc_stat.econ = false; |
| |
| /* Check the header. */ |
| rc = read(fd, hdr, sizeof(struct master_header)); |
| if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header)) |
| { |
| /* Success. */ |
| return fd; |
| } |
| |
| /* Trying to read again, this time with endianess correction enabled. */ |
| lseek(fd, 0, SEEK_SET); |
| |
| rc = ecread(fd, hdr, 1, master_header_ec, true); |
| if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header)) |
| { |
| logf("header error"); |
| tc_stat.ready = false; |
| close(fd); |
| return -2; |
| } |
| |
| tc_stat.econ = true; |
| |
| return fd; |
| } |
| |
| #ifndef __PCTOOL__ |
| static bool do_timed_yield(void) |
| { |
| /* Sorting can lock up for quite a while, so yield occasionally */ |
| static long wakeup_tick = 0; |
| if (TIME_AFTER(current_tick, wakeup_tick)) |
| { |
| yield(); |
| wakeup_tick = current_tick + (HZ/50); |
| return true; |
| } |
| return false; |
| } |
| #endif /* __PCTOOL__ */ |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| /* find the ramcache entry corresponding to the file indicated by |
| * filename and dc (it's corresponding dircache id). */ |
| static long find_entry_ram(const char *filename) |
| { |
| static long last_pos = 0; |
| struct dircache_fileref dcfref; |
| |
| /* Check if tagcache is loaded into ram. */ |
| if (!tc_stat.ramcache) |
| return -1; |
| |
| if (dircache_search(DCS_CACHED_PATH | DCS_UPDATE_FILEREF, &dcfref, |
| filename) <= 0) |
| { |
| logf("tagcache: file not found."); |
| return -1; |
| } |
| |
| /* Search references */ |
| int end_pos = current_tcmh.tch.entry_count; |
| while (1) |
| { |
| for (int i = last_pos; i < end_pos; i++) |
| { |
| do_timed_yield(); |
| |
| if (!(tcramcache.hdr->indices[i].flag & FLAG_DIRCACHE)) |
| continue; |
| |
| int cmp = dircache_fileref_cmp(&tcrc_dcfrefs[i], &dcfref); |
| if (cmp < 3) |
| continue; |
| |
| last_pos = MAX(0, i - 3); |
| return i; |
| } |
| |
| if (last_pos == 0) |
| { |
| last_pos = MAX(0, end_pos - 3); |
| break; |
| } |
| |
| end_pos = last_pos; |
| last_pos = 0; |
| } |
| |
| return -1; |
| } |
| #endif /* defined (HAVE_TC_RAMCACHE) && defined (HAVE_DIRCACHE) */ |
| |
| static long find_entry_disk(const char *filename_raw, bool localfd) |
| { |
| struct tagcache_header tch; |
| static long last_pos = -1; |
| long pos_history[POS_HISTORY_COUNT]; |
| long pos_history_idx = 0; |
| bool found = false; |
| struct tagfile_entry tfe; |
| int fd; |
| char buf[TAG_MAXLEN+32]; |
| int i; |
| int pos = -1; |
| |
| const char *filename = filename_raw; |
| #ifdef APPLICATION |
| char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */ |
| if (realpath(filename, pathbuf) == pathbuf) |
| filename = pathbuf; |
| #endif /* APPLICATION */ |
| |
| if (!tc_stat.ready) |
| return -2; |
| |
| fd = filenametag_fd; |
| if (fd < 0 || localfd) |
| { |
| last_pos = -1; |
| if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0) |
| return -1; |
| } |
| |
| check_again: |
| |
| if (last_pos > 0) |
| lseek(fd, last_pos, SEEK_SET); |
| else |
| lseek(fd, sizeof(struct tagcache_header), SEEK_SET); |
| |
| while (true) |
| { |
| pos = lseek(fd, 0, SEEK_CUR); |
| for (i = pos_history_idx-1; i >= 0; i--) |
| pos_history[i+1] = pos_history[i]; |
| pos_history[0] = pos; |
| |
| if (ecread_tagfile_entry(fd, &tfe) |
| != sizeof(struct tagfile_entry)) |
| { |
| break ; |
| } |
| |
| if (tfe.tag_length >= (long)sizeof(buf)) |
| { |
| logf("too long tag #1"); |
| close(fd); |
| if (!localfd) |
| filenametag_fd = -1; |
| last_pos = -1; |
| return -2; |
| } |
| |
| if (read(fd, buf, tfe.tag_length) != tfe.tag_length) |
| { |
| logf("read error #2"); |
| close(fd); |
| if (!localfd) |
| filenametag_fd = -1; |
| last_pos = -1; |
| return -3; |
| } |
| |
| if (!strcmp(filename, buf)) |
| { |
| last_pos = pos_history[pos_history_idx]; |
| found = true; |
| break ; |
| } |
| |
| if (pos_history_idx < POS_HISTORY_COUNT - 1) |
| pos_history_idx++; |
| } |
| |
| /* Not found? */ |
| if (!found) |
| { |
| if (last_pos > 0) |
| { |
| last_pos = -1; |
| logf("seek again"); |
| goto check_again; |
| } |
| |
| if (fd != filenametag_fd || localfd) |
| close(fd); |
| return -4; |
| } |
| |
| if (fd != filenametag_fd || localfd) |
| close(fd); |
| |
| return tfe.idx_id; |
| } |
| |
| static int find_index(const char *filename) |
| { |
| long idx_id = -1; |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| idx_id = find_entry_ram(filename); |
| #endif |
| |
| if (idx_id < 0) |
| idx_id = find_entry_disk(filename, true); |
| |
| return idx_id; |
| } |
| |
| bool tagcache_find_index(struct tagcache_search *tcs, const char *filename) |
| { |
| int idx_id; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| idx_id = find_index(filename); |
| if (idx_id < 0) |
| return false; |
| |
| if (!tagcache_search(tcs, tag_filename)) |
| return false; |
| |
| tcs->entry_count = 0; |
| tcs->idx_id = idx_id; |
| |
| return true; |
| } |
| |
| static bool get_index(int masterfd, int idxid, |
| struct index_entry *idx, bool use_ram) |
| { |
| bool localfd = false; |
| |
| if (idxid < 0) |
| { |
| logf("Incorrect idxid: %d", idxid); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tc_stat.ramcache && use_ram) |
| { |
| if (tcramcache.hdr->indices[idxid].flag & FLAG_DELETED) |
| return false; |
| |
| *idx = tcramcache.hdr->indices[idxid]; |
| return true; |
| } |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| if (masterfd < 0) |
| { |
| struct master_header tcmh; |
| |
| localfd = true; |
| masterfd = open_master_fd(&tcmh, false); |
| if (masterfd < 0) |
| return false; |
| } |
| |
| lseek(masterfd, idxid * sizeof(struct index_entry) |
| + sizeof(struct master_header), SEEK_SET); |
| if (ecread_index_entry(masterfd, idx) |
| != sizeof(struct index_entry)) |
| { |
| logf("read error #3"); |
| if (localfd) |
| close(masterfd); |
| |
| return false; |
| } |
| |
| if (localfd) |
| close(masterfd); |
| |
| if (idx->flag & FLAG_DELETED) |
| return false; |
| |
| return true; |
| |
| (void)use_ram; |
| } |
| |
| #ifndef __PCTOOL__ |
| |
| static bool write_index(int masterfd, int idxid, struct index_entry *idx) |
| { |
| /* We need to exclude all memory only flags & tags when writing to disk. */ |
| if (idx->flag & FLAG_DIRCACHE) |
| { |
| logf("memory only flags!"); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| /* Only update numeric data. Writing the whole index to RAM by memcpy |
| * destroys dircache pointers! |
| */ |
| if (tc_stat.ramcache) |
| { |
| struct index_entry *idx_ram = &tcramcache.hdr->indices[idxid]; |
| |
| for (int tag = 0; tag < TAG_COUNT; tag++) |
| { |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| { |
| idx_ram->tag_seek[tag] = idx->tag_seek[tag]; |
| } |
| } |
| |
| /* Don't touch the dircache flag or attributes. */ |
| idx_ram->flag = (idx->flag & 0x0000ffff) |
| | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE)); |
| } |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| lseek(masterfd, idxid * sizeof(struct index_entry) |
| + sizeof(struct master_header), SEEK_SET); |
| if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry)) |
| { |
| logf("write error #3"); |
| logf("idxid: %d", idxid); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| #endif /* !__PCTOOL__ */ |
| |
| static bool open_files(struct tagcache_search *tcs, int tag) |
| { |
| if (tcs->idxfd[tag] < 0) |
| { |
| char fn[MAX_PATH]; |
| |
| snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag); |
| tcs->idxfd[tag] = open(fn, O_RDONLY); |
| } |
| |
| if (tcs->idxfd[tag] < 0) |
| { |
| logf("File not open!"); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static bool retrieve(struct tagcache_search *tcs, IF_DIRCACHE(int idx_id,) |
| struct index_entry *idx, int tag, char *buf, long size) |
| { |
| struct tagfile_entry tfe; |
| long seek; |
| |
| *buf = '\0'; |
| |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| return false; |
| |
| seek = idx->tag_seek[tag]; |
| if (seek < 0) |
| { |
| logf("Retrieve failed"); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| #ifdef HAVE_DIRCACHE |
| if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)) |
| { |
| if (dircache_get_fileref_path(&tcrc_dcfrefs[idx_id], buf, size) >= 0) |
| return true; |
| } |
| else |
| #endif /* HAVE_DIRCACHE */ |
| if (tag != tag_filename) |
| { |
| struct tagfile_entry *ep = |
| (struct tagfile_entry *)&tcramcache.hdr->tags[tag][seek]; |
| strlcpy(buf, ep->tag_data, size); |
| |
| return true; |
| } |
| } |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| if (!open_files(tcs, tag)) |
| return false; |
| |
| lseek(tcs->idxfd[tag], seek, SEEK_SET); |
| if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe) |
| != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #5"); |
| return false; |
| } |
| |
| if (tfe.tag_length >= size) |
| { |
| logf("too small buffer"); |
| return false; |
| } |
| |
| if (read(tcs->idxfd[tag], buf, tfe.tag_length) != |
| tfe.tag_length) |
| { |
| logf("read error #6"); |
| return false; |
| } |
| |
| buf[tfe.tag_length] = '\0'; |
| |
| return true; |
| } |
| |
| #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx) |
| |
| static long find_tag(int tag, int idx_id, const struct index_entry *idx) |
| { |
| #ifndef __PCTOOL__ |
| if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag)) |
| { |
| /* Attempt to find tag data through store-to-load forwarding in |
| command queue */ |
| long result = -1; |
| |
| mutex_lock(&command_queue_mutex); |
| |
| int ridx = command_queue_widx; |
| |
| while (ridx != command_queue_ridx) |
| { |
| if (--ridx < 0) |
| ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1; |
| |
| if (command_queue[ridx].command == CMD_UPDATE_NUMERIC |
| && command_queue[ridx].idx_id == idx_id |
| && command_queue[ridx].tag == tag) |
| { |
| result = command_queue[ridx].data; |
| break; |
| } |
| } |
| |
| mutex_unlock(&command_queue_mutex); |
| |
| if (result >= 0) |
| { |
| logf("find_tag: " |
| "Recovered tag %d value %lX from write queue", |
| tag, result); |
| return result; |
| } |
| } |
| #else |
| (void)idx_id; |
| #endif |
| |
| return idx->tag_seek[tag]; |
| } |
| |
| |
| static long check_virtual_tags(int tag, int idx_id, |
| const struct index_entry *idx) |
| { |
| long data = 0; |
| |
| switch (tag) |
| { |
| case tag_virt_length_sec: |
| data = (find_tag(tag_length, idx_id, idx)/1000) % 60; |
| break; |
| |
| case tag_virt_length_min: |
| data = (find_tag(tag_length, idx_id, idx)/1000) / 60; |
| break; |
| |
| case tag_virt_playtime_sec: |
| data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60; |
| break; |
| |
| case tag_virt_playtime_min: |
| data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60; |
| break; |
| |
| case tag_virt_autoscore: |
| if (find_tag(tag_length, idx_id, idx) == 0 |
| || find_tag(tag_playcount, idx_id, idx) == 0) |
| { |
| data = 0; |
| } |
| else |
| { |
| /* A straight calculus gives: |
| autoscore = 100 * playtime / length / playcout (1) |
| Now, consider the euclidian division of playtime by length: |
| playtime = alpha * length + beta |
| With: |
| 0 <= beta < length |
| Now, (1) becomes: |
| autoscore = 100 * (alpha / playcout + beta / length / playcount) |
| Both terms should be small enough to avoid any overflow |
| */ |
| data = 100 * (find_tag(tag_playtime, idx_id, idx) |
| / find_tag(tag_length, idx_id, idx)) |
| + (100 * (find_tag(tag_playtime, idx_id, idx) |
| % find_tag(tag_length, idx_id, idx))) |
| / find_tag(tag_length, idx_id, idx); |
| data /= find_tag(tag_playcount, idx_id, idx); |
| } |
| break; |
| |
| /* How many commits before the file has been added to the DB. */ |
| case tag_virt_entryage: |
| data = current_tcmh.commitid |
| - find_tag(tag_commitid, idx_id, idx) - 1; |
| break; |
| |
| case tag_virt_basename: |
| tag = tag_filename; /* return filename; caller handles basename */ |
| /* FALLTHRU */ |
| |
| default: |
| data = find_tag(tag, idx_id, idx); |
| } |
| |
| return data; |
| } |
| |
| long tagcache_get_numeric(const struct tagcache_search *tcs, int tag) |
| { |
| struct index_entry idx; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| if (!TAGCACHE_IS_NUMERIC(tag)) |
| return -1; |
| |
| if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) |
| return -2; |
| |
| return check_virtual_tags(tag, tcs->idx_id, &idx); |
| } |
| |
| inline static bool str_ends_with(const char *str1, const char *str2) |
| { |
| int str_len = strlen(str1); |
| int clause_len = strlen(str2); |
| |
| if (clause_len > str_len) |
| return false; |
| |
| return !strcasecmp(&str1[str_len - clause_len], str2); |
| } |
| |
| inline static bool str_oneof(const char *str, const char *list) |
| { |
| const char *sep; |
| int l, len = strlen(str); |
| |
| while (*list) |
| { |
| sep = strchr(list, '|'); |
| l = sep ? (intptr_t)sep - (intptr_t)list : (int)strlen(list); |
| if ((l==len) && !strncasecmp(str, list, len)) |
| return true; |
| list += sep ? l + 1 : l; |
| } |
| |
| return false; |
| } |
| |
| static bool check_against_clause(long numeric, const char *str, |
| const struct tagcache_search_clause *clause) |
| { |
| if (clause->numeric) |
| { |
| switch (clause->type) |
| { |
| case clause_is: |
| return numeric == clause->numeric_data; |
| case clause_is_not: |
| return numeric != clause->numeric_data; |
| case clause_gt: |
| return numeric > clause->numeric_data; |
| case clause_gteq: |
| return numeric >= clause->numeric_data; |
| case clause_lt: |
| return numeric < clause->numeric_data; |
| case clause_lteq: |
| return numeric <= clause->numeric_data; |
| default: |
| logf("Incorrect numeric tag: %d", clause->type); |
| } |
| } |
| else |
| { |
| switch (clause->type) |
| { |
| case clause_is: |
| return !strcasecmp(clause->str, str); |
| case clause_is_not: |
| return strcasecmp(clause->str, str); |
| case clause_gt: |
| return 0>strcasecmp(clause->str, str); |
| case clause_gteq: |
| return 0>=strcasecmp(clause->str, str); |
| case clause_lt: |
| return 0<strcasecmp(clause->str, str); |
| case clause_lteq: |
| return 0<=strcasecmp(clause->str, str); |
| case clause_contains: |
| return (strcasestr(str, clause->str) != NULL); |
| case clause_not_contains: |
| return (strcasestr(str, clause->str) == NULL); |
| case clause_begins_with: |
| return (strcasestr(str, clause->str) == str); |
| case clause_not_begins_with: |
| return (strcasestr(str, clause->str) != str); |
| case clause_ends_with: |
| return str_ends_with(str, clause->str); |
| case clause_not_ends_with: |
| return !str_ends_with(str, clause->str); |
| case clause_oneof: |
| return str_oneof(str, clause->str); |
| |
| default: |
| logf("Incorrect tag: %d", clause->type); |
| } |
| } |
| |
| return false; |
| } |
| |
| static bool check_clauses(struct tagcache_search *tcs, |
| struct index_entry *idx, |
| struct tagcache_search_clause **clauses, int count) |
| { |
| int i; |
| |
| /* Go through all conditional clauses. */ |
| for (i = 0; i < count; i++) |
| { |
| int seek; |
| char buf[256]; |
| char *str = buf; |
| struct tagcache_search_clause *clause = clauses[i]; |
| |
| if (clause->type == clause_logical_or) |
| break; /* all conditions before logical-or satisfied -- |
| stop processing clauses */ |
| |
| seek = check_virtual_tags(clause->tag, tcs->idx_id, idx); |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| struct tagfile_entry *tfe; |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag)) |
| { |
| if (clause->tag == tag_filename |
| || clause->tag == tag_virt_basename) |
| { |
| retrieve(tcs, IF_DIRCACHE(tcs->idx_id,) idx, tag_filename, |
| buf, sizeof buf); |
| } |
| else |
| { |
| tfe = (struct tagfile_entry *) |
| &tcramcache.hdr->tags[clause->tag][seek]; |
| /* str points to movable data, but no locking required here, |
| * as no yield() is following */ |
| str = tfe->tag_data; |
| } |
| } |
| } |
| else |
| #endif /* HAVE_TC_RAMCACHE */ |
| { |
| struct tagfile_entry tfe; |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag)) |
| { |
| int tag = clause->tag; |
| if (tag == tag_virt_basename) |
| tag = tag_filename; |
| |
| int fd = tcs->idxfd[tag]; |
| lseek(fd, seek, SEEK_SET); |
| ecread_tagfile_entry(fd, &tfe); |
| if (tfe.tag_length >= (int)sizeof(buf)) |
| { |
| logf("Too long tag read!"); |
| return false; |
| } |
| |
| read(fd, str, tfe.tag_length); |
| str[tfe.tag_length] = '\0'; |
| |
| /* Check if entry has been deleted. */ |
| if (str[0] == '\0') |
| return false; |
| } |
| } |
| |
| if (clause->tag == tag_virt_basename) |
| { |
| char *basename = strrchr(str, '/'); |
| if (basename) |
| str = basename + 1; |
| } |
| |
| if (!check_against_clause(seek, str, clause)) |
| { |
| /* Clause failed -- try finding a logical-or clause */ |
| while (++i < count) |
| { |
| if (clauses[i]->type == clause_logical_or) |
| break; |
| } |
| |
| if (i < count) /* Found logical-or? */ |
| continue; /* Check clauses after logical-or */ |
| |
| return false; |
| } |
| } |
| |
| return true; |
| } |
| |
| bool tagcache_check_clauses(struct tagcache_search *tcs, |
| struct tagcache_search_clause **clause, int count) |
| { |
| struct index_entry idx; |
| |
| if (count == 0) |
| return true; |
| |
| if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) |
| return false; |
| |
| return check_clauses(tcs, &idx, clause, count); |
| } |
| |
| static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id) |
| { |
| int i; |
| |
| /* If uniq buffer is not defined we must return true for search to work. */ |
| if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type) |
| && !TAGCACHE_IS_NUMERIC(tcs->type))) |
| { |
| return true; |
| } |
| |
| for (i = 0; i < tcs->unique_list_count; i++) |
| { |
| /* Return false if entry is found. */ |
| if (tcs->unique_list[i] == id) |
| return false; |
| } |
| |
| if (tcs->unique_list_count < tcs->unique_list_capacity) |
| { |
| tcs->unique_list[i] = id; |
| tcs->unique_list_count++; |
| } |
| |
| return true; |
| } |
| |
| static bool build_lookup_list(struct tagcache_search *tcs) |
| { |
| struct index_entry entry; |
| int i, j; |
| |
| tcs->seek_list_count = 0; |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| tcrc_buffer_lock(); /* lock because below makes a pointer to movable data */ |
| |
| for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| /* idx points to movable data, don't yield or reload */ |
| struct index_entry *idx = &tcramcache.hdr->indices[i]; |
| if (tcs->seek_list_count == SEEK_LIST_SIZE) |
| break ; |
| |
| /* Skip deleted files. */ |
| if (idx->flag & FLAG_DELETED) |
| continue; |
| |
| /* Go through all filters.. */ |
| for (j = 0; j < tcs->filter_count; j++) |
| { |
| if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) |
| { |
| break ; |
| } |
| } |
| |
| if (j < tcs->filter_count) |
| continue ; |
| |
| /* Check for conditions. */ |
| if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count)) |
| continue; |
| /* Add to the seek list if not already in uniq buffer (doesn't yield)*/ |
| if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type])) |
| continue; |
| |
| /* Lets add it. */ |
| seeklist = &tcs->seeklist[tcs->seek_list_count]; |
| seeklist->seek = idx->tag_seek[tcs->type]; |
| seeklist->flag = idx->flag; |
| seeklist->idx_id = i; |
| tcs->seek_list_count++; |
| } |
| |
| tcrc_buffer_unlock(); |
| |
| tcs->seek_pos = i; |
| |
| return tcs->seek_list_count > 0; |
| } |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| if (tcs->masterfd < 0) |
| { |
| struct master_header tcmh; |
| tcs->masterfd = open_master_fd(&tcmh, false); |
| } |
| |
| lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) + |
| sizeof(struct master_header), SEEK_SET); |
| |
| while (ecread_index_entry(tcs->masterfd, &entry) |
| == sizeof(struct index_entry)) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| |
| if (tcs->seek_list_count == SEEK_LIST_SIZE) |
| break ; |
| |
| i = tcs->seek_pos; |
| tcs->seek_pos++; |
| |
| /* Check if entry has been deleted. */ |
| if (entry.flag & FLAG_DELETED) |
| continue; |
| |
| /* Go through all filters.. */ |
| for (j = 0; j < tcs->filter_count; j++) |
| { |
| if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) |
| break ; |
| } |
| |
| if (j < tcs->filter_count) |
| continue ; |
| |
| /* Check for conditions. */ |
| if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count)) |
| continue; |
| |
| /* Add to the seek list if not already in uniq buffer. */ |
| if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type])) |
| continue; |
| |
| /* Lets add it. */ |
| seeklist = &tcs->seeklist[tcs->seek_list_count]; |
| seeklist->seek = entry.tag_seek[tcs->type]; |
| seeklist->flag = entry.flag; |
| seeklist->idx_id = i; |
| tcs->seek_list_count++; |
| |
| yield(); |
| } |
| |
| return tcs->seek_list_count > 0; |
| } |
| |
| |
| static void remove_files(void) |
| { |
| int i; |
| char buf[MAX_PATH]; |
| |
| tc_stat.ready = false; |
| tc_stat.ramcache = false; |
| tc_stat.econ = false; |
| remove(TAGCACHE_FILE_MASTER); |
| for (i = 0; i < TAG_COUNT; i++) |
| { |
| if (TAGCACHE_IS_NUMERIC(i)) |
| continue; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i); |
| remove(buf); |
| } |
| } |
| |
| |
| static bool check_all_headers(void) |
| { |
| struct master_header myhdr; |
| struct tagcache_header tch; |
| int tag; |
| int fd; |
| |
| if ( (fd = open_master_fd(&myhdr, false)) < 0) |
| return false; |
| |
| close(fd); |
| if (myhdr.dirty) |
| { |
| logf("tagcache is dirty!"); |
| return false; |
| } |
| |
| memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); |
| |
| for (tag = 0; tag < TAG_COUNT; tag++) |
| { |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| continue; |
| |
| if ( (fd = open_tag_fd(&tch, tag, false)) < 0) |
| return false; |
| |
| close(fd); |
| } |
| |
| return true; |
| } |
| |
| bool tagcache_search(struct tagcache_search *tcs, int tag) |
| { |
| struct tagcache_header tag_hdr; |
| struct master_header master_hdr; |
| int i; |
| |
| while (read_lock) |
| sleep(1); |
| |
| memset(tcs, 0, sizeof(struct tagcache_search)); |
| if (tc_stat.commit_step > 0 || !tc_stat.ready) |
| return false; |
| |
| tcs->position = sizeof(struct tagcache_header); |
| tcs->type = tag; |
| tcs->seek_pos = 0; |
| tcs->list_position = 0; |
| tcs->seek_list_count = 0; |
| tcs->filter_count = 0; |
| tcs->masterfd = -1; |
| |
| for (i = 0; i < TAG_COUNT; i++) |
| tcs->idxfd[i] = -1; |
| |
| #ifndef HAVE_TC_RAMCACHE |
| tcs->ramsearch = false; |
| #else |
| tcs->ramsearch = tc_stat.ramcache; |
| if (tcs->ramsearch) |
| { |
| tcs->entry_count = tcramcache.hdr->entry_count[tcs->type]; |
| } |
| else |
| #endif |
| { |
| /* Always open as R/W so we can pass tcs to functions that modify data also |
| * without failing. */ |
| tcs->masterfd = open_master_fd(&master_hdr, true); |
| if (tcs->masterfd < 0) |
| return false; |
| |
| if (!TAGCACHE_IS_NUMERIC(tcs->type)) |
| { |
| tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false); |
| if (tcs->idxfd[tcs->type] < 0) |
| return false; |
| |
| tcs->entry_count = tag_hdr.entry_count; |
| } |
| else |
| { |
| tcs->entry_count = master_hdr.tch.entry_count; |
| } |
| } |
| |
| tcs->valid = true; |
| tcs->initialized = true; |
| write_lock++; |
| |
| return true; |
| } |
| |
| void tagcache_search_set_uniqbuf(struct tagcache_search *tcs, |
| void *buffer, long length) |
| { |
| tcs->unique_list = (unsigned long *)buffer; |
| tcs->unique_list_capacity = length / sizeof(*tcs->unique_list); |
| tcs->unique_list_count = 0; |
| } |
| |
| bool tagcache_search_add_filter(struct tagcache_search *tcs, |
| int tag, int seek) |
| { |
| if (tcs->filter_count == TAGCACHE_MAX_FILTERS) |
| return false; |
| |
| if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag)) |
| return false; |
| |
| tcs->filter_tag[tcs->filter_count] = tag; |
| tcs->filter_seek[tcs->filter_count] = seek; |
| tcs->filter_count++; |
| |
| return true; |
| } |
| |
| bool tagcache_search_add_clause(struct tagcache_search *tcs, |
| struct tagcache_search_clause *clause) |
| { |
| int i; |
| |
| if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES) |
| { |
| logf("Too many clauses"); |
| return false; |
| } |
| |
| if (clause->type != clause_logical_or) |
| { |
| /* Check if there is already a similar filter in present (filters are |
| * much faster than clauses). |
| */ |
| for (i = 0; i < tcs->filter_count; i++) |
| { |
| if (tcs->filter_tag[i] == clause->tag) |
| return true; |
| } |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0) |
| { |
| char buf[MAX_PATH]; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag); |
| tcs->idxfd[clause->tag] = open(buf, O_RDONLY); |
| } |
| } |
| |
| tcs->clause[tcs->clause_count] = clause; |
| tcs->clause_count++; |
| |
| return true; |
| } |
| |
| static bool get_next(struct tagcache_search *tcs) |
| { |
| static char buf[TAG_MAXLEN+32]; |
| struct tagfile_entry entry; |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| long flag = 0; |
| #endif |
| |
| if (!tcs->valid || !tc_stat.ready) |
| return false; |
| |
| if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type) |
| #ifdef HAVE_TC_RAMCACHE |
| && !tcs->ramsearch |
| #endif |
| ) |
| return false; |
| |
| /* Relative fetch. */ |
| if (tcs->filter_count > 0 || tcs->clause_count > 0 |
| || TAGCACHE_IS_NUMERIC(tcs->type) |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| /* We need to retrieve flag status for dircache. */ |
| || (tcs->ramsearch && tcs->type == tag_filename) |
| #endif |
| ) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| |
| /* Check for end of list. */ |
| if (tcs->list_position == tcs->seek_list_count) |
| { |
| tcs->list_position = 0; |
| |
| /* Try to fetch more. */ |
| if (!build_lookup_list(tcs)) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| } |
| |
| seeklist = &tcs->seeklist[tcs->list_position]; |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| flag = seeklist->flag; |
| #endif |
| tcs->position = seeklist->seek; |
| tcs->idx_id = seeklist->idx_id; |
| tcs->list_position++; |
| } |
| else |
| { |
| if (tcs->entry_count == 0) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| |
| tcs->entry_count--; |
| } |
| |
| tcs->result_seek = tcs->position; |
| |
| if (TAGCACHE_IS_NUMERIC(tcs->type)) |
| { |
| snprintf(buf, sizeof(buf), "%ld", tcs->position); |
| tcs->result = buf; |
| tcs->result_len = strlen(buf) + 1; |
| return true; |
| } |
| |
| /* Direct fetch. */ |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| #ifdef HAVE_DIRCACHE |
| if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE)) |
| { |
| ssize_t len = dircache_get_fileref_path(&tcrc_dcfrefs[tcs->idx_id], |
| buf, sizeof (buf)); |
| if (len >= 0) |
| { |
| tcs->result_len = len + 1; |
| tcs->result = buf; |
| tcs->ramresult = false; |
| return true; |
| } |
| /* else do it the hard way */ |
| } |
| #endif /* HAVE_DIRCACHE */ |
| |
| if (tcs->type != tag_filename) |
| { |
| struct tagfile_entry *ep; |
| |
| ep = (struct tagfile_entry *)&tcramcache.hdr->tags[tcs->type][tcs->position]; |
| /* don't return ep->tag_data directly as it may move */ |
| tcs->result_len = strlcpy(buf, ep->tag_data, sizeof(buf)) + 1; |
| tcs->result = buf; |
| tcs->idx_id = ep->idx_id; |
| tcs->ramresult = false; /* was true before we copied to buf too */ |
| |
| /* Increase position for the next run. This may get overwritten. */ |
| tcs->position += sizeof(struct tagfile_entry) + ep->tag_length; |
| |
| return true; |
| } |
| } |
| #endif /* HAVE_TC_RAMCACHE */ |
| |
| if (!open_files(tcs, tcs->type)) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| |
| /* Seek stream to the correct position and continue to direct fetch. */ |
| lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET); |
| |
| if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #5"); |
| tcs->valid = false; |
| return false; |
| } |
| |
| if (entry.tag_length > (long)sizeof(buf)) |
| { |
| tcs->valid = false; |
| logf("too long tag #2"); |
| logf("P:%lX/%lX", tcs->position, entry.tag_length); |
| return false; |
| } |
| |
| if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length) |
| { |
| tcs->valid = false; |
| logf("read error #4"); |
| return false; |
| } |
| |
| /** |
| Update the position for the next read (this may be overridden |
| if filters or clauses are being used). |
| */ |
| tcs->position += sizeof(struct tagfile_entry) + entry.tag_length; |
| tcs->result = buf; |
| tcs->result_len = strlen(tcs->result) + 1; |
| tcs->idx_id = entry.idx_id; |
| tcs->ramresult = false; |
| |
| return true; |
| } |
| |
| bool tagcache_get_next(struct tagcache_search *tcs) |
| { |
| while (get_next(tcs)) |
| { |
| if (tcs->result_len > 1) |
| return true; |
| } |
| |
| return false; |
| } |
| |
| bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, |
| int tag, char *buf, long size) |
| { |
| struct index_entry idx; |
| |
| *buf = '\0'; |
| if (!get_index(tcs->masterfd, idxid, &idx, true)) |
| return false; |
| |
| return retrieve(tcs, IF_DIRCACHE(idxid,) &idx, tag, buf, size); |
| } |
| |
| static bool update_master_header(void) |
| { |
| struct master_header myhdr; |
| int fd; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| if ( (fd = open_master_fd(&myhdr, true)) < 0) |
| return false; |
| |
| myhdr.serial = current_tcmh.serial; |
| myhdr.commitid = current_tcmh.commitid; |
| myhdr.dirty = current_tcmh.dirty; |
| |
| /* Write it back */ |
| lseek(fd, 0, SEEK_SET); |
| ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ); |
| close(fd); |
| |
| return true; |
| } |
| |
| void tagcache_search_finish(struct tagcache_search *tcs) |
| { |
| int i; |
| |
| if (!tcs->initialized) |
| return; |
| |
| if (tcs->masterfd >= 0) |
| { |
| close(tcs->masterfd); |
| tcs->masterfd = -1; |
| } |
| |
| for (i = 0; i < TAG_COUNT; i++) |
| { |
| if (tcs->idxfd[i] >= 0) |
| { |
| close(tcs->idxfd[i]); |
| tcs->idxfd[i] = -1; |
| } |
| } |
| |
| tcs->ramsearch = false; |
| tcs->valid = false; |
| tcs->initialized = 0; |
| if (write_lock > 0) |
| write_lock--; |
| } |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag) |
| { |
| return (struct tagfile_entry *)&tcramcache.hdr->tags[tag][entry->tag_seek[tag]]; |
| } |
| |
| static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id) |
| { |
| return check_virtual_tags(tag, idx_id, entry); |
| } |
| |
| static char* get_tag_string(const struct index_entry *entry, int tag) |
| { |
| char* s = get_tag(entry, tag)->tag_data; |
| return strcmp(s, UNTAGGED) ? s : NULL; |
| } |
| |
| bool tagcache_fill_tags(struct mp3entry *id3, const char *filename) |
| { |
| struct index_entry *entry; |
| int idx_id; |
| |
| if (!tc_stat.ready || !tc_stat.ramcache) |
| return false; |
| |
| /* Find the corresponding entry in tagcache. */ |
| idx_id = find_entry_ram(filename); |
| if (idx_id < 0) |
| return false; |
| |
| entry = &tcramcache.hdr->indices[idx_id]; |
| |
| memset(id3, 0, sizeof(struct mp3entry)); |
| char* buf = id3->id3v2buf; |
| ssize_t remaining = sizeof(id3->id3v2buf); |
| |
| /* this macro sets id3 strings by copying to the id3v2buf */ |
| #define SET(x, y) do \ |
| { \ |
| if (remaining > 0) \ |
| { \ |
| x = NULL; /* initialize with null if tag doesn't exist */ \ |
| char* src = get_tag_string(entry, y); \ |
| if (src) \ |
| { \ |
| x = buf; \ |
| size_t len = strlcpy(buf, src, remaining) +1; \ |
| buf += len; remaining -= len; \ |
| } \ |
| } \ |
| } while(0) |
| |
| |
| SET(id3->title, tag_title); |
| SET(id3->artist, tag_artist); |
| SET(id3->album, tag_album); |
| SET(id3->genre_string, tag_genre); |
| SET(id3->composer, tag_composer); |
| SET(id3->comment, tag_comment); |
| SET(id3->albumartist, tag_albumartist); |
| SET(id3->grouping, tag_grouping); |
| |
| id3->length = get_tag_numeric(entry, tag_length, idx_id); |
| id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id); |
| id3->rating = get_tag_numeric(entry, tag_rating, idx_id); |
| id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id); |
| id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10; |
| id3->year = get_tag_numeric(entry, tag_year, idx_id); |
| |
| id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id); |
| id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id); |
| id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id); |
| if (id3->bitrate == 0) |
| id3->bitrate = 1; |
| |
| #if CONFIG_CODEC == SWCODEC |
| if (global_settings.autoresume_enable) |
| { |
| id3->elapsed = get_tag_numeric(entry, tag_lastelapsed, idx_id); |
| logf("tagcache_fill_tags: Set elapsed for %s to %lX\n", |
| id3->title, id3->elapsed); |
| |
| id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id); |
| logf("tagcache_fill_tags: Set offset for %s to %lX\n", |
| id3->title, id3->offset); |
| } |
| #endif |
| |
| return true; |
| } |
| #endif /* defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) */ |
| |
| static inline void write_item(const char *item) |
| { |
| int len = strlen(item) + 1; |
| |
| data_size += len; |
| write(cachefd, item, len); |
| } |
| |
| static int check_if_empty(char **tag) |
| { |
| int length; |
| |
| if (*tag == NULL || **tag == '\0') |
| { |
| *tag = UNTAGGED; |
| return sizeof(UNTAGGED); /* Tag length */ |
| } |
| |
| length = strlen(*tag); |
| if (length > TAG_MAXLEN) |
| { |
| logf("over length tag: %s", *tag); |
| length = TAG_MAXLEN; |
| (*tag)[length] = '\0'; |
| } |
| |
| return length + 1; |
| } |
| |
| /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good |
| * idea, as it uses lots of stack and is called from a recursive function |
| * (check_dir). |
| */ |
| static void NO_INLINE add_tagcache(char *path, unsigned long mtime) |
| { |
| #define ADD_TAG(entry, tag, data) \ |
| /* Adding tag */ \ |
| entry.tag_offset[tag] = offset; \ |
| entry.tag_length[tag] = check_if_empty(data); \ |
| offset += entry.tag_length[tag] |
| |
| struct mp3entry id3; |
| struct temp_file_entry entry; |
| bool ret; |
| int fd; |
| int idx_id = -1; |
| char tracknumfix[3]; |
| int offset = 0; |
| int path_length = strlen(path); |
| bool has_albumartist; |
| bool has_grouping; |
| |
| #ifdef SIMULATOR |
| /* Crude logging for the sim - to aid in debugging */ |
| int logfd = open(ROCKBOX_DIR "/database.log", |
| O_WRONLY | O_APPEND | O_CREAT, 0666); |
| if (logfd >= 0) |
| { |
| write(logfd, path, strlen(path)); |
| write(logfd, "\n", 1); |
| close(logfd); |
| } |
| #endif /* SIMULATOR */ |
| |
| if (cachefd < 0) |
| return ; |
| |
| /* Check for overlength file path. */ |
| if (path_length > TAG_MAXLEN) |
| { |
| /* Path can't be shortened. */ |
| logf("Too long path: %s", path); |
| return ; |
| } |
| |
| /* Check if the file is supported. */ |
| if (probe_file_format(path) == AFMT_UNKNOWN) |
| return ; |
| |
| /* Check if the file is already cached. */ |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| idx_id = find_entry_ram(path); |
| #endif |
| |
| /* Be sure the entry doesn't exist. */ |
| if (filenametag_fd >= 0 && idx_id < 0) |
| idx_id = find_entry_disk(path, false); |
| |
| /* Check if file has been modified. */ |
| if (idx_id >= 0) |
| { |
| struct index_entry idx; |
| |
| /* TODO: Mark that the index exists (for fast reverse scan) */ |
| //found_idx[idx_id/8] |= idx_id%8; |
| |
| if (!get_index(-1, idx_id, &idx, true)) |
| { |
| logf("failed to retrieve index entry"); |
| return ; |
| } |
| |
| if ((unsigned long)idx.tag_seek[tag_mtime] == mtime) |
| { |
| /* No changes to file. */ |
| return ; |
| } |
| |
| /* Metadata might have been changed. Delete the entry. */ |
| logf("Re-adding: %s", path); |
| if (!delete_entry(idx_id)) |
| { |
| logf("delete_entry failed: %d", idx_id); |
| return ; |
| } |
| } |
| |
| fd = open(path, O_RDONLY); |
| if (fd < 0) |
| { |
| logf("open fail: %s", path); |
| return ; |
| } |
| |
| memset(&id3, 0, sizeof(struct mp3entry)); |
| memset(&entry, 0, sizeof(struct temp_file_entry)); |
| memset(&tracknumfix, 0, sizeof(tracknumfix)); |
| ret = get_metadata(&id3, fd, path); |
| close(fd); |
| |
| if (!ret) |
| return ; |
| |
| logf("-> %s", path); |
| |
| if (id3.tracknum <= 0) /* Track number missing? */ |
| { |
| id3.tracknum = -1; |
| } |
| |
| /* Numeric tags */ |
| entry.tag_offset[tag_year] = id3.year; |
| entry.tag_offset[tag_discnumber] = id3.discnum; |
| entry.tag_offset[tag_tracknumber] = id3.tracknum; |
| entry.tag_offset[tag_length] = id3.length; |
| entry.tag_offset[tag_bitrate] = id3.bitrate; |
| entry.tag_offset[tag_mtime] = mtime; |
| |
| /* String tags. */ |
| has_albumartist = id3.albumartist != NULL |
| && strlen(id3.albumartist) > 0; |
| has_grouping = id3.grouping != NULL |
| && strlen(id3.grouping) > 0; |
| |
| ADD_TAG(entry, tag_filename, &path); |
| ADD_TAG(entry, tag_title, &id3.title); |
| ADD_TAG(entry, tag_artist, &id3.artist); |
| ADD_TAG(entry, tag_album, &id3.album); |
| ADD_TAG(entry, tag_genre, &id3.genre_string); |
| ADD_TAG(entry, tag_composer, &id3.composer); |
| ADD_TAG(entry, tag_comment, &id3.comment); |
| if (has_albumartist) |
| { |
| ADD_TAG(entry, tag_albumartist, &id3.albumartist); |
| } |
| else |
| { |
| ADD_TAG(entry, tag_albumartist, &id3.artist); |
| } |
| if (has_grouping) |
| { |
| ADD_TAG(entry, tag_grouping, &id3.grouping); |
| } |
| else |
| { |
| ADD_TAG(entry, tag_grouping, &id3.title); |
| } |
| entry.data_length = offset; |
| |
| /* Write the header */ |
| write(cachefd, &entry, sizeof(struct temp_file_entry)); |
| |
| /* And tags also... Correct order is critical */ |
| write_item(path); |
| write_item(id3.title); |
| write_item(id3.artist); |
| write_item(id3.album); |
| write_item(id3.genre_string); |
| write_item(id3.composer); |
| write_item(id3.comment); |
| if (has_albumartist) |
| { |
| write_item(id3.albumartist); |
| } |
| else |
| { |
| write_item(id3.artist); |
| } |
| if (has_grouping) |
| { |
| write_item(id3.grouping); |
| } |
| else |
| { |
| write_item(id3.title); |
| } |
| |
| total_entry_count++; |
| |
| #undef ADD_TAG |
| } |
| |
| static bool tempbuf_insert(char *str, int id, int idx_id, bool unique) |
| { |
| struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; |
| int len = strlen(str)+1; |
| int i; |
| unsigned crc32; |
| unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; |
| char buf[TAG_MAXLEN+32]; |
| |
| for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++) |
| buf[i] = tolower(str[i]); |
| buf[i] = '\0'; |
| |
| crc32 = crc_32(buf, i, 0xffffffff); |
| |
| if (unique) |
| { |
| /* Check if the crc does not exist -> entry does not exist for sure. */ |
| for (i = 0; i < tempbufidx; i++) |
| { |
| if (crcbuf[-i] != crc32) |
| continue; |
| |
| if (!strcasecmp(str, index[i].str)) |
| { |
| if (id < 0 || id >= lookup_buffer_depth) |
| { |
| logf("lookup buf overf.: %d", id); |
| return false; |
| } |
| |
| lookup[id] = &index[i]; |
| return true; |
| } |
| } |
| } |
| |
| /* Insert to CRC buffer. */ |
| crcbuf[-tempbufidx] = crc32; |
| tempbuf_left -= 4; |
| |
| /* Insert it to the buffer. */ |
| tempbuf_left -= len; |
| if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1) |
| return false; |
| |
| if (id >= lookup_buffer_depth) |
| { |
| logf("lookup buf overf. #2: %d", id); |
| return false; |
| } |
| |
| if (id >= 0) |
| { |
| lookup[id] = &index[tempbufidx]; |
| index[tempbufidx].idlist.id = id; |
| } |
| else |
| index[tempbufidx].idlist.id = -1; |
| |
| index[tempbufidx].idlist.next = NULL; |
| index[tempbufidx].idx_id = idx_id; |
| index[tempbufidx].seek = -1; |
| index[tempbufidx].str = &tempbuf[tempbuf_pos]; |
| memcpy(index[tempbufidx].str, str, len); |
| tempbuf_pos += len; |
| tempbufidx++; |
| |
| return true; |
| } |
| |
| static int compare(const void *p1, const void *p2) |
| { |
| do_timed_yield(); |
| |
| struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1; |
| struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2; |
| |
| if (strcmp(e1->str, UNTAGGED) == 0) |
| { |
| if (strcmp(e2->str, UNTAGGED) == 0) |
| return 0; |
| return -1; |
| } |
| else if (strcmp(e2->str, UNTAGGED) == 0) |
| return 1; |
| |
| return strncasecmp(e1->str, e2->str, TAG_MAXLEN); |
| } |
| |
| static int tempbuf_sort(int fd) |
| { |
| struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; |
| struct tagfile_entry fe; |
| int i; |
| int length; |
| |
| /* Generate reverse lookup entries. */ |
| for (i = 0; i < lookup_buffer_depth; i++) |
| { |
| struct tempbuf_id_list *idlist; |
| |
| if (!lookup[i]) |
| continue; |
| |
| if (lookup[i]->idlist.id == i) |
| continue; |
| |
| idlist = &lookup[i]->idlist; |
| while (idlist->next != NULL) |
| idlist = idlist->next; |
| |
| tempbuf_left -= sizeof(struct tempbuf_id_list); |
| if (tempbuf_left - 4 < 0) |
| return -1; |
| |
| idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos]; |
| if (tempbuf_pos & 0x03) |
| { |
| tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04; |
| tempbuf_left -= 3; |
| idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos]; |
| } |
| tempbuf_pos += sizeof(struct tempbuf_id_list); |
| |
| idlist = idlist->next; |
| idlist->id = i; |
| idlist->next = NULL; |
| |
| do_timed_yield(); |
| } |
| |
| qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare); |
| memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **)); |
| |
| for (i = 0; i < tempbufidx; i++) |
| { |
| struct tempbuf_id_list *idlist = &index[i].idlist; |
| |
| /* Fix the lookup list. */ |
| while (idlist != NULL) |
| { |
| if (idlist->id >= 0) |
| lookup[idlist->id] = &index[i]; |
| idlist = idlist->next; |
| } |
| |
| index[i].seek = lseek(fd, 0, SEEK_CUR); |
| length = strlen(index[i].str) + 1; |
| fe.tag_length = length; |
| fe.idx_id = index[i].idx_id; |
| |
| /* Check the chunk alignment. */ |
| if ((fe.tag_length + sizeof(struct tagfile_entry)) |
| % TAGFILE_ENTRY_CHUNK_LENGTH) |
| { |
| fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH - |
| ((fe.tag_length + sizeof(struct tagfile_entry)) |
| % TAGFILE_ENTRY_CHUNK_LENGTH); |
| } |
| |
| #ifdef TAGCACHE_STRICT_ALIGN |
| /* Make sure the entry is long aligned. */ |
| if (index[i].seek & 0x03) |
| { |
| logf("tempbuf_sort: alignment error!"); |
| return -3; |
| } |
| #endif |
| |
| if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) != |
| sizeof(struct tagfile_entry)) |
| { |
| logf("tempbuf_sort: write error #1"); |
| return -1; |
| } |
| |
| if (write(fd, index[i].str, length) != length) |
| { |
| logf("tempbuf_sort: write error #2"); |
| return -2; |
| } |
| |
| /* Write some padding. */ |
| if (fe.tag_length - length > 0) |
| write(fd, "XXXXXXXX", fe.tag_length - length); |
| } |
| |
| return i; |
| } |
| |
| inline static struct tempbuf_searchidx* tempbuf_locate(int id) |
| { |
| if (id < 0 || id >= lookup_buffer_depth) |
| return NULL; |
| |
| return lookup[id]; |
| } |
| |
| |
| inline static int tempbuf_find_location(int id) |
| { |
| struct tempbuf_searchidx *entry; |
| |
| entry = tempbuf_locate(id); |
| if (entry == NULL) |
| return -1; |
| |
| return entry->seek; |
| } |
| |
| static bool build_numeric_indices(struct tagcache_header *h, int tmpfd) |
| { |
| struct master_header tcmh; |
| struct index_entry idx; |
| int masterfd; |
| int masterfd_pos; |
| struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf; |
| int max_entries; |
| int entries_processed = 0; |
| int i, j; |
| char buf[TAG_MAXLEN]; |
| |
| max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; |
| |
| logf("Building numeric indices..."); |
| lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); |
| |
| if ( (masterfd = open_master_fd(&tcmh, true)) < 0) |
| return false; |
| |
| masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), |
| SEEK_CUR); |
| if (masterfd_pos == filesize(masterfd)) |
| { |
| logf("we can't append!"); |
| close(masterfd); |
| return false; |
| } |
| |
| while (entries_processed < h->entry_count) |
| { |
| int count = MIN(h->entry_count - entries_processed, max_entries); |
| |
| /* Read in as many entries as possible. */ |
| for (i = 0; i < count; i++) |
| { |
| struct temp_file_entry *tfe = &entrybuf[i]; |
| int datastart; |
| |
| /* Read in numeric data. */ |
| if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) != |
| sizeof(struct temp_file_entry)) |
| { |
| logf("read fail #1"); |
| close(masterfd); |
| return false; |
| } |
| |
| datastart = lseek(tmpfd, 0, SEEK_CUR); |
| |
| /** |
| * Read string data from the following tags: |
| * - tag_filename |
| * - tag_artist |
| * - tag_album |
| * - tag_title |
| * |
| * A crc32 hash is calculated from the read data |
| * and stored back to the data offset field kept in memory. |
| */ |
| #define tmpdb_read_string_tag(tag) \ |
| lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \ |
| if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \ |
| { \ |
| logf("read fail: buffer overflow"); \ |
| close(masterfd); \ |
| return false; \ |
| } \ |
| \ |
| if (read(tmpfd, buf, tfe->tag_length[tag]) != \ |
| tfe->tag_length[tag]) \ |
| { \ |
| logf("read fail #2"); \ |
| close(masterfd); \ |
| return false; \ |
| } \ |
| \ |
| tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \ |
| lseek(tmpfd, datastart, SEEK_SET) |
| |
| tmpdb_read_string_tag(tag_filename); |
| tmpdb_read_string_tag(tag_artist); |
| tmpdb_read_string_tag(tag_album); |
| tmpdb_read_string_tag(tag_title); |
| |
| /* Seek to the end of the string data. */ |
| lseek(tmpfd, tfe->data_length, SEEK_CUR); |
| } |
| |
| /* Backup the master index position. */ |
| masterfd_pos = lseek(masterfd, 0, SEEK_CUR); |
| lseek(masterfd, sizeof(struct master_header), SEEK_SET); |
| |
| /* Check if we can resurrect some deleted runtime statistics data. */ |
| for (i = 0; i < tcmh.tch.entry_count; i++) |
| { |
| /* Read the index entry. */ |
| if (ecread_index_entry(masterfd, &idx) |
| != sizeof(struct index_entry)) |
| { |
| logf("read fail #3"); |
| close(masterfd); |
| return false; |
| } |
| |
| /** |
| * Skip unless the entry is marked as being deleted |
| * or the data has already been resurrected. |
| */ |
| if (!(idx.flag & FLAG_DELETED) || (idx.flag & FLAG_RESURRECTED)) |
| continue; |
| |
| /* Now try to match the entry. */ |
| /** |
| * To succesfully match a song, the following conditions |
| * must apply: |
| * |
| * For numeric fields: tag_length |
| * - Full identical match is required |
| * |
| * If tag_filename matches, no further checking necessary. |
| * |
| * For string hashes: tag_artist, tag_album, tag_title |
| * - All three of these must match |
| */ |
| for (j = 0; j < count; j++) |
| { |
| struct temp_file_entry *tfe = &entrybuf[j]; |
| |
| /* Try to match numeric fields first. */ |
| if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length]) |
| continue; |
| |
| /* Now it's time to do the hash matching. */ |
| if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename]) |
| { |
| int match_count = 0; |
| |
| /* No filename match, check if we can match two other tags. */ |
| #define tmpdb_match(tag) \ |
| if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \ |
| match_count++ |
| |
| tmpdb_match(tag_artist); |
| tmpdb_match(tag_album); |
| tmpdb_match(tag_title); |
| |
| if (match_count < 3) |
| { |
| /* Still no match found, give up. */ |
| continue; |
| } |
| } |
| |
| /* A match found, now copy & resurrect the statistical data. */ |
| #define tmpdb_copy_tag(tag) \ |
| tfe->tag_offset[tag] = idx.tag_seek[tag] |
| |
| tmpdb_copy_tag(tag_playcount); |
| tmpdb_copy_tag(tag_rating); |
| tmpdb_copy_tag(tag_playtime); |
| tmpdb_copy_tag(tag_lastplayed); |
| tmpdb_copy_tag(tag_commitid); |
| tmpdb_copy_tag(tag_lastelapsed); |
| tmpdb_copy_tag(tag_lastoffset); |
| |
| /* Avoid processing this entry again. */ |
| idx.flag |= FLAG_RESURRECTED; |
| |
| lseek(masterfd, -(off_t)sizeof(struct index_entry), SEEK_CUR); |
| if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("masterfd writeback fail #1"); |
| close(masterfd); |
| return false; |
| } |
| |
| logf("Entry resurrected"); |
| } |
| } |
| |
| |
| /* Restore the master index position. */ |
| lseek(masterfd, masterfd_pos, SEEK_SET); |
| |
| /* Commit the data to the index. */ |
| for (i = 0; i < count; i++) |
| { |
| int loc = lseek(masterfd, 0, SEEK_CUR); |
| |
| if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("read fail #3"); |
| close(masterfd); |
| return false; |
| } |
| |
| for (j = 0; j < TAG_COUNT; j++) |
| { |
| if (!TAGCACHE_IS_NUMERIC(j)) |
| continue; |
| |
| idx.tag_seek[j] = entrybuf[i].tag_offset[j]; |
| } |
| idx.flag = entrybuf[i].flag; |
| |
| if (idx.tag_seek[tag_commitid]) |
| { |
| /* Data has been resurrected. */ |
| idx.flag |= FLAG_DIRTYNUM; |
| } |
| else if (tc_stat.ready && current_tcmh.commitid > 0) |
| { |
| idx.tag_seek[tag_commitid] = current_tcmh.commitid; |
| idx.flag |= FLAG_DIRTYNUM; |
| } |
| |
| /* Write back the updated index. */ |
| lseek(masterfd, loc, SEEK_SET); |
| if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("write fail"); |
| close(masterfd); |
| return false; |
| } |
| } |
| |
| entries_processed += count; |
| logf("%d/%ld entries processed", entries_processed, h->entry_count); |
| } |
| |
| close(masterfd); |
| |
| return true; |
| } |
| |
| /** |
| * Return values: |
| * > 0 success |
| * == 0 temporary failure |
| * < 0 fatal error |
| */ |
| static int build_index(int index_type, struct tagcache_header *h, int tmpfd) |
| { |
| int i; |
| struct tagcache_header tch; |
| struct master_header tcmh; |
| struct index_entry idxbuf[IDX_BUF_DEPTH]; |
| int idxbuf_pos; |
| char buf[TAG_MAXLEN+32]; |
| int fd = -1, masterfd; |
| bool error = false; |
| int init; |
| int masterfd_pos; |
| |
| logf("Building index: %d", index_type); |
| |
| /* Check the number of entries we need to allocate ram for. */ |
| commit_entry_count = h->entry_count + 1; |
| |
| masterfd = open_master_fd(&tcmh, false); |
| if (masterfd >= 0) |
| { |
| commit_entry_count += tcmh.tch.entry_count; |
| close(masterfd); |
| } |
| else |
| remove_files(); /* Just to be sure we are clean. */ |
| |
| /* Open the index file, which contains the tag names. */ |
| fd = open_tag_fd(&tch, index_type, true); |
| if (fd >= 0) |
| { |
| logf("tch.datasize=%ld", tch.datasize); |
| lookup_buffer_depth = 1 + |
| /* First part */ commit_entry_count + |
| /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH); |
| } |
| else |
| { |
| lookup_buffer_depth = 1 + |
| /* First part */ commit_entry_count + |
| /* Second part */ 0; |
| } |
| |
| logf("lookup_buffer_depth=%ld", lookup_buffer_depth); |
| logf("commit_entry_count=%ld", commit_entry_count); |
| |
| /* Allocate buffer for all index entries from both old and new |
| * tag files. */ |
| tempbufidx = 0; |
| tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx); |
| |
| /* Allocate lookup buffer. The first portion of commit_entry_count |
| * contains the new tags in the temporary file and the second |
| * part for locating entries already in the db. |
| * |
| * New tags Old tags |
| * +---------+---------------------------+ |
| * | index | position/ENTRY_CHUNK_SIZE | lookup buffer |
| * +---------+---------------------------+ |
| * |
| * Old tags are inserted to a temporary buffer with position: |
| * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...); |
| * And new tags with index: |
| * tempbuf_insert(idx, ...); |
| * |
| * The buffer is sorted and written into tag file: |
| * tempbuf_sort(...); |
| * leaving master index locations messed up. |
| * |
| * That is fixed using the lookup buffer for old tags: |
| * new_seek = tempbuf_find_location(old_seek, ...); |
| * and for new tags: |
| * new_seek = tempbuf_find_location(idx); |
| */ |
| lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos]; |
| tempbuf_pos += lookup_buffer_depth * sizeof(void **); |
| memset(lookup, 0, lookup_buffer_depth * sizeof(void **)); |
| |
| /* And calculate the remaining data space used mainly for storing |
| * tag data (strings). */ |
| tempbuf_left = tempbuf_size - tempbuf_pos - 8; |
| if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0) |
| { |
| logf("Buffer way too small!"); |
| return 0; |
| } |
| |
| if (fd >= 0) |
| { |
| /** |
| * If tag file contains unique tags (sorted index), we will load |
| * it entirely into memory so we can resort it later for use with |
| * chunked browsing. |
| */ |
| if (TAGCACHE_IS_SORTED(index_type)) |
| { |
| logf("loading tags..."); |
| for (i = 0; i < tch.entry_count; i++) |
| { |
| struct tagfile_entry entry; |
| int loc = lseek(fd, 0, SEEK_CUR); |
| bool ret; |
| |
| if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #7"); |
| close(fd); |
| return -2; |
| } |
| |
| if (entry.tag_length >= (int)sizeof(buf)) |
| { |
| logf("too long tag #3"); |
| close(fd); |
| return -2; |
| } |
| |
| if (read(fd, buf, entry.tag_length) != entry.tag_length) |
| { |
| logf("read error #8"); |
| close(fd); |
| return -2; |
| } |
| |
| /* Skip deleted entries. */ |
| if (buf[0] == '\0') |
| continue; |
| |
| /** |
| * Save the tag and tag id in the memory buffer. Tag id |
| * is saved so we can later reindex the master lookup |
| * table when the index gets resorted. |
| */ |
| ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH |
| + commit_entry_count, entry.idx_id, |
| TAGCACHE_IS_UNIQUE(index_type)); |
| if (!ret) |
| { |
| close(fd); |
| return -3; |
| } |
| do_timed_yield(); |
| } |
| logf("done"); |
| } |
| else |
| tempbufidx = tch.entry_count; |
| } |
| else |
| { |
| /** |
| * Creating new index file to store the tags. No need to preload |
| * anything whether the index type is sorted or not. |
| */ |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type); |
| fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666); |
| if (fd < 0) |
| { |
| logf("%s open fail", buf); |
| return -2; |
| } |
| |
| tch.magic = TAGCACHE_MAGIC; |
| tch.entry_count = 0; |
| tch.datasize = 0; |
| |
| if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ) |
| != sizeof(struct tagcache_header)) |
| { |
| logf("header write failed"); |
| close(fd); |
| return -2; |
| } |
| } |
| |
| /* Loading the tag lookup file as "master file". */ |
| logf("Loading index file"); |
| masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR); |
| |
| if (masterfd < 0) |
| { |
| logf("Creating new DB"); |
| masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666); |
| |
| if (masterfd < 0) |
| { |
| logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER); |
| close(fd); |
| return -2; |
| } |
| |
| /* Write the header (write real values later). */ |
| memset(&tcmh, 0, sizeof(struct master_header)); |
| tcmh.tch = *h; |
| tcmh.tch.entry_count = 0; |
| tcmh.tch.datasize = 0; |
| tcmh.dirty = true; |
| ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ); |
| init = true; |
| masterfd_pos = lseek(masterfd, 0, SEEK_CUR); |
| } |
| else |
| { |
| /** |
| * Master file already exists so we need to process the current |
| * file first. |
| */ |
| init = false; |
| |
| if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) != |
| sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC) |
| { |
| logf("header error"); |
| close(fd); |
| close(masterfd); |
| return -2; |
| } |
| |
| /** |
| * If we reach end of the master file, we need to expand it to |
| * hold new tags. If the current index is not sorted, we can |
| * simply append new data to end of the file. |
| * However, if the index is sorted, we need to update all tag |
| * pointers in the master file for the current index. |
| */ |
| masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), |
| SEEK_CUR); |
| if (masterfd_pos == filesize(masterfd)) |
| { |
| logf("appending..."); |
| init = true; |
| } |
| } |
| |
| /** |
| * Load new unique tags in memory to be sorted later and added |
| * to the master lookup file. |
| */ |
| if (TAGCACHE_IS_SORTED(index_type)) |
| { |
| lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); |
| /* h is the header of the temporary file containing new tags. */ |
| logf("inserting new tags..."); |
| for (i = 0; i < h->entry_count; i++) |
| { |
| struct temp_file_entry entry; |
| |
| if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) != |
| sizeof(struct temp_file_entry)) |
| { |
| logf("read fail #3"); |
| error = true; |
| goto error_exit; |
| } |
| |
| /* Read data. */ |
| if (entry.tag_length[index_type] >= (long)sizeof(buf)) |
| { |
| logf("too long entry!"); |
| error = true; |
| goto error_exit; |
| } |
| |
| lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR); |
| if (read(tmpfd, buf, entry.tag_length[index_type]) != |
| entry.tag_length[index_type]) |
| { |
| logf("read fail #4"); |
| error = true; |
| goto error_exit; |
| } |
| |
| if (TAGCACHE_IS_UNIQUE(index_type)) |
| error = !tempbuf_insert(buf, i, -1, true); |
| else |
| error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false); |
| |
|