| /*************************************************************************** |
| * __________ __ ___. |
| * Open \______ \ ____ ____ | | _\_ |__ _______ ___ |
| * Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ / |
| * Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < < |
| * Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \ |
| * \/ \/ \/ \/ \/ |
| * $Id$ |
| * |
| * Copyright (C) 2005 by Miika Pekkarinen |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version 2 |
| * of the License, or (at your option) any later version. |
| * |
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY |
| * KIND, either express or implied. |
| * |
| ****************************************************************************/ |
| |
| /* |
| * TagCache API |
| * |
| * ----------x---------x------------------x----- |
| * | | | External |
| * +---------------x-------+ | TagCache | Libraries |
| * | Modification routines | | Core | |
| * +-x---------x-----------+ | | |
| * | (R/W) | | | | |
| * | +------x-------------x-+ +-------------x-----+ | |
| * | | x==x Filters & clauses | | |
| * | | Search routines | +-------------------+ | |
| * | | x============================x DirCache |
| * | +-x--------------------+ | (optional) |
| * | | (R) | |
| * | | +-------------------------------+ +---------+ | |
| * | | | DB Commit (sort,unique,index) | | | | |
| * | | +-x--------------------------x--+ | Control | | |
| * | | | (R/W) | (R) | Thread | | |
| * | | | +----------------------+ | | | | |
| * | | | | TagCache DB Builder | | +---------+ | |
| * | | | +-x-------------x------+ | | |
| * | | | | (R) | (W) | | |
| * | | | | +--x--------x---------+ | |
| * | | | | | Temporary Commit DB | | |
| * | | | | +---------------------+ | |
| * +-x----x-------x--+ | |
| * | TagCache RAM DB x==\(W) +-----------------+ | |
| * +-----------------+ \===x | | |
| * | | | | (R) | Ram DB Loader x============x DirCache |
| * +-x----x---x---x---+ /==x | | (optional) |
| * | Tagcache Disk DB x==/ +-----------------+ | |
| * +------------------+ | |
| * |
| */ |
| |
| /*#define LOGF_ENABLE*/ |
| |
| #include <stdio.h> |
| #include <stdlib.h> |
| #include <ctype.h> |
| #ifdef APPLICATION |
| #include <unistd.h> /* readlink() */ |
| #include <limits.h> /* PATH_MAX */ |
| #endif |
| #include "config.h" |
| #include "ata_idle_notify.h" |
| #include "thread.h" |
| #include "kernel.h" |
| #include "system.h" |
| #include "logf.h" |
| #include "string-extra.h" |
| #include "usb.h" |
| #include "metadata.h" |
| #include "tagcache.h" |
| #include "buffer.h" |
| #include "crc32.h" |
| #include "misc.h" |
| #include "settings.h" |
| #include "dir.h" |
| #include "filefuncs.h" |
| #include "structec.h" |
| #include "debug.h" |
| |
| #ifndef __PCTOOL__ |
| #include "lang.h" |
| #include "eeprom_settings.h" |
| #endif |
| |
| #ifdef __PCTOOL__ |
| #define yield() do { } while(0) |
| #define sim_sleep(timeout) do { } while(0) |
| #define do_timed_yield() do { } while(0) |
| #endif |
| |
| #ifndef __PCTOOL__ |
| /* Tag Cache thread. */ |
| static struct event_queue tagcache_queue SHAREDBSS_ATTR; |
| static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)]; |
| static const char tagcache_thread_name[] = "tagcache"; |
| #endif |
| |
| /* Previous path when scanning directory tree recursively. */ |
| static char curpath[TAG_MAXLEN+32]; |
| |
| /* Used when removing duplicates. */ |
| static char *tempbuf; /* Allocated when needed. */ |
| static long tempbufidx; /* Current location in buffer. */ |
| static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */ |
| static long tempbuf_left; /* Buffer space left. */ |
| static long tempbuf_pos; |
| |
| #define SORTED_TAGS_COUNT 8 |
| #define TAGCACHE_IS_UNIQUE(tag) (BIT_N(tag) & TAGCACHE_UNIQUE_TAGS) |
| #define TAGCACHE_IS_SORTED(tag) (BIT_N(tag) & TAGCACHE_SORTED_TAGS) |
| #define TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag) \ |
| (BIT_N(tag) & (TAGCACHE_NUMERIC_TAGS | ~TAGCACHE_UNIQUE_TAGS)) |
| /* Tags we want to get sorted (loaded to the tempbuf). */ |
| #define TAGCACHE_SORTED_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ |
| (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ |
| (1LU << tag_albumartist) | (1LU << tag_grouping) | (1LU << tag_title)) |
| |
| /* Uniqued tags (we can use these tags with filters and conditional clauses). */ |
| #define TAGCACHE_UNIQUE_TAGS ((1LU << tag_artist) | (1LU << tag_album) | \ |
| (1LU << tag_genre) | (1LU << tag_composer) | (1LU << tag_comment) | \ |
| (1LU << tag_albumartist) | (1LU << tag_grouping)) |
| |
| /* String presentation of the tags defined in tagcache.h. Must be in correct order! */ |
| static const char *tags_str[] = { "artist", "album", "genre", "title", |
| "filename", "composer", "comment", "albumartist", "grouping", "year", |
| "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating", |
| "playtime", "lastplayed", "commitid", "mtime", "lastoffset" }; |
| |
| /* Status information of the tagcache. */ |
| static struct tagcache_stat tc_stat; |
| |
| /* Queue commands. */ |
| enum tagcache_queue { |
| Q_STOP_SCAN = 0, |
| Q_START_SCAN, |
| Q_IMPORT_CHANGELOG, |
| Q_UPDATE, |
| Q_REBUILD, |
| |
| /* Internal tagcache command queue. */ |
| CMD_UPDATE_MASTER_HEADER, |
| CMD_UPDATE_NUMERIC, |
| }; |
| |
| struct tagcache_command_entry { |
| int32_t command; |
| int32_t idx_id; |
| int32_t tag; |
| int32_t data; |
| }; |
| |
| #ifndef __PCTOOL__ |
| static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH]; |
| static volatile int command_queue_widx = 0; |
| static volatile int command_queue_ridx = 0; |
| static struct mutex command_queue_mutex SHAREDBSS_ATTR; |
| #endif |
| |
| /* Tag database structures. */ |
| |
| /* Variable-length tag entry in tag files. */ |
| struct tagfile_entry { |
| int32_t tag_length; /* Length of the data in bytes including '\0' */ |
| int32_t idx_id; /* Corresponding entry location in index file of not unique tags */ |
| char tag_data[0]; /* Begin of the tag data */ |
| }; |
| |
| /* Fixed-size tag entry in master db index. */ |
| struct index_entry { |
| int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */ |
| int32_t flag; /* Status flags */ |
| }; |
| |
| /* Header is the same in every file. */ |
| struct tagcache_header { |
| int32_t magic; /* Header version number */ |
| int32_t datasize; /* Data size in bytes */ |
| int32_t entry_count; /* Number of entries in this file */ |
| }; |
| |
| struct master_header { |
| struct tagcache_header tch; |
| int32_t serial; /* Increasing counting number */ |
| int32_t commitid; /* Number of commits so far */ |
| int32_t dirty; |
| }; |
| |
| /* For the endianess correction */ |
| static const char *tagfile_entry_ec = "ll"; |
| /** |
| Note: This should be (1 + TAG_COUNT) amount of l's. |
| */ |
| static const char *index_entry_ec = "llllllllllllllllllllll"; |
| |
| static const char *tagcache_header_ec = "lll"; |
| static const char *master_header_ec = "llllll"; |
| |
| static struct master_header current_tcmh; |
| |
| #ifdef HAVE_TC_RAMCACHE |
| /* Header is created when loading database to ram. */ |
| struct ramcache_header { |
| struct index_entry *indices; /* Master index file content */ |
| char *tags[TAG_COUNT]; /* Tag file content (not including filename tag) */ |
| int entry_count[TAG_COUNT]; /* Number of entries in the indices. */ |
| }; |
| |
| # ifdef HAVE_EEPROM_SETTINGS |
| struct statefile_header { |
| struct master_header mh; /* Header from the master index */ |
| struct ramcache_header *hdr; |
| struct tagcache_stat tc_stat; |
| }; |
| # endif |
| |
| /* Pointer to allocated ramcache_header */ |
| static struct ramcache_header *hdr; |
| #endif |
| |
| /** |
| * Full tag entries stored in a temporary file waiting |
| * for commit to the cache. */ |
| struct temp_file_entry { |
| long tag_offset[TAG_COUNT]; |
| short tag_length[TAG_COUNT]; |
| long flag; |
| |
| long data_length; |
| }; |
| |
| struct tempbuf_id_list { |
| long id; |
| struct tempbuf_id_list *next; |
| }; |
| |
| struct tempbuf_searchidx { |
| long idx_id; |
| char *str; |
| int seek; |
| struct tempbuf_id_list idlist; |
| }; |
| |
| /* Lookup buffer for fixing messed up index while after sorting. */ |
| static long commit_entry_count; |
| static long lookup_buffer_depth; |
| static struct tempbuf_searchidx **lookup; |
| |
| /* Used when building the temporary file. */ |
| static int cachefd = -1, filenametag_fd; |
| static int total_entry_count = 0; |
| static int data_size = 0; |
| static int processed_dir_count; |
| |
| /* Thread safe locking */ |
| static volatile int write_lock; |
| static volatile int read_lock; |
| |
| static bool delete_entry(long idx_id); |
| |
| const char* tagcache_tag_to_str(int tag) |
| { |
| return tags_str[tag]; |
| } |
| |
| /* Helper functions for the two most read/write data structure: tagfile_entry and index_entry */ |
| static ssize_t ecread_tagfile_entry(int fd, struct tagfile_entry *buf) |
| { |
| return ecread(fd, buf, 1, tagfile_entry_ec, tc_stat.econ); |
| } |
| |
| static ssize_t ecread_index_entry(int fd, struct index_entry *buf) |
| { |
| return ecread(fd, buf, 1, index_entry_ec, tc_stat.econ); |
| } |
| |
| static ssize_t ecwrite_index_entry(int fd, struct index_entry *buf) |
| { |
| return ecwrite(fd, buf, 1, index_entry_ec, tc_stat.econ); |
| } |
| |
| #ifdef HAVE_DIRCACHE |
| /** |
| * Returns true if specified flag is still present, i.e., dircache |
| * has not been reloaded. |
| */ |
| static bool is_dircache_intact(void) |
| { |
| return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE); |
| } |
| #endif |
| |
| static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write) |
| { |
| int fd; |
| char buf[MAX_PATH]; |
| int rc; |
| |
| if (TAGCACHE_IS_NUMERIC(tag) || tag < 0 || tag >= TAG_COUNT) |
| return -1; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag); |
| |
| fd = open(buf, write ? O_RDWR : O_RDONLY); |
| if (fd < 0) |
| { |
| logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf); |
| tc_stat.ready = false; |
| return fd; |
| } |
| |
| /* Check the header. */ |
| rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ); |
| if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header)) |
| { |
| logf("header error"); |
| tc_stat.ready = false; |
| close(fd); |
| return -2; |
| } |
| |
| return fd; |
| } |
| |
| static int open_master_fd(struct master_header *hdr, bool write) |
| { |
| int fd; |
| int rc; |
| |
| fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY); |
| if (fd < 0) |
| { |
| logf("master file open failed for R/W"); |
| tc_stat.ready = false; |
| return fd; |
| } |
| |
| tc_stat.econ = false; |
| |
| /* Check the header. */ |
| rc = read(fd, hdr, sizeof(struct master_header)); |
| if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header)) |
| { |
| /* Success. */ |
| return fd; |
| } |
| |
| /* Trying to read again, this time with endianess correction enabled. */ |
| lseek(fd, 0, SEEK_SET); |
| |
| rc = ecread(fd, hdr, 1, master_header_ec, true); |
| if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header)) |
| { |
| logf("header error"); |
| tc_stat.ready = false; |
| close(fd); |
| return -2; |
| } |
| |
| tc_stat.econ = true; |
| |
| return fd; |
| } |
| |
| #ifndef __PCTOOL__ |
| static bool do_timed_yield(void) |
| { |
| /* Sorting can lock up for quite a while, so yield occasionally */ |
| static long wakeup_tick = 0; |
| if (TIME_AFTER(current_tick, wakeup_tick)) |
| { |
| wakeup_tick = current_tick + (HZ/4); |
| yield(); |
| return true; |
| } |
| return false; |
| } |
| #endif |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| static long find_entry_ram(const char *filename, |
| const struct dircache_entry *dc) |
| { |
| static long last_pos = 0; |
| int i; |
| |
| /* Check if tagcache is loaded into ram. */ |
| if (!tc_stat.ramcache) |
| return -1; |
| |
| if (dc == NULL) |
| dc = dircache_get_entry_ptr(filename); |
| |
| if (dc == NULL) |
| { |
| logf("tagcache: file not found."); |
| return -1; |
| } |
| |
| try_again: |
| |
| if (last_pos > 0) |
| i = last_pos; |
| else |
| i = 0; |
| |
| for (; i < current_tcmh.tch.entry_count; i++) |
| { |
| if (hdr->indices[i].tag_seek[tag_filename] == (long)dc) |
| { |
| last_pos = MAX(0, i - 3); |
| return i; |
| } |
| |
| do_timed_yield(); |
| } |
| |
| if (last_pos > 0) |
| { |
| last_pos = 0; |
| goto try_again; |
| } |
| |
| return -1; |
| } |
| #endif |
| |
| static long find_entry_disk(const char *filename_raw, bool localfd) |
| { |
| struct tagcache_header tch; |
| static long last_pos = -1; |
| long pos_history[POS_HISTORY_COUNT]; |
| long pos_history_idx = 0; |
| bool found = false; |
| struct tagfile_entry tfe; |
| int fd; |
| char buf[TAG_MAXLEN+32]; |
| int i; |
| int pos = -1; |
| |
| const char *filename = filename_raw; |
| #ifdef APPLICATION |
| char pathbuf[PATH_MAX]; /* Note: Don't use MAX_PATH here, it's too small */ |
| if (realpath(filename, pathbuf) == pathbuf) |
| filename = pathbuf; |
| #endif |
| |
| if (!tc_stat.ready) |
| return -2; |
| |
| fd = filenametag_fd; |
| if (fd < 0 || localfd) |
| { |
| last_pos = -1; |
| if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0) |
| return -1; |
| } |
| |
| check_again: |
| |
| if (last_pos > 0) |
| lseek(fd, last_pos, SEEK_SET); |
| else |
| lseek(fd, sizeof(struct tagcache_header), SEEK_SET); |
| |
| while (true) |
| { |
| pos = lseek(fd, 0, SEEK_CUR); |
| for (i = pos_history_idx-1; i >= 0; i--) |
| pos_history[i+1] = pos_history[i]; |
| pos_history[0] = pos; |
| |
| if (ecread_tagfile_entry(fd, &tfe) |
| != sizeof(struct tagfile_entry)) |
| { |
| break ; |
| } |
| |
| if (tfe.tag_length >= (long)sizeof(buf)) |
| { |
| logf("too long tag #1"); |
| close(fd); |
| if (!localfd) |
| filenametag_fd = -1; |
| last_pos = -1; |
| return -2; |
| } |
| |
| if (read(fd, buf, tfe.tag_length) != tfe.tag_length) |
| { |
| logf("read error #2"); |
| close(fd); |
| if (!localfd) |
| filenametag_fd = -1; |
| last_pos = -1; |
| return -3; |
| } |
| |
| if (!strcasecmp(filename, buf)) |
| { |
| last_pos = pos_history[pos_history_idx]; |
| found = true; |
| break ; |
| } |
| |
| if (pos_history_idx < POS_HISTORY_COUNT - 1) |
| pos_history_idx++; |
| } |
| |
| /* Not found? */ |
| if (!found) |
| { |
| if (last_pos > 0) |
| { |
| last_pos = -1; |
| logf("seek again"); |
| goto check_again; |
| } |
| |
| if (fd != filenametag_fd || localfd) |
| close(fd); |
| return -4; |
| } |
| |
| if (fd != filenametag_fd || localfd) |
| close(fd); |
| |
| return tfe.idx_id; |
| } |
| |
| static int find_index(const char *filename) |
| { |
| long idx_id = -1; |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| if (tc_stat.ramcache && is_dircache_intact()) |
| idx_id = find_entry_ram(filename, NULL); |
| #endif |
| |
| if (idx_id < 0) |
| idx_id = find_entry_disk(filename, true); |
| |
| return idx_id; |
| } |
| |
| bool tagcache_find_index(struct tagcache_search *tcs, const char *filename) |
| { |
| int idx_id; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| idx_id = find_index(filename); |
| if (idx_id < 0) |
| return false; |
| |
| if (!tagcache_search(tcs, tag_filename)) |
| return false; |
| |
| tcs->entry_count = 0; |
| tcs->idx_id = idx_id; |
| |
| return true; |
| } |
| |
| static bool get_index(int masterfd, int idxid, |
| struct index_entry *idx, bool use_ram) |
| { |
| bool localfd = false; |
| |
| if (idxid < 0) |
| { |
| logf("Incorrect idxid: %d", idxid); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tc_stat.ramcache && use_ram) |
| { |
| if (hdr->indices[idxid].flag & FLAG_DELETED) |
| return false; |
| |
| # ifdef HAVE_DIRCACHE |
| if (!(hdr->indices[idxid].flag & FLAG_DIRCACHE) |
| || is_dircache_intact()) |
| #endif |
| { |
| memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry)); |
| return true; |
| } |
| } |
| #else |
| (void)use_ram; |
| #endif |
| |
| if (masterfd < 0) |
| { |
| struct master_header tcmh; |
| |
| localfd = true; |
| masterfd = open_master_fd(&tcmh, false); |
| if (masterfd < 0) |
| return false; |
| } |
| |
| lseek(masterfd, idxid * sizeof(struct index_entry) |
| + sizeof(struct master_header), SEEK_SET); |
| if (ecread_index_entry(masterfd, idx) |
| != sizeof(struct index_entry)) |
| { |
| logf("read error #3"); |
| if (localfd) |
| close(masterfd); |
| |
| return false; |
| } |
| |
| if (localfd) |
| close(masterfd); |
| |
| if (idx->flag & FLAG_DELETED) |
| return false; |
| |
| return true; |
| } |
| |
| #ifndef __PCTOOL__ |
| |
| static bool write_index(int masterfd, int idxid, struct index_entry *idx) |
| { |
| /* We need to exclude all memory only flags & tags when writing to disk. */ |
| if (idx->flag & FLAG_DIRCACHE) |
| { |
| logf("memory only flags!"); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| /* Only update numeric data. Writing the whole index to RAM by memcpy |
| * destroys dircache pointers! |
| */ |
| if (tc_stat.ramcache) |
| { |
| int tag; |
| struct index_entry *idx_ram = &hdr->indices[idxid]; |
| |
| for (tag = 0; tag < TAG_COUNT; tag++) |
| { |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| { |
| idx_ram->tag_seek[tag] = idx->tag_seek[tag]; |
| } |
| } |
| |
| /* Don't touch the dircache flag or attributes. */ |
| idx_ram->flag = (idx->flag & 0x0000ffff) |
| | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE)); |
| } |
| #endif |
| |
| lseek(masterfd, idxid * sizeof(struct index_entry) |
| + sizeof(struct master_header), SEEK_SET); |
| if (ecwrite_index_entry(masterfd, idx) != sizeof(struct index_entry)) |
| { |
| logf("write error #3"); |
| logf("idxid: %d", idxid); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| #endif /* !__PCTOOL__ */ |
| |
| static bool open_files(struct tagcache_search *tcs, int tag) |
| { |
| if (tcs->idxfd[tag] < 0) |
| { |
| char fn[MAX_PATH]; |
| |
| snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag); |
| tcs->idxfd[tag] = open(fn, O_RDONLY); |
| } |
| |
| if (tcs->idxfd[tag] < 0) |
| { |
| logf("File not open!"); |
| return false; |
| } |
| |
| return true; |
| } |
| |
| static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx, |
| int tag, char *buf, long size) |
| { |
| struct tagfile_entry tfe; |
| long seek; |
| |
| *buf = '\0'; |
| |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| return false; |
| |
| seek = idx->tag_seek[tag]; |
| if (seek < 0) |
| { |
| logf("Retrieve failed"); |
| return false; |
| } |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| struct tagfile_entry *ep; |
| |
| # ifdef HAVE_DIRCACHE |
| if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE) |
| && is_dircache_intact()) |
| { |
| dircache_copy_path((struct dircache_entry *)seek, |
| buf, size); |
| return true; |
| } |
| else |
| # endif |
| if (tag != tag_filename) |
| { |
| ep = (struct tagfile_entry *)&hdr->tags[tag][seek]; |
| strlcpy(buf, ep->tag_data, size); |
| |
| return true; |
| } |
| } |
| #endif |
| |
| if (!open_files(tcs, tag)) |
| return false; |
| |
| lseek(tcs->idxfd[tag], seek, SEEK_SET); |
| if (ecread_tagfile_entry(tcs->idxfd[tag], &tfe) |
| != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #5"); |
| return false; |
| } |
| |
| if (tfe.tag_length >= size) |
| { |
| logf("too small buffer"); |
| return false; |
| } |
| |
| if (read(tcs->idxfd[tag], buf, tfe.tag_length) != |
| tfe.tag_length) |
| { |
| logf("read error #6"); |
| return false; |
| } |
| |
| buf[tfe.tag_length] = '\0'; |
| |
| return true; |
| } |
| |
| #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx) |
| |
| static long find_tag(int tag, int idx_id, const struct index_entry *idx) |
| { |
| #ifndef __PCTOOL__ |
| if (! COMMAND_QUEUE_IS_EMPTY && TAGCACHE_IS_NUMERIC(tag)) |
| { |
| /* Attempt to find tag data through store-to-load forwarding in |
| command queue */ |
| long result = -1; |
| |
| mutex_lock(&command_queue_mutex); |
| |
| int ridx = command_queue_widx; |
| |
| while (ridx != command_queue_ridx) |
| { |
| if (--ridx < 0) |
| ridx = TAGCACHE_COMMAND_QUEUE_LENGTH - 1; |
| |
| if (command_queue[ridx].command == CMD_UPDATE_NUMERIC |
| && command_queue[ridx].idx_id == idx_id |
| && command_queue[ridx].tag == tag) |
| { |
| result = command_queue[ridx].data; |
| break; |
| } |
| } |
| |
| mutex_unlock(&command_queue_mutex); |
| |
| if (result >= 0) |
| { |
| logf("find_tag: " |
| "Recovered tag %d value %lX from write queue", |
| tag, result); |
| return result; |
| } |
| } |
| #endif |
| |
| return idx->tag_seek[tag]; |
| } |
| |
| |
| static long check_virtual_tags(int tag, int idx_id, |
| const struct index_entry *idx) |
| { |
| long data = 0; |
| |
| switch (tag) |
| { |
| case tag_virt_length_sec: |
| data = (find_tag(tag_length, idx_id, idx)/1000) % 60; |
| break; |
| |
| case tag_virt_length_min: |
| data = (find_tag(tag_length, idx_id, idx)/1000) / 60; |
| break; |
| |
| case tag_virt_playtime_sec: |
| data = (find_tag(tag_playtime, idx_id, idx)/1000) % 60; |
| break; |
| |
| case tag_virt_playtime_min: |
| data = (find_tag(tag_playtime, idx_id, idx)/1000) / 60; |
| break; |
| |
| case tag_virt_autoscore: |
| if (find_tag(tag_length, idx_id, idx) == 0 |
| || find_tag(tag_playcount, idx_id, idx) == 0) |
| { |
| data = 0; |
| } |
| else |
| { |
| /* A straight calculus gives: |
| autoscore = 100 * playtime / length / playcout (1) |
| Now, consider the euclidian division of playtime by length: |
| playtime = alpha * length + beta |
| With: |
| 0 <= beta < length |
| Now, (1) becomes: |
| autoscore = 100 * (alpha / playcout + beta / length / playcount) |
| Both terms should be small enough to avoid any overflow |
| */ |
| data = 100 * (find_tag(tag_playtime, idx_id, idx) |
| / find_tag(tag_length, idx_id, idx)) |
| + (100 * (find_tag(tag_playtime, idx_id, idx) |
| % find_tag(tag_length, idx_id, idx))) |
| / find_tag(tag_length, idx_id, idx); |
| data /= find_tag(tag_playcount, idx_id, idx); |
| } |
| break; |
| |
| /* How many commits before the file has been added to the DB. */ |
| case tag_virt_entryage: |
| data = current_tcmh.commitid |
| - find_tag(tag_commitid, idx_id, idx) - 1; |
| break; |
| |
| default: |
| data = find_tag(tag, idx_id, idx); |
| } |
| |
| return data; |
| } |
| |
| long tagcache_get_numeric(const struct tagcache_search *tcs, int tag) |
| { |
| struct index_entry idx; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| if (!TAGCACHE_IS_NUMERIC(tag)) |
| return -1; |
| |
| if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) |
| return -2; |
| |
| return check_virtual_tags(tag, tcs->idx_id, &idx); |
| } |
| |
| inline static bool str_ends_with(const char *str1, const char *str2) |
| { |
| int str_len = strlen(str1); |
| int clause_len = strlen(str2); |
| |
| if (clause_len > str_len) |
| return false; |
| |
| return !strcasecmp(&str1[str_len - clause_len], str2); |
| } |
| |
| inline static bool str_oneof(const char *str, const char *list) |
| { |
| const char *sep; |
| int l, len = strlen(str); |
| |
| while (*list) |
| { |
| sep = strchr(list, '|'); |
| l = sep ? (long)sep - (long)list : (int)strlen(list); |
| if ((l==len) && !strncasecmp(str, list, len)) |
| return true; |
| list += sep ? l + 1 : l; |
| } |
| |
| return false; |
| } |
| |
| static bool check_against_clause(long numeric, const char *str, |
| const struct tagcache_search_clause *clause) |
| { |
| if (clause->numeric) |
| { |
| switch (clause->type) |
| { |
| case clause_is: |
| return numeric == clause->numeric_data; |
| case clause_is_not: |
| return numeric != clause->numeric_data; |
| case clause_gt: |
| return numeric > clause->numeric_data; |
| case clause_gteq: |
| return numeric >= clause->numeric_data; |
| case clause_lt: |
| return numeric < clause->numeric_data; |
| case clause_lteq: |
| return numeric <= clause->numeric_data; |
| default: |
| logf("Incorrect numeric tag: %d", clause->type); |
| } |
| } |
| else |
| { |
| switch (clause->type) |
| { |
| case clause_is: |
| return !strcasecmp(clause->str, str); |
| case clause_is_not: |
| return strcasecmp(clause->str, str); |
| case clause_gt: |
| return 0>strcasecmp(clause->str, str); |
| case clause_gteq: |
| return 0>=strcasecmp(clause->str, str); |
| case clause_lt: |
| return 0<strcasecmp(clause->str, str); |
| case clause_lteq: |
| return 0<=strcasecmp(clause->str, str); |
| case clause_contains: |
| return (strcasestr(str, clause->str) != NULL); |
| case clause_not_contains: |
| return (strcasestr(str, clause->str) == NULL); |
| case clause_begins_with: |
| return (strcasestr(str, clause->str) == str); |
| case clause_not_begins_with: |
| return (strcasestr(str, clause->str) != str); |
| case clause_ends_with: |
| return str_ends_with(str, clause->str); |
| case clause_not_ends_with: |
| return !str_ends_with(str, clause->str); |
| case clause_oneof: |
| return str_oneof(str, clause->str); |
| |
| default: |
| logf("Incorrect tag: %d", clause->type); |
| } |
| } |
| |
| return false; |
| } |
| |
| static bool check_clauses(struct tagcache_search *tcs, |
| struct index_entry *idx, |
| struct tagcache_search_clause **clauses, int count) |
| { |
| int i; |
| |
| /* Go through all conditional clauses. */ |
| for (i = 0; i < count; i++) |
| { |
| int seek; |
| char buf[256]; |
| char *str = buf; |
| struct tagcache_search_clause *clause = clauses[i]; |
| |
| if (clause->type == clause_logical_or) |
| break; /* all conditions before logical-or satisfied -- |
| stop processing clauses */ |
| |
| seek = check_virtual_tags(clause->tag, tcs->idx_id, idx); |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| struct tagfile_entry *tfe; |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag)) |
| { |
| if (clause->tag == tag_filename) |
| { |
| retrieve(tcs, idx, tag_filename, buf, sizeof buf); |
| } |
| else |
| { |
| tfe = (struct tagfile_entry *)&hdr->tags[clause->tag][seek]; |
| str = tfe->tag_data; |
| } |
| } |
| } |
| else |
| #endif |
| { |
| struct tagfile_entry tfe; |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag)) |
| { |
| int fd = tcs->idxfd[clause->tag]; |
| lseek(fd, seek, SEEK_SET); |
| ecread_tagfile_entry(fd, &tfe); |
| if (tfe.tag_length >= (int)sizeof(buf)) |
| { |
| logf("Too long tag read!"); |
| return false; |
| } |
| |
| read(fd, str, tfe.tag_length); |
| str[tfe.tag_length] = '\0'; |
| |
| /* Check if entry has been deleted. */ |
| if (str[0] == '\0') |
| return false; |
| } |
| } |
| |
| if (!check_against_clause(seek, str, clause)) |
| { |
| /* Clause failed -- try finding a logical-or clause */ |
| while (++i < count) |
| { |
| if (clauses[i]->type == clause_logical_or) |
| break; |
| } |
| |
| if (i < count) /* Found logical-or? */ |
| continue; /* Check clauses after logical-or */ |
| |
| return false; |
| } |
| } |
| |
| return true; |
| } |
| |
| bool tagcache_check_clauses(struct tagcache_search *tcs, |
| struct tagcache_search_clause **clause, int count) |
| { |
| struct index_entry idx; |
| |
| if (count == 0) |
| return true; |
| |
| if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true)) |
| return false; |
| |
| return check_clauses(tcs, &idx, clause, count); |
| } |
| |
| static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id) |
| { |
| int i; |
| |
| /* If uniq buffer is not defined we must return true for search to work. */ |
| if (tcs->unique_list == NULL || (!TAGCACHE_IS_UNIQUE(tcs->type) |
| && !TAGCACHE_IS_NUMERIC(tcs->type))) |
| { |
| return true; |
| } |
| |
| for (i = 0; i < tcs->unique_list_count; i++) |
| { |
| /* Return false if entry is found. */ |
| if (tcs->unique_list[i] == id) |
| return false; |
| } |
| |
| if (tcs->unique_list_count < tcs->unique_list_capacity) |
| { |
| tcs->unique_list[i] = id; |
| tcs->unique_list_count++; |
| } |
| |
| return true; |
| } |
| |
| static bool build_lookup_list(struct tagcache_search *tcs) |
| { |
| struct index_entry entry; |
| int i, j; |
| |
| tcs->seek_list_count = 0; |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch |
| # ifdef HAVE_DIRCACHE |
| && (tcs->type != tag_filename || is_dircache_intact()) |
| # endif |
| ) |
| { |
| for (i = tcs->seek_pos; i < current_tcmh.tch.entry_count; i++) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| struct index_entry *idx = &hdr->indices[i]; |
| if (tcs->seek_list_count == SEEK_LIST_SIZE) |
| break ; |
| |
| /* Skip deleted files. */ |
| if (idx->flag & FLAG_DELETED) |
| continue; |
| |
| /* Go through all filters.. */ |
| for (j = 0; j < tcs->filter_count; j++) |
| { |
| if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) |
| { |
| break ; |
| } |
| } |
| |
| if (j < tcs->filter_count) |
| continue ; |
| |
| /* Check for conditions. */ |
| if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count)) |
| continue; |
| |
| /* Add to the seek list if not already in uniq buffer. */ |
| if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type])) |
| continue; |
| |
| /* Lets add it. */ |
| seeklist = &tcs->seeklist[tcs->seek_list_count]; |
| seeklist->seek = idx->tag_seek[tcs->type]; |
| seeklist->flag = idx->flag; |
| seeklist->idx_id = i; |
| tcs->seek_list_count++; |
| } |
| |
| tcs->seek_pos = i; |
| |
| return tcs->seek_list_count > 0; |
| } |
| #endif |
| |
| if (tcs->masterfd < 0) |
| { |
| struct master_header tcmh; |
| tcs->masterfd = open_master_fd(&tcmh, false); |
| } |
| |
| lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) + |
| sizeof(struct master_header), SEEK_SET); |
| |
| while (ecread_index_entry(tcs->masterfd, &entry) |
| == sizeof(struct index_entry)) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| |
| if (tcs->seek_list_count == SEEK_LIST_SIZE) |
| break ; |
| |
| i = tcs->seek_pos; |
| tcs->seek_pos++; |
| |
| /* Check if entry has been deleted. */ |
| if (entry.flag & FLAG_DELETED) |
| continue; |
| |
| /* Go through all filters.. */ |
| for (j = 0; j < tcs->filter_count; j++) |
| { |
| if (entry.tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j]) |
| break ; |
| } |
| |
| if (j < tcs->filter_count) |
| continue ; |
| |
| /* Check for conditions. */ |
| if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count)) |
| continue; |
| |
| /* Add to the seek list if not already in uniq buffer. */ |
| if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type])) |
| continue; |
| |
| /* Lets add it. */ |
| seeklist = &tcs->seeklist[tcs->seek_list_count]; |
| seeklist->seek = entry.tag_seek[tcs->type]; |
| seeklist->flag = entry.flag; |
| seeklist->idx_id = i; |
| tcs->seek_list_count++; |
| |
| yield(); |
| } |
| |
| return tcs->seek_list_count > 0; |
| } |
| |
| |
| static void remove_files(void) |
| { |
| int i; |
| char buf[MAX_PATH]; |
| |
| tc_stat.ready = false; |
| tc_stat.ramcache = false; |
| tc_stat.econ = false; |
| remove(TAGCACHE_FILE_MASTER); |
| for (i = 0; i < TAG_COUNT; i++) |
| { |
| if (TAGCACHE_IS_NUMERIC(i)) |
| continue; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i); |
| remove(buf); |
| } |
| } |
| |
| |
| static bool check_all_headers(void) |
| { |
| struct master_header myhdr; |
| struct tagcache_header tch; |
| int tag; |
| int fd; |
| |
| if ( (fd = open_master_fd(&myhdr, false)) < 0) |
| return false; |
| |
| close(fd); |
| if (myhdr.dirty) |
| { |
| logf("tagcache is dirty!"); |
| return false; |
| } |
| |
| memcpy(¤t_tcmh, &myhdr, sizeof(struct master_header)); |
| |
| for (tag = 0; tag < TAG_COUNT; tag++) |
| { |
| if (TAGCACHE_IS_NUMERIC(tag)) |
| continue; |
| |
| if ( (fd = open_tag_fd(&tch, tag, false)) < 0) |
| return false; |
| |
| close(fd); |
| } |
| |
| return true; |
| } |
| |
| bool tagcache_is_busy(void) |
| { |
| return read_lock || write_lock; |
| } |
| |
| bool tagcache_search(struct tagcache_search *tcs, int tag) |
| { |
| struct tagcache_header tag_hdr; |
| struct master_header master_hdr; |
| int i; |
| |
| while (read_lock) |
| sleep(1); |
| |
| memset(tcs, 0, sizeof(struct tagcache_search)); |
| if (tc_stat.commit_step > 0 || !tc_stat.ready) |
| return false; |
| |
| tcs->position = sizeof(struct tagcache_header); |
| tcs->type = tag; |
| tcs->seek_pos = 0; |
| tcs->list_position = 0; |
| tcs->seek_list_count = 0; |
| tcs->filter_count = 0; |
| tcs->masterfd = -1; |
| |
| for (i = 0; i < TAG_COUNT; i++) |
| tcs->idxfd[i] = -1; |
| |
| #ifndef HAVE_TC_RAMCACHE |
| tcs->ramsearch = false; |
| #else |
| tcs->ramsearch = tc_stat.ramcache; |
| if (tcs->ramsearch) |
| { |
| tcs->entry_count = hdr->entry_count[tcs->type]; |
| } |
| else |
| #endif |
| { |
| /* Always open as R/W so we can pass tcs to functions that modify data also |
| * without failing. */ |
| tcs->masterfd = open_master_fd(&master_hdr, true); |
| if (tcs->masterfd < 0) |
| return false; |
| |
| if (!TAGCACHE_IS_NUMERIC(tcs->type)) |
| { |
| tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false); |
| if (tcs->idxfd[tcs->type] < 0) |
| return false; |
| |
| tcs->entry_count = tag_hdr.entry_count; |
| } |
| else |
| { |
| tcs->entry_count = master_hdr.tch.entry_count; |
| } |
| } |
| |
| tcs->valid = true; |
| tcs->initialized = true; |
| write_lock++; |
| |
| return true; |
| } |
| |
| void tagcache_search_set_uniqbuf(struct tagcache_search *tcs, |
| void *buffer, long length) |
| { |
| tcs->unique_list = (unsigned long *)buffer; |
| tcs->unique_list_capacity = length / sizeof(*tcs->unique_list); |
| tcs->unique_list_count = 0; |
| } |
| |
| bool tagcache_search_add_filter(struct tagcache_search *tcs, |
| int tag, int seek) |
| { |
| if (tcs->filter_count == TAGCACHE_MAX_FILTERS) |
| return false; |
| |
| if (TAGCACHE_IS_NUMERIC_OR_NONUNIQUE(tag)) |
| return false; |
| |
| tcs->filter_tag[tcs->filter_count] = tag; |
| tcs->filter_seek[tcs->filter_count] = seek; |
| tcs->filter_count++; |
| |
| return true; |
| } |
| |
| bool tagcache_search_add_clause(struct tagcache_search *tcs, |
| struct tagcache_search_clause *clause) |
| { |
| int i; |
| |
| if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES) |
| { |
| logf("Too many clauses"); |
| return false; |
| } |
| |
| if (clause->type != clause_logical_or) |
| { |
| /* Check if there is already a similar filter in present (filters are |
| * much faster than clauses). |
| */ |
| for (i = 0; i < tcs->filter_count; i++) |
| { |
| if (tcs->filter_tag[i] == clause->tag) |
| return true; |
| } |
| |
| if (!TAGCACHE_IS_NUMERIC(clause->tag) && tcs->idxfd[clause->tag] < 0) |
| { |
| char buf[MAX_PATH]; |
| |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag); |
| tcs->idxfd[clause->tag] = open(buf, O_RDONLY); |
| } |
| } |
| |
| tcs->clause[tcs->clause_count] = clause; |
| tcs->clause_count++; |
| |
| return true; |
| } |
| |
| static bool get_next(struct tagcache_search *tcs) |
| { |
| static char buf[TAG_MAXLEN+32]; |
| struct tagfile_entry entry; |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| long flag = 0; |
| #endif |
| |
| if (!tcs->valid || !tc_stat.ready) |
| return false; |
| |
| if (tcs->idxfd[tcs->type] < 0 && !TAGCACHE_IS_NUMERIC(tcs->type) |
| #ifdef HAVE_TC_RAMCACHE |
| && !tcs->ramsearch |
| #endif |
| ) |
| return false; |
| |
| /* Relative fetch. */ |
| if (tcs->filter_count > 0 || tcs->clause_count > 0 |
| || TAGCACHE_IS_NUMERIC(tcs->type) |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| /* We need to retrieve flag status for dircache. */ |
| || (tcs->ramsearch && tcs->type == tag_filename) |
| #endif |
| ) |
| { |
| struct tagcache_seeklist_entry *seeklist; |
| |
| /* Check for end of list. */ |
| if (tcs->list_position == tcs->seek_list_count) |
| { |
| tcs->list_position = 0; |
| |
| /* Try to fetch more. */ |
| if (!build_lookup_list(tcs)) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| } |
| |
| seeklist = &tcs->seeklist[tcs->list_position]; |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| flag = seeklist->flag; |
| #endif |
| tcs->position = seeklist->seek; |
| tcs->idx_id = seeklist->idx_id; |
| tcs->list_position++; |
| } |
| else |
| { |
| if (tcs->entry_count == 0) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| |
| tcs->entry_count--; |
| } |
| |
| tcs->result_seek = tcs->position; |
| |
| if (TAGCACHE_IS_NUMERIC(tcs->type)) |
| { |
| snprintf(buf, sizeof(buf), "%ld", tcs->position); |
| tcs->result = buf; |
| tcs->result_len = strlen(buf) + 1; |
| return true; |
| } |
| |
| /* Direct fetch. */ |
| #ifdef HAVE_TC_RAMCACHE |
| if (tcs->ramsearch) |
| { |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| if (tcs->type == tag_filename && (flag & FLAG_DIRCACHE) |
| && is_dircache_intact()) |
| { |
| dircache_copy_path((struct dircache_entry *)tcs->position, |
| buf, sizeof buf); |
| tcs->result = buf; |
| tcs->result_len = strlen(buf) + 1; |
| tcs->ramresult = false; |
| |
| return true; |
| } |
| else |
| #endif |
| if (tcs->type != tag_filename) |
| { |
| struct tagfile_entry *ep; |
| |
| ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position]; |
| tcs->result = ep->tag_data; |
| tcs->result_len = strlen(tcs->result) + 1; |
| tcs->idx_id = ep->idx_id; |
| tcs->ramresult = true; |
| |
| /* Increase position for the next run. This may get overwritten. */ |
| tcs->position += sizeof(struct tagfile_entry) + ep->tag_length; |
| |
| return true; |
| } |
| } |
| #endif |
| |
| if (!open_files(tcs, tcs->type)) |
| { |
| tcs->valid = false; |
| return false; |
| } |
| |
| /* Seek stream to the correct position and continue to direct fetch. */ |
| lseek(tcs->idxfd[tcs->type], tcs->position, SEEK_SET); |
| |
| if (ecread_tagfile_entry(tcs->idxfd[tcs->type], &entry) != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #5"); |
| tcs->valid = false; |
| return false; |
| } |
| |
| if (entry.tag_length > (long)sizeof(buf)) |
| { |
| tcs->valid = false; |
| logf("too long tag #2"); |
| logf("P:%lX/%lX", tcs->position, entry.tag_length); |
| return false; |
| } |
| |
| if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length) |
| { |
| tcs->valid = false; |
| logf("read error #4"); |
| return false; |
| } |
| |
| /** |
| Update the position for the next read (this may be overridden |
| if filters or clauses are being used). |
| */ |
| tcs->position += sizeof(struct tagfile_entry) + entry.tag_length; |
| tcs->result = buf; |
| tcs->result_len = strlen(tcs->result) + 1; |
| tcs->idx_id = entry.idx_id; |
| tcs->ramresult = false; |
| |
| return true; |
| } |
| |
| bool tagcache_get_next(struct tagcache_search *tcs) |
| { |
| while (get_next(tcs)) |
| { |
| if (tcs->result_len > 1) |
| return true; |
| } |
| |
| return false; |
| } |
| |
| bool tagcache_retrieve(struct tagcache_search *tcs, int idxid, |
| int tag, char *buf, long size) |
| { |
| struct index_entry idx; |
| |
| *buf = '\0'; |
| if (!get_index(tcs->masterfd, idxid, &idx, true)) |
| return false; |
| |
| return retrieve(tcs, &idx, tag, buf, size); |
| } |
| |
| static bool update_master_header(void) |
| { |
| struct master_header myhdr; |
| int fd; |
| |
| if (!tc_stat.ready) |
| return false; |
| |
| if ( (fd = open_master_fd(&myhdr, true)) < 0) |
| return false; |
| |
| myhdr.serial = current_tcmh.serial; |
| myhdr.commitid = current_tcmh.commitid; |
| myhdr.dirty = current_tcmh.dirty; |
| |
| /* Write it back */ |
| lseek(fd, 0, SEEK_SET); |
| ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ); |
| close(fd); |
| |
| return true; |
| } |
| |
| #if 0 |
| |
| void tagcache_modify(struct tagcache_search *tcs, int type, const char *text) |
| { |
| struct tagentry *entry; |
| |
| if (tcs->type != tag_title) |
| return ; |
| |
| /* We will need reserve buffer for this. */ |
| if (tcs->ramcache) |
| { |
| struct tagfile_entry *ep; |
| |
| ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek]; |
| tcs->seek_list[tcs->seek_list_count]; |
| } |
| |
| entry = find_entry_ram(); |
| |
| } |
| #endif |
| |
| void tagcache_search_finish(struct tagcache_search *tcs) |
| { |
| int i; |
| |
| if (!tcs->initialized) |
| return; |
| |
| if (tcs->masterfd >= 0) |
| { |
| close(tcs->masterfd); |
| tcs->masterfd = -1; |
| } |
| |
| for (i = 0; i < TAG_COUNT; i++) |
| { |
| if (tcs->idxfd[i] >= 0) |
| { |
| close(tcs->idxfd[i]); |
| tcs->idxfd[i] = -1; |
| } |
| } |
| |
| tcs->ramsearch = false; |
| tcs->valid = false; |
| tcs->initialized = 0; |
| if (write_lock > 0) |
| write_lock--; |
| } |
| |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag) |
| { |
| return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]]; |
| } |
| |
| static long get_tag_numeric(const struct index_entry *entry, int tag, int idx_id) |
| { |
| return check_virtual_tags(tag, idx_id, entry); |
| } |
| |
| static char* get_tag_string(const struct index_entry *entry, int tag) |
| { |
| char* s = get_tag(entry, tag)->tag_data; |
| return strcmp(s, UNTAGGED) ? s : NULL; |
| } |
| |
| bool tagcache_fill_tags(struct mp3entry *id3, const char *filename) |
| { |
| struct index_entry *entry; |
| int idx_id; |
| |
| if (!tc_stat.ready || !tc_stat.ramcache) |
| return false; |
| |
| /* Find the corresponding entry in tagcache. */ |
| idx_id = find_entry_ram(filename, NULL); |
| if (idx_id < 0) |
| return false; |
| |
| entry = &hdr->indices[idx_id]; |
| |
| memset(id3, 0, sizeof(struct mp3entry)); |
| |
| id3->title = get_tag_string(entry, tag_title); |
| id3->artist = get_tag_string(entry, tag_artist); |
| id3->album = get_tag_string(entry, tag_album); |
| id3->genre_string = get_tag_string(entry, tag_genre); |
| id3->composer = get_tag_string(entry, tag_composer); |
| id3->comment = get_tag_string(entry, tag_comment); |
| id3->albumartist = get_tag_string(entry, tag_albumartist); |
| id3->grouping = get_tag_string(entry, tag_grouping); |
| |
| id3->length = get_tag_numeric(entry, tag_length, idx_id); |
| id3->playcount = get_tag_numeric(entry, tag_playcount, idx_id); |
| id3->rating = get_tag_numeric(entry, tag_rating, idx_id); |
| id3->lastplayed = get_tag_numeric(entry, tag_lastplayed, idx_id); |
| id3->score = get_tag_numeric(entry, tag_virt_autoscore, idx_id) / 10; |
| id3->year = get_tag_numeric(entry, tag_year, idx_id); |
| |
| id3->discnum = get_tag_numeric(entry, tag_discnumber, idx_id); |
| id3->tracknum = get_tag_numeric(entry, tag_tracknumber, idx_id); |
| id3->bitrate = get_tag_numeric(entry, tag_bitrate, idx_id); |
| if (id3->bitrate == 0) |
| id3->bitrate = 1; |
| |
| #if CONFIG_CODEC == SWCODEC |
| if (global_settings.autoresume_enable) |
| { |
| id3->offset = get_tag_numeric(entry, tag_lastoffset, idx_id); |
| logf("tagcache_fill_tags: Set offset for %s to %lX\n", |
| id3->title, id3->offset); |
| } |
| #endif |
| |
| return true; |
| } |
| #endif |
| |
| static inline void write_item(const char *item) |
| { |
| int len = strlen(item) + 1; |
| |
| data_size += len; |
| write(cachefd, item, len); |
| } |
| |
| static int check_if_empty(char **tag) |
| { |
| int length; |
| |
| if (*tag == NULL || **tag == '\0') |
| { |
| *tag = UNTAGGED; |
| return sizeof(UNTAGGED); /* Tag length */ |
| } |
| |
| length = strlen(*tag); |
| if (length > TAG_MAXLEN) |
| { |
| logf("over length tag: %s", *tag); |
| length = TAG_MAXLEN; |
| (*tag)[length] = '\0'; |
| } |
| |
| return length + 1; |
| } |
| |
| #define ADD_TAG(entry,tag,data) \ |
| /* Adding tag */ \ |
| entry.tag_offset[tag] = offset; \ |
| entry.tag_length[tag] = check_if_empty(data); \ |
| offset += entry.tag_length[tag] |
| /* GCC 3.4.6 for Coldfire can choose to inline this function. Not a good |
| * idea, as it uses lots of stack and is called from a recursive function |
| * (check_dir). |
| */ |
| static void __attribute__ ((noinline)) add_tagcache(char *path, |
| unsigned long mtime |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| ,const struct dircache_entry *dc |
| #endif |
| ) |
| { |
| struct mp3entry id3; |
| struct temp_file_entry entry; |
| bool ret; |
| int fd; |
| int idx_id = -1; |
| char tracknumfix[3]; |
| int offset = 0; |
| int path_length = strlen(path); |
| bool has_albumartist; |
| bool has_grouping; |
| |
| #ifdef SIMULATOR |
| /* Crude logging for the sim - to aid in debugging */ |
| int logfd = open(ROCKBOX_DIR "/database.log", |
| O_WRONLY | O_APPEND | O_CREAT, 0666); |
| if (logfd >= 0) { |
| write(logfd, path, strlen(path)); |
| write(logfd, "\n", 1); |
| close(logfd); |
| } |
| #endif |
| |
| if (cachefd < 0) |
| return ; |
| |
| /* Check for overlength file path. */ |
| if (path_length > TAG_MAXLEN) |
| { |
| /* Path can't be shortened. */ |
| logf("Too long path: %s", path); |
| return ; |
| } |
| |
| /* Check if the file is supported. */ |
| if (probe_file_format(path) == AFMT_UNKNOWN) |
| return ; |
| |
| /* Check if the file is already cached. */ |
| #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE) |
| if (tc_stat.ramcache && is_dircache_intact()) |
| { |
| idx_id = find_entry_ram(path, dc); |
| } |
| #endif |
| |
| /* Be sure the entry doesn't exist. */ |
| if (filenametag_fd >= 0 && idx_id < 0) |
| idx_id = find_entry_disk(path, false); |
| |
| /* Check if file has been modified. */ |
| if (idx_id >= 0) |
| { |
| struct index_entry idx; |
| |
| /* TODO: Mark that the index exists (for fast reverse scan) */ |
| //found_idx[idx_id/8] |= idx_id%8; |
| |
| if (!get_index(-1, idx_id, &idx, true)) |
| { |
| logf("failed to retrieve index entry"); |
| return ; |
| } |
| |
| if ((unsigned long)idx.tag_seek[tag_mtime] == mtime) |
| { |
| /* No changes to file. */ |
| return ; |
| } |
| |
| /* Metadata might have been changed. Delete the entry. */ |
| logf("Re-adding: %s", path); |
| if (!delete_entry(idx_id)) |
| { |
| logf("delete_entry failed: %d", idx_id); |
| return ; |
| } |
| } |
| |
| fd = open(path, O_RDONLY); |
| if (fd < 0) |
| { |
| logf("open fail: %s", path); |
| return ; |
| } |
| |
| memset(&id3, 0, sizeof(struct mp3entry)); |
| memset(&entry, 0, sizeof(struct temp_file_entry)); |
| memset(&tracknumfix, 0, sizeof(tracknumfix)); |
| ret = get_metadata(&id3, fd, path); |
| close(fd); |
| |
| if (!ret) |
| return ; |
| |
| logf("-> %s", path); |
| |
| if (id3.tracknum <= 0) /* Track number missing? */ |
| { |
| id3.tracknum = -1; |
| } |
| |
| /* Numeric tags */ |
| entry.tag_offset[tag_year] = id3.year; |
| entry.tag_offset[tag_discnumber] = id3.discnum; |
| entry.tag_offset[tag_tracknumber] = id3.tracknum; |
| entry.tag_offset[tag_length] = id3.length; |
| entry.tag_offset[tag_bitrate] = id3.bitrate; |
| entry.tag_offset[tag_mtime] = mtime; |
| |
| /* String tags. */ |
| has_albumartist = id3.albumartist != NULL |
| && strlen(id3.albumartist) > 0; |
| has_grouping = id3.grouping != NULL |
| && strlen(id3.grouping) > 0; |
| |
| ADD_TAG(entry, tag_filename, &path); |
| ADD_TAG(entry, tag_title, &id3.title); |
| ADD_TAG(entry, tag_artist, &id3.artist); |
| ADD_TAG(entry, tag_album, &id3.album); |
| ADD_TAG(entry, tag_genre, &id3.genre_string); |
| ADD_TAG(entry, tag_composer, &id3.composer); |
| ADD_TAG(entry, tag_comment, &id3.comment); |
| if (has_albumartist) |
| { |
| ADD_TAG(entry, tag_albumartist, &id3.albumartist); |
| } |
| else |
| { |
| ADD_TAG(entry, tag_albumartist, &id3.artist); |
| } |
| if (has_grouping) |
| { |
| ADD_TAG(entry, tag_grouping, &id3.grouping); |
| } |
| else |
| { |
| ADD_TAG(entry, tag_grouping, &id3.title); |
| } |
| entry.data_length = offset; |
| |
| /* Write the header */ |
| write(cachefd, &entry, sizeof(struct temp_file_entry)); |
| |
| /* And tags also... Correct order is critical */ |
| write_item(path); |
| write_item(id3.title); |
| write_item(id3.artist); |
| write_item(id3.album); |
| write_item(id3.genre_string); |
| write_item(id3.composer); |
| write_item(id3.comment); |
| if (has_albumartist) |
| { |
| write_item(id3.albumartist); |
| } |
| else |
| { |
| write_item(id3.artist); |
| } |
| if (has_grouping) |
| { |
| write_item(id3.grouping); |
| } |
| else |
| { |
| write_item(id3.title); |
| } |
| total_entry_count++; |
| } |
| |
| static bool tempbuf_insert(char *str, int id, int idx_id, bool unique) |
| { |
| struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; |
| int len = strlen(str)+1; |
| int i; |
| unsigned crc32; |
| unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4]; |
| char buf[TAG_MAXLEN+32]; |
| |
| for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++) |
| buf[i] = tolower(str[i]); |
| buf[i] = '\0'; |
| |
| crc32 = crc_32(buf, i, 0xffffffff); |
| |
| if (unique) |
| { |
| /* Check if the crc does not exist -> entry does not exist for sure. */ |
| for (i = 0; i < tempbufidx; i++) |
| { |
| if (crcbuf[-i] != crc32) |
| continue; |
| |
| if (!strcasecmp(str, index[i].str)) |
| { |
| if (id < 0 || id >= lookup_buffer_depth) |
| { |
| logf("lookup buf overf.: %d", id); |
| return false; |
| } |
| |
| lookup[id] = &index[i]; |
| return true; |
| } |
| } |
| } |
| |
| /* Insert to CRC buffer. */ |
| crcbuf[-tempbufidx] = crc32; |
| tempbuf_left -= 4; |
| |
| /* Insert it to the buffer. */ |
| tempbuf_left -= len; |
| if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1) |
| return false; |
| |
| if (id >= lookup_buffer_depth) |
| { |
| logf("lookup buf overf. #2: %d", id); |
| return false; |
| } |
| |
| if (id >= 0) |
| { |
| lookup[id] = &index[tempbufidx]; |
| index[tempbufidx].idlist.id = id; |
| } |
| else |
| index[tempbufidx].idlist.id = -1; |
| |
| index[tempbufidx].idlist.next = NULL; |
| index[tempbufidx].idx_id = idx_id; |
| index[tempbufidx].seek = -1; |
| index[tempbufidx].str = &tempbuf[tempbuf_pos]; |
| memcpy(index[tempbufidx].str, str, len); |
| tempbuf_pos += len; |
| tempbufidx++; |
| |
| return true; |
| } |
| |
| static int compare(const void *p1, const void *p2) |
| { |
| do_timed_yield(); |
| |
| struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1; |
| struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2; |
| |
| if (strcmp(e1->str, UNTAGGED) == 0) |
| { |
| if (strcmp(e2->str, UNTAGGED) == 0) |
| return 0; |
| return -1; |
| } |
| else if (strcmp(e2->str, UNTAGGED) == 0) |
| return 1; |
| |
| return strncasecmp(e1->str, e2->str, TAG_MAXLEN); |
| } |
| |
| static int tempbuf_sort(int fd) |
| { |
| struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf; |
| struct tagfile_entry fe; |
| int i; |
| int length; |
| |
| /* Generate reverse lookup entries. */ |
| for (i = 0; i < lookup_buffer_depth; i++) |
| { |
| struct tempbuf_id_list *idlist; |
| |
| if (!lookup[i]) |
| continue; |
| |
| if (lookup[i]->idlist.id == i) |
| continue; |
| |
| idlist = &lookup[i]->idlist; |
| while (idlist->next != NULL) |
| idlist = idlist->next; |
| |
| tempbuf_left -= sizeof(struct tempbuf_id_list); |
| if (tempbuf_left - 4 < 0) |
| return -1; |
| |
| idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos]; |
| if (tempbuf_pos & 0x03) |
| { |
| tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04; |
| tempbuf_left -= 3; |
| idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos]; |
| } |
| tempbuf_pos += sizeof(struct tempbuf_id_list); |
| |
| idlist = idlist->next; |
| idlist->id = i; |
| idlist->next = NULL; |
| |
| do_timed_yield(); |
| } |
| |
| qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare); |
| memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **)); |
| |
| for (i = 0; i < tempbufidx; i++) |
| { |
| struct tempbuf_id_list *idlist = &index[i].idlist; |
| |
| /* Fix the lookup list. */ |
| while (idlist != NULL) |
| { |
| if (idlist->id >= 0) |
| lookup[idlist->id] = &index[i]; |
| idlist = idlist->next; |
| } |
| |
| index[i].seek = lseek(fd, 0, SEEK_CUR); |
| length = strlen(index[i].str) + 1; |
| fe.tag_length = length; |
| fe.idx_id = index[i].idx_id; |
| |
| /* Check the chunk alignment. */ |
| if ((fe.tag_length + sizeof(struct tagfile_entry)) |
| % TAGFILE_ENTRY_CHUNK_LENGTH) |
| { |
| fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH - |
| ((fe.tag_length + sizeof(struct tagfile_entry)) |
| % TAGFILE_ENTRY_CHUNK_LENGTH); |
| } |
| |
| #ifdef TAGCACHE_STRICT_ALIGN |
| /* Make sure the entry is long aligned. */ |
| if (index[i].seek & 0x03) |
| { |
| logf("tempbuf_sort: alignment error!"); |
| return -3; |
| } |
| #endif |
| |
| if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) != |
| sizeof(struct tagfile_entry)) |
| { |
| logf("tempbuf_sort: write error #1"); |
| return -1; |
| } |
| |
| if (write(fd, index[i].str, length) != length) |
| { |
| logf("tempbuf_sort: write error #2"); |
| return -2; |
| } |
| |
| /* Write some padding. */ |
| if (fe.tag_length - length > 0) |
| write(fd, "XXXXXXXX", fe.tag_length - length); |
| } |
| |
| return i; |
| } |
| |
| inline static struct tempbuf_searchidx* tempbuf_locate(int id) |
| { |
| if (id < 0 || id >= lookup_buffer_depth) |
| return NULL; |
| |
| return lookup[id]; |
| } |
| |
| |
| inline static int tempbuf_find_location(int id) |
| { |
| struct tempbuf_searchidx *entry; |
| |
| entry = tempbuf_locate(id); |
| if (entry == NULL) |
| return -1; |
| |
| return entry->seek; |
| } |
| |
| static bool build_numeric_indices(struct tagcache_header *h, int tmpfd) |
| { |
| struct master_header tcmh; |
| struct index_entry idx; |
| int masterfd; |
| int masterfd_pos; |
| struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf; |
| int max_entries; |
| int entries_processed = 0; |
| int i, j; |
| char buf[TAG_MAXLEN]; |
| |
| max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1; |
| |
| logf("Building numeric indices..."); |
| lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); |
| |
| if ( (masterfd = open_master_fd(&tcmh, true)) < 0) |
| return false; |
| |
| masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), |
| SEEK_CUR); |
| if (masterfd_pos == filesize(masterfd)) |
| { |
| logf("we can't append!"); |
| close(masterfd); |
| return false; |
| } |
| |
| while (entries_processed < h->entry_count) |
| { |
| int count = MIN(h->entry_count - entries_processed, max_entries); |
| |
| /* Read in as many entries as possible. */ |
| for (i = 0; i < count; i++) |
| { |
| struct temp_file_entry *tfe = &entrybuf[i]; |
| int datastart; |
| |
| /* Read in numeric data. */ |
| if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) != |
| sizeof(struct temp_file_entry)) |
| { |
| logf("read fail #1"); |
| close(masterfd); |
| return false; |
| } |
| |
| datastart = lseek(tmpfd, 0, SEEK_CUR); |
| |
| /** |
| * Read string data from the following tags: |
| * - tag_filename |
| * - tag_artist |
| * - tag_album |
| * - tag_title |
| * |
| * A crc32 hash is calculated from the read data |
| * and stored back to the data offset field kept in memory. |
| */ |
| #define tmpdb_read_string_tag(tag) \ |
| lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \ |
| if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \ |
| { \ |
| logf("read fail: buffer overflow"); \ |
| close(masterfd); \ |
| return false; \ |
| } \ |
| \ |
| if (read(tmpfd, buf, tfe->tag_length[tag]) != \ |
| tfe->tag_length[tag]) \ |
| { \ |
| logf("read fail #2"); \ |
| close(masterfd); \ |
| return false; \ |
| } \ |
| \ |
| tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \ |
| lseek(tmpfd, datastart, SEEK_SET) |
| |
| tmpdb_read_string_tag(tag_filename); |
| tmpdb_read_string_tag(tag_artist); |
| tmpdb_read_string_tag(tag_album); |
| tmpdb_read_string_tag(tag_title); |
| |
| /* Seek to the end of the string data. */ |
| lseek(tmpfd, tfe->data_length, SEEK_CUR); |
| } |
| |
| /* Backup the master index position. */ |
| masterfd_pos = lseek(masterfd, 0, SEEK_CUR); |
| lseek(masterfd, sizeof(struct master_header), SEEK_SET); |
| |
| /* Check if we can resurrect some deleted runtime statistics data. */ |
| for (i = 0; i < tcmh.tch.entry_count; i++) |
| { |
| /* Read the index entry. */ |
| if (ecread_index_entry(masterfd, &idx) |
| != sizeof(struct index_entry)) |
| { |
| logf("read fail #3"); |
| close(masterfd); |
| return false; |
| } |
| |
| /** |
| * Skip unless the entry is marked as being deleted |
| * or the data has already been resurrected. |
| */ |
| if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED) |
| continue; |
| |
| /* Now try to match the entry. */ |
| /** |
| * To succesfully match a song, the following conditions |
| * must apply: |
| * |
| * For numeric fields: tag_length |
| * - Full identical match is required |
| * |
| * If tag_filename matches, no further checking necessary. |
| * |
| * For string hashes: tag_artist, tag_album, tag_title |
| * - All three of these must match |
| */ |
| for (j = 0; j < count; j++) |
| { |
| struct temp_file_entry *tfe = &entrybuf[j]; |
| |
| /* Try to match numeric fields first. */ |
| if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length]) |
| continue; |
| |
| /* Now it's time to do the hash matching. */ |
| if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename]) |
| { |
| int match_count = 0; |
| |
| /* No filename match, check if we can match two other tags. */ |
| #define tmpdb_match(tag) \ |
| if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \ |
| match_count++ |
| |
| tmpdb_match(tag_artist); |
| tmpdb_match(tag_album); |
| tmpdb_match(tag_title); |
| |
| if (match_count < 3) |
| { |
| /* Still no match found, give up. */ |
| continue; |
| } |
| } |
| |
| /* A match found, now copy & resurrect the statistical data. */ |
| #define tmpdb_copy_tag(tag) \ |
| tfe->tag_offset[tag] = idx.tag_seek[tag] |
| |
| tmpdb_copy_tag(tag_playcount); |
| tmpdb_copy_tag(tag_rating); |
| tmpdb_copy_tag(tag_playtime); |
| tmpdb_copy_tag(tag_lastplayed); |
| tmpdb_copy_tag(tag_commitid); |
| tmpdb_copy_tag(tag_lastoffset); |
| |
| /* Avoid processing this entry again. */ |
| idx.flag |= FLAG_RESURRECTED; |
| |
| lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR); |
| if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("masterfd writeback fail #1"); |
| close(masterfd); |
| return false; |
| } |
| |
| logf("Entry resurrected"); |
| } |
| } |
| |
| |
| /* Restore the master index position. */ |
| lseek(masterfd, masterfd_pos, SEEK_SET); |
| |
| /* Commit the data to the index. */ |
| for (i = 0; i < count; i++) |
| { |
| int loc = lseek(masterfd, 0, SEEK_CUR); |
| |
| if (ecread_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("read fail #3"); |
| close(masterfd); |
| return false; |
| } |
| |
| for (j = 0; j < TAG_COUNT; j++) |
| { |
| if (!TAGCACHE_IS_NUMERIC(j)) |
| continue; |
| |
| idx.tag_seek[j] = entrybuf[i].tag_offset[j]; |
| } |
| idx.flag = entrybuf[i].flag; |
| |
| if (idx.tag_seek[tag_commitid]) |
| { |
| /* Data has been resurrected. */ |
| idx.flag |= FLAG_DIRTYNUM; |
| } |
| else if (tc_stat.ready && current_tcmh.commitid > 0) |
| { |
| idx.tag_seek[tag_commitid] = current_tcmh.commitid; |
| idx.flag |= FLAG_DIRTYNUM; |
| } |
| |
| /* Write back the updated index. */ |
| lseek(masterfd, loc, SEEK_SET); |
| if (ecwrite_index_entry(masterfd, &idx) != sizeof(struct index_entry)) |
| { |
| logf("write fail"); |
| close(masterfd); |
| return false; |
| } |
| } |
| |
| entries_processed += count; |
| logf("%d/%ld entries processed", entries_processed, h->entry_count); |
| } |
| |
| close(masterfd); |
| |
| return true; |
| } |
| |
| /** |
| * Return values: |
| * > 0 success |
| * == 0 temporary failure |
| * < 0 fatal error |
| */ |
| static int build_index(int index_type, struct tagcache_header *h, int tmpfd) |
| { |
| int i; |
| struct tagcache_header tch; |
| struct master_header tcmh; |
| struct index_entry idxbuf[IDX_BUF_DEPTH]; |
| int idxbuf_pos; |
| char buf[TAG_MAXLEN+32]; |
| int fd = -1, masterfd; |
| bool error = false; |
| int init; |
| int masterfd_pos; |
| |
| logf("Building index: %d", index_type); |
| |
| /* Check the number of entries we need to allocate ram for. */ |
| commit_entry_count = h->entry_count + 1; |
| |
| masterfd = open_master_fd(&tcmh, false); |
| if (masterfd >= 0) |
| { |
| commit_entry_count += tcmh.tch.entry_count; |
| close(masterfd); |
| } |
| else |
| remove_files(); /* Just to be sure we are clean. */ |
| |
| /* Open the index file, which contains the tag names. */ |
| fd = open_tag_fd(&tch, index_type, true); |
| if (fd >= 0) |
| { |
| logf("tch.datasize=%ld", tch.datasize); |
| lookup_buffer_depth = 1 + |
| /* First part */ commit_entry_count + |
| /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH); |
| } |
| else |
| { |
| lookup_buffer_depth = 1 + |
| /* First part */ commit_entry_count + |
| /* Second part */ 0; |
| } |
| |
| logf("lookup_buffer_depth=%ld", lookup_buffer_depth); |
| logf("commit_entry_count=%ld", commit_entry_count); |
| |
| /* Allocate buffer for all index entries from both old and new |
| * tag files. */ |
| tempbufidx = 0; |
| tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx); |
| |
| /* Allocate lookup buffer. The first portion of commit_entry_count |
| * contains the new tags in the temporary file and the second |
| * part for locating entries already in the db. |
| * |
| * New tags Old tags |
| * +---------+---------------------------+ |
| * | index | position/ENTRY_CHUNK_SIZE | lookup buffer |
| * +---------+---------------------------+ |
| * |
| * Old tags are inserted to a temporary buffer with position: |
| * tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...); |
| * And new tags with index: |
| * tempbuf_insert(idx, ...); |
| * |
| * The buffer is sorted and written into tag file: |
| * tempbuf_sort(...); |
| * leaving master index locations messed up. |
| * |
| * That is fixed using the lookup buffer for old tags: |
| * new_seek = tempbuf_find_location(old_seek, ...); |
| * and for new tags: |
| * new_seek = tempbuf_find_location(idx); |
| */ |
| lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos]; |
| tempbuf_pos += lookup_buffer_depth * sizeof(void **); |
| memset(lookup, 0, lookup_buffer_depth * sizeof(void **)); |
| |
| /* And calculate the remaining data space used mainly for storing |
| * tag data (strings). */ |
| tempbuf_left = tempbuf_size - tempbuf_pos - 8; |
| if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0) |
| { |
| logf("Buffer way too small!"); |
| return 0; |
| } |
| |
| if (fd >= 0) |
| { |
| /** |
| * If tag file contains unique tags (sorted index), we will load |
| * it entirely into memory so we can resort it later for use with |
| * chunked browsing. |
| */ |
| if (TAGCACHE_IS_SORTED(index_type)) |
| { |
| logf("loading tags..."); |
| for (i = 0; i < tch.entry_count; i++) |
| { |
| struct tagfile_entry entry; |
| int loc = lseek(fd, 0, SEEK_CUR); |
| bool ret; |
| |
| if (ecread_tagfile_entry(fd, &entry) != sizeof(struct tagfile_entry)) |
| { |
| logf("read error #7"); |
| close(fd); |
| return -2; |
| } |
| |
| if (entry.tag_length >= (int)sizeof(buf)) |
| { |
| logf("too long tag #3"); |
| close(fd); |
| return -2; |
| } |
| |
| if (read(fd, buf, entry.tag_length) != entry.tag_length) |
| { |
| logf("read error #8"); |
| close(fd); |
| return -2; |
| } |
| |
| /* Skip deleted entries. */ |
| if (buf[0] == '\0') |
| continue; |
| |
| /** |
| * Save the tag and tag id in the memory buffer. Tag id |
| * is saved so we can later reindex the master lookup |
| * table when the index gets resorted. |
| */ |
| ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH |
| + commit_entry_count, entry.idx_id, |
| TAGCACHE_IS_UNIQUE(index_type)); |
| if (!ret) |
| { |
| close(fd); |
| return -3; |
| } |
| do_timed_yield(); |
| } |
| logf("done"); |
| } |
| else |
| tempbufidx = tch.entry_count; |
| } |
| else |
| { |
| /** |
| * Creating new index file to store the tags. No need to preload |
| * anything whether the index type is sorted or not. |
| */ |
| snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type); |
| fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC, 0666); |
| if (fd < 0) |
| { |
| logf("%s open fail", buf); |
| return -2; |
| } |
| |
| tch.magic = TAGCACHE_MAGIC; |
| tch.entry_count = 0; |
| tch.datasize = 0; |
| |
| if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ) |
| != sizeof(struct tagcache_header)) |
| { |
| logf("header write failed"); |
| close(fd); |
| return -2; |
| } |
| } |
| |
| /* Loading the tag lookup file as "master file". */ |
| logf("Loading index file"); |
| masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR); |
| |
| if (masterfd < 0) |
| { |
| logf("Creating new DB"); |
| masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC, 0666); |
| |
| if (masterfd < 0) |
| { |
| logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER); |
| close(fd); |
| return -2; |
| } |
| |
| /* Write the header (write real values later). */ |
| memset(&tcmh, 0, sizeof(struct master_header)); |
| tcmh.tch = *h; |
| tcmh.tch.entry_count = 0; |
| tcmh.tch.datasize = 0; |
| tcmh.dirty = true; |
| ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ); |
| init = true; |
| masterfd_pos = lseek(masterfd, 0, SEEK_CUR); |
| } |
| else |
| { |
| /** |
| * Master file already exists so we need to process the current |
| * file first. |
| */ |
| init = false; |
| |
| if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) != |
| sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC) |
| { |
| logf("header error"); |
| close(fd); |
| close(masterfd); |
| return -2; |
| } |
| |
| /** |
| * If we reach end of the master file, we need to expand it to |
| * hold new tags. If the current index is not sorted, we can |
| * simply append new data to end of the file. |
| * However, if the index is sorted, we need to update all tag |
| * pointers in the master file for the current index. |
| */ |
| masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry), |
| SEEK_CUR); |
| if (masterfd_pos == filesize(masterfd)) |
| { |
| logf("appending..."); |
| init = true; |
| } |
| } |
| |
| /** |
| * Load new unique tags in memory to be sorted later and added |
| * to the master lookup file. |
| */ |
| if (TAGCACHE_IS_SORTED(index_type)) |
| { |
| lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); |
| /* h is the header of the temporary file containing new tags. */ |
| logf("inserting new tags..."); |
| for (i = 0; i < h->entry_count; i++) |
| { |
| struct temp_file_entry entry; |
| |
| if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) != |
| sizeof(struct temp_file_entry)) |
| { |
| logf("read fail #3"); |
| error = true; |
| goto error_exit; |
| } |
| |
| /* Read data. */ |
| if (entry.tag_length[index_type] >= (long)sizeof(buf)) |
| { |
| logf("too long entry!"); |
| error = true; |
| goto error_exit; |
| } |
| |
| lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR); |
| if (read(tmpfd, buf, entry.tag_length[index_type]) != |
| entry.tag_length[index_type]) |
| { |
| logf("read fail #4"); |
| error = true; |
| goto error_exit; |
| } |
| |
| if (TAGCACHE_IS_UNIQUE(index_type)) |
| error = !tempbuf_insert(buf, i, -1, true); |
| else |
| error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false); |
| |
| if (error) |
| { |
| logf("insert error"); |
| goto error_exit; |
| } |
| |
| /* Skip to next. */ |
| lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] - |
| entry.tag_length[index_type], SEEK_CUR); |
| do_timed_yield(); |
| } |
| logf("done"); |
| |
| /* Sort the buffer data and write it to the index file. */ |
| lseek(fd, sizeof(struct tagcache_header), SEEK_SET); |
| /** |
| * We need to truncate the index file now. There can be junk left |
| * at the end of file (however, we _should_ always follow the |
| * entry_count and don't crash with that). |
| */ |
| ftruncate(fd, lseek(fd, 0, SEEK_CUR)); |
| |
| i = tempbuf_sort(fd); |
| if (i < 0) |
| goto error_exit; |
| logf("sorted %d tags", i); |
| |
| /** |
| * Now update all indexes in the master lookup file. |
| */ |
| logf("updating indices..."); |
| lseek(masterfd, sizeof(struct master_header), SEEK_SET); |
| for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos) |
| { |
| int j; |
| int loc = lseek(masterfd, 0, SEEK_CUR); |
| |
| idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH); |
| |
| if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ) |
| != (int)sizeof(struct index_entry)*idxbuf_pos) |
| { |
| logf("read fail #5"); |
| error = true; |
| goto error_exit ; |
| } |
| lseek(masterfd, loc, SEEK_SET); |
| |
| for (j = 0; j < idxbuf_pos; j++) |
| { |
| if (idxbuf[j].flag & FLAG_DELETED) |
| { |
| /* We can just ignore deleted entries. */ |
| // idxbuf[j].tag_seek[index_type] = 0; |
| continue; |
| } |
| |
| idxbuf[j].tag_seek[index_type] = tempbuf_find_location( |
| idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH |
| + commit_entry_count); |
| |
| if (idxbuf[j].tag_seek[index_type] < 0) |
| { |
| logf("update error: %ld/%d/%ld", |
| idxbuf[j].flag, i+j, tcmh.tch.entry_count); |
| error = true; |
| goto error_exit; |
| } |
| |
| do_timed_yield(); |
| } |
| |
| /* Write back the updated index. */ |
| if (ecwrite(masterfd, idxbuf, idxbuf_pos, |
| index_entry_ec, tc_stat.econ) != |
| (int)sizeof(struct index_entry)*idxbuf_pos) |
| { |
| logf("write fail"); |
| error = true; |
| goto error_exit; |
| } |
| } |
| logf("done"); |
| } |
| |
| /** |
| * Walk through the temporary file containing the new tags. |
| */ |
| // build_normal_index(h, tmpfd, masterfd, idx); |
| logf("updating new indices..."); |
| lseek(masterfd, masterfd_pos, SEEK_SET); |
| lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET); |
| lseek(fd, 0, SEEK_END); |
| for (i = 0; i < h->entry_count; i += idxbuf_pos) |
| { |
| int j; |
| |
| idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH); |
| if (init) |
| { |
| memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH); |
| } |
| else |
| { |
| int loc = lseek(masterfd, 0, SEEK_CUR); |
| |
| if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ) |
| != (int)sizeof(struct index_entry)*idxbuf_pos) |
| { |
| logf("read fail #6"); |
| error = true; |
| break ; |
| } |
| lseek(masterfd, loc, SEEK_SET); |
| } |
| |
| /* Read entry headers. */ |
| for (j = 0; j < idxbuf_pos; j++) |
| { |
| if (!TAGCACHE_IS_SORTED(index_type)) |
| { |
| struct temp_file_entry entry; |
| struct tagfile_entry fe; |
| |
| if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) != |
| sizeof(struct temp_file_entry)) |
| { |
| logf("read fail #7"); |
| error = true; |
| break ; |
| } |
| |
| /* Read data. */ |
| if (entry.tag_length[index_type] >= (int)sizeof(buf)) |
| { |
| logf("too long entry!"); |
| logf("length=%d", entry.tag_length[index_type]); |
| logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR)); |
| error = true; |
| break ; |
| } |
| |
| lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR); |
| if (read(tmpfd, buf, entry.tag_length[index_type]) != |
| entry.tag_length[index_type]) |
| { |
| logf("read fail #8"); |
| logf("offset=0x%02lx", entry.tag_offset[index_type]); |
| logf("length=0x%02x", entry.tag_length[index_type]); |
| error = true; |
| break ; |
| } |
| |
| /* Write to index file. */ |
| idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR); |
| fe.tag_length = entry.tag_length[index_type]; |
| fe.idx_id = tcmh.tch.entry_count + i + j; |
| ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ); |
| write(fd, buf, fe.tag_length); |
| tempbufidx++; |
| |
| /* Skip to next. */ |
| lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] - |
| entry.tag_length[index_type], SEEK_CUR); |
| } |
| else |
| { |
| /* Locate the correct entry from the sorted array. */ |
| idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j); |
| if (idxbuf[j].tag_seek[index_type] < 0) |
| { |
| logf("entry not found (%d)", j); |
| error = true; |
| break ; |
| } |
| } |
| } |
| |
| /* Write index. */ |
| if (ecwrite(masterfd, idxbuf, idxbuf_pos, |
| index_entry_ec, tc_stat.econ) != |
| (int)sizeof(struct index_entry)*idxbuf_pos) |
| { |
| logf("tagcache: write fail #4"); |
| error = true; |
| break ; |
| } |
| |
| do_timed_yield(); |
| } |
| logf("done"); |
| |
| /* Finally write the header. */ |
| tch.magic = TAGCACHE_MAGIC; |
| tch.entry_count = tempbufidx; |
| tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header); |
| lseek(fd, 0, SEEK_SET); |
| ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ); |
| |
| if (index_type != tag_filename) |
| h->datasize += tch.datasize; |
| logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize); |
| error_exit: |
| |
| close(fd); |
| close(masterfd); |
| |
| if (error) |
| return -2; |
| |
| return 1; |
| } |
| |
| static bool commit(void) |
| { |
| struct tagcache_header tch; |
| struct master_header tcmh; |
| int i, len, rc; |
| int tmpfd; |
| int masterfd; |
| #ifdef HAVE_DIRCACHE |
| bool dircache_buffer_stolen = false; |
| #endif |
| bool local_allocation = false; |
| |
| logf("committing tagcache"); |
| |
| while (write_lock) |
| sleep(1); |
| |
| tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY); |
| if (tmpfd < 0) |
| { |
| logf("nothing to commit"); |
| return true; |
| } |
| |
| |
| /* Load the header. */ |
| len = sizeof(struct tagcache_header); |
| rc = read(tmpfd, &tch, len); |
| |
| if (tch.magic != TAGCACHE_MAGIC || rc != len) |
| { |
| logf("incorrect tmpheader"); |
| close(tmpfd); |
| remove(TAGCACHE_FILE_TEMP); |
| return false; |
| } |
| |
| if (tch.entry_count == 0) |
| { |
| logf("nothing to commit"); |
| close(tmpfd); |
| remove(TAGCACHE_FILE_TEMP); |
| return true; |
| } |
| |
| /* Fully initialize existing headers (if any) before going further. */ |
| tc_stat.ready = check_all_headers(); |
| |
| #ifdef HAVE_EEPROM_SETTINGS |
| remove(TAGCACHE_STATEFILE); |
| #endif |
| |
| /* At first be sure to unload the ramcache! */ |
| #ifdef HAVE_TC_RAMCACHE |
| tc_stat.ramcache = false; |
| #endif |
| |
| read_lock++; |
| |
| /* Try to steal every buffer we can :) */ |
| if (tempbuf_size == 0) |
| local_allocation = true; |
| |
| #ifdef HAVE_DIRCACHE |
| if (tempbuf_size == 0) |
| { |
| /* Try to steal the dircache buffer. */ |
| tempbuf = dircache_steal_buffer(&tempbuf_size); |
| tempbuf_size &= ~0x03; |
| |
| if (tempbuf_size > 0) |
| { |
| dircache_buffer_stolen = true; |
| } |
| } |
| #endif |
| |
| #ifdef HAVE_TC_RAMCACHE |
| if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0) |
| { |
| tempbuf = (char *)(hdr + 1); |
| tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128; |
| tempbuf_size &= ~0x03; |
| } |
| #endif |
| |
| /* And finally fail if there are no buffers available. */ |
| if (tempbuf_size == 0) |
| { |
| logf("delaying commit until next boot"); |
| tc_stat.commit_delayed = true; |
| close(tmpfd); |
| read_lock--; |
| return false; |
| } |
| |
| logf("commit %ld entries...", tch.entry_count); |
| |
| /* Mark DB dirty so it will stay disabled if commit fails. */ |
| current_tcmh.dirty = true; |
| update_master_header(); |
| |
| /* Now create the index files. */ |
| tc_stat.commit_step = 0; |
| tch.datasize = 0; |
| tc_stat.commit_delayed = false; |
| |
| for (i = 0; i < TAG_COUNT; i++) |
| { |
| int ret; |
| |
| if (TAGCACHE_IS_NUMERIC(i)) |
| continue; |
| |
| tc_stat.commit_step++; |
| ret = build_index(i, &tch, tmpfd); |
| if (ret <= 0) |
| { |
| close(tmpfd); |
| logf("tagcache failed init"); |
| if (ret == 0) |
| tc_stat.commit_delayed = true; |
| |
| tc_stat.commit_step = 0; |
| read_lock--; |
| return false; |
| } |
| } |
| |
| if (!build_numeric_indices(&tch, tmpfd)) |
| { |
| logf("Failure to commit numeric indices"); |
| close(tmpfd); |
| tc_stat.commit_step = 0; |
| read_lock--; |
| return false; |
| } |
| |
| close(tmpfd); |
| |
| tc_stat.commit_step = 0; |
| |
| /* Update the master index headers. */ |
| if ( (masterfd = open_master_fd(&tcmh, true)) < 0) |
| { |
| read_lock--; |
| return false; |
| } |
| |
| remove(TAGCACHE_FILE_TEMP); |
| |
| tcmh.tch.entry_count += tch.entry_count; |
| tcmh.tch.datasize = sizeof(struct master_header) |
| + sizeof(struct index_entry) * tcmh.tch.entry_count |
| + tch.datasize; |
| tcmh.dirty = false; |
| tcmh.commitid++; |
| |
| lseek(masterfd, 0, SEEK_SET); |
| ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ); |
| close(masterfd); |
| |
| logf("tagcache committed"); |
| tc_stat.ready = check_all_headers(); |
| tc_stat.readyvalid = true; |
| |
| if (local_allocation) |
| { |
| tempbuf = NULL; |
| tempbuf_size = 0; |
| } |
| |
| #ifdef HAVE_DIRCACHE |
| /* Rebuild the dircache, if we stole the buffer. */ |
| if (dircache_buffer_stolen) |
| dircache_build(0); |
| #endif |
| |
| #ifdef HAVE_TC_RAMCACHE |
| /* Reload tagcache. */ |
| if
|