diff options
Diffstat (limited to '')
| -rw-r--r-- | project/src/lib-checksum.c | 235 |
1 files changed, 214 insertions, 21 deletions
diff --git a/project/src/lib-checksum.c b/project/src/lib-checksum.c index 37215e8..ad29b19 100644 --- a/project/src/lib-checksum.c +++ b/project/src/lib-checksum.c @@ -175,6 +175,50 @@ void CreateSSChecksumDBBySZS ( char *buf, uint bufsize, const szs_file_t *szs ) } /////////////////////////////////////////////////////////////////////////////// + +void CreateSSXChecksumDBByData ( char *buf, uint bufsize, cvp data, uint size, file_format_t ff ) +{ + DASSERT(data||!size); + DASSERT(bufsize>CHECKSUM_DB_SIZE); + + if (!data) + *buf = 0; + else + { + sha1_size_hash_t info; + SHA1(data,size,info.hash); + info.size = htonl(size); + CreateSSChecksumDB(buf,bufsize,&info); + + if ( bufsize > CHECKSUM_DB_SIZE + 2 ) + { + ccp append = 0; + switch((int)ff) + { + case FF_YAZ0: append = ".y"; break; + case FF_BZIP2: append = ".b"; break; + case FF_LZMA: append = ".l"; break; + } + if (append) + memcpy(buf+CHECKSUM_DB_SIZE,append,3); + } + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void CreateSSXChecksumDBBySZS ( char *buf, uint bufsize, const szs_file_t *szs ) +{ + DASSERT(buf); + DASSERT(bufsize>CHECKSUM_DB_SIZE); + + if (szs) + CreateSSXChecksumDBByData(buf,bufsize,szs->data,szs->size,szs->fform_file); + else + *buf = 0; +} + +/////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////// enumError GetSSByFile ( sha1_size_hash_t *ss, ccp path1, ccp path2 ) @@ -277,7 +321,7 @@ void LogCacheActivity ( ccp keyword, ccp format, ... ) { if (!opt_log_cache) return; - f = fopen(opt_log_cache,"a"); + f = strcmp(opt_log_cache,"-") ? fopen(opt_log_cache,"a") : stdlog; if (!f) { ERROR1(ERR_CANT_CREATE,"Can't create log file: %s\n",opt_log_cache); @@ -377,7 +421,7 @@ enumError LoadSZSCache(void) char fbuf[PATH_MAX], buf[PATH_MAX]; ccp path = PathCatPP(fbuf,sizeof(fbuf),szs_cache_dir,SZS_CACHE_FNAME); - PRINT("LoadSZSCache() %s\n",path); + PRINT0("LoadSZSCache() %s\n",path); File_t F; enumError err = OpenFile(&F,true,path,FM_SILENT,0,0); @@ -393,9 +437,9 @@ enumError LoadSZSCache(void) while (fgets(buf,sizeof(buf)-1,F.f)) { char *ptr = buf; - while ( *(uchar*)ptr <= ' ' ) + while ( *ptr > 0 && *ptr <= ' ' ) ptr++; - if ( *ptr == '#' || *ptr == '!' ) + if ( *ptr == '#' || *ptr == '!' || !*ptr ) continue; if ( *ptr == '@' ) @@ -426,8 +470,6 @@ enumError LoadSZSCache(void) *name = 0; if (!strcmp(name_buf,"LAST-CACHE-SCAN")) szs_cache_last_scan = strtoul(ptr,0,10); - // else if (!strcmp(name_buf,"APPEND")) - // szs_cache_last_append = strtoul(ptr,0,10); } continue; } @@ -437,16 +479,25 @@ enumError LoadSZSCache(void) len--; ptr[len+1] = 0; - if ( len <= CHECKSUM_DB_SIZE || ptr[CHECKSUM_DB_SIZE] != ' ' ) + char *fname = strchr(ptr,' '); + if (!fname) + continue; + + const uint cslen = fname - ptr; + if ( cslen != CHECKSUM_DB_SIZE && cslen != CHECKSUM_DB_SIZE+2 ) continue; - char *fname = ptr + CHECKSUM_DB_SIZE; - *fname++ = 0; - while ( *(uchar*)fname <= ' ' ) + while ( *fname > 0 && *fname <= ' ' ) fname++; + + char csum[CHECKSUM_DB_SIZE+3]; + memcpy(csum,ptr,cslen); + csum[cslen] = 0; + if ( cslen == CHECKSUM_DB_SIZE ) + memcpy(csum+CHECKSUM_DB_SIZE,".y",3); if (!*fname) continue; - StoreSZSCache(fname,ptr,false,0); + StoreSZSCache(fname,csum,false,0); } CloseFile(&F,0); @@ -487,7 +538,7 @@ enumError SaveSZSCache ( bool force ) char path_buf[PATH_MAX]; ccp path = PathCatPP(path_buf,sizeof(path_buf),szs_cache_dir,SZS_CACHE_FNAME); - PRINT1("SaveSZSCache() N=%u, %s\n",szs_cache.used,path); + PRINT("SaveSZSCache() N=%u, %s\n",szs_cache.used,path); File_t F; enumError err = CreateFile(&F,true,path,FM_REMOVE); @@ -550,7 +601,7 @@ enumError AppendSZSCache() char path_buf[PATH_MAX]; ccp path = PathCatPP(path_buf,sizeof(path_buf),szs_cache_dir,SZS_CACHE_FNAME); - PRINT("SaveSZSCache() N=%u, %s\n",szs_cache.used,path); + PRINT("AppendSZSCache() N=%u, %s\n",szs_cache.used,path); File_t F; enumError err = CreateFile(&F,true,path,FM_APPEND); @@ -585,7 +636,7 @@ ParamFieldItem_t * StoreSZSCache // return NULL or pointer to cache item ccp fname, // NULL or filename, only basename(fname) is used - ccp checksum, // DB checksum; if NULL: load file and calc checksum + ccp checksum, // DB checksum + ".x"; if NULL: load file and calc checksum bool rename_file, // true: rename existing file bool *r_exists // not NULL: store a status here // => true: file result->data exists @@ -608,10 +659,8 @@ ParamFieldItem_t * StoreSZSCache if (!*fname) fname = 0; - PRINT("StoreSZSCache(%s,%s,%d,)\n",fname,checksum,rename_file); - ccp path = 0; - char path_buf[PATH_MAX], oldpath_buf[PATH_MAX], checksum_buf[CHECKSUM_DB_SIZE+1]; + char path_buf[PATH_MAX], oldpath_buf[PATH_MAX], checksum_buf[CHECKSUM_DB_SIZE+3]; if (!checksum) { @@ -619,7 +668,6 @@ ParamFieldItem_t * StoreSZSCache return 0; path = PathCatPP(path_buf,sizeof(path_buf),szs_cache_dir,fname); - noPRINT(">> %s\n",path); szs_file_t szs; InitializeSZS(&szs); @@ -630,15 +678,16 @@ ParamFieldItem_t * StoreSZSCache return 0; } - CreateSSChecksumDBBySZS(checksum_buf,sizeof(checksum_buf),&szs); - ResetSZS(&szs); + CreateSSXChecksumDBBySZS(checksum_buf,sizeof(checksum_buf),&szs); checksum = checksum_buf; + ResetSZS(&szs); } bool found; ParamFieldItem_t *it = FindInsertParamField(&szs_cache,checksum,false,0,&found); if ( !found && parallel_count > 0 ) it = FindInsertParamField(&szs_cache_append,checksum,false,0,&found); + PRINT0("N=%d, found[%s]=%d\n",szs_cache.used,checksum,found); if (found) { @@ -657,7 +706,7 @@ ParamFieldItem_t * StoreSZSCache if (!path) path = PathCatPP(path_buf,sizeof(path_buf),szs_cache_dir,fname); - PRINT("RENAME: %s -> %s\n",oldpath,path); + PRINT0("RENAME: %s -> %s\n",oldpath,path); if (rename(oldpath,path)) return it; @@ -674,6 +723,150 @@ ParamFieldItem_t * StoreSZSCache return it; } +/////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////////// + +void ResetCheckCache ( check_cache_t *cc ) +{ + if (cc) + { + ResetSZS(&cc->cache); + memset(cc,0,sizeof(*cc)); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +static void fix_ext ( char *buf, uint bufsize, ccp ext ) +{ + char *point = strrchr(buf,'.'); + if ( !point || strcmp(point,ext) ) + { + if ( point && !strcmp(point,".szs") ) + *point = 0; + StringCat2S(buf,bufsize,buf,ext); + } +} + +//----------------------------------------------------------------------------- + +bool CheckSZSCache +( + // returns true if file found in cache (same as cc->found) + + check_cache_t *cc, // initialized by CheckSZSCache() + szs_file_t *szs, // related SZS file + cvp data, // data + uint size, // size of 'data' + file_format_t fform, // compression format + ccp ext // wanted file extension including leading '.' +) +{ + ASSERT(cc); + ASSERT(szs); + ASSERT(ext); + + memset(cc,0,sizeof(*cc)); + cc->fform = fform; + + LoadSZSCache(); + CreateSSXChecksumDBByData(cc->csum,sizeof(cc->csum),data,size,fform); + + ccp dest_fname; + if ( opt_cname && *opt_cname ) + { + dest_fname = opt_cname; + opt_cname = 0; + } + else + dest_fname = szs->dest_fname ? szs->dest_fname : szs->fname; + + char path[PATH_MAX]; + StringCopyS(path,sizeof(path),dest_fname); + fix_ext(path,sizeof(path),ext); + dest_fname = path; + + bool exists; + const ParamFieldItem_t *cit + = StoreSZSCache( dest_fname, cc->csum, true, &exists ); +//PRINT1("checklsum = %s, cit=%d, exists=%d\n",cc->csum,cit!=0,exists); + + ccp cache_fname = 0; + if ( exists && cit ) + { + cache_fname = PathAllocPP(szs_cache_dir,cit->data); +//PRINT1("CACHE FOUND[%s]: %s\n",cit->key,cache_fname); + + InitializeSZS(&cc->cache); + + if ( !LoadSZS(&cc->cache,cache_fname,false,true,true) + && cc->cache.cdata + && cc->cache.csize + && cc->cache.fform_file == fform ) + { + PRINT("CACHE LOADED, %zu bytes\n",cc->cache.csize); + //HEXDUMP16(0,0,cc->cache.cdata,32); + DecompressSZS(&cc->cache,false,0); + if ( cc->cache.size == size && !memcmp(data,cc->cache.data,cc->cache.size)) + { + LogCacheActivity("USE","%s %s",cit->key,(ccp)cit->data); + FreeString(cache_fname); + return cc->found = true; + } + } + else + LogCacheActivity("!USE","%s %s",cit->key,(ccp)cit->data); + ResetSZS(&cc->cache); + } + + if (cit) + { + if (cache_fname) + StringCopyS(path,sizeof(path),cache_fname); + else + PathCatPP(path,sizeof(path),szs_cache_dir,cit->data); + fix_ext(path,sizeof(path),ext); + + FreeString(szs->cache_fname); + szs->cache_fname = STRDUP(path); +//PRINT1("szs->cache_fname = %s\n",szs->cache_fname); + } + + return false; +} + +/////////////////////////////////////////////////////////////////////////////// + +bool CheckSZSCacheSZS +( + // returns true if file found in cache (same as cc->found) + + szs_file_t *szs, // related SZS file + file_format_t fform, // compression format + ccp ext, // wanted file extension including leading '.' + bool rm_uncompressed // true: remove uncompressed data if cache is used +) +{ + ASSERT(szs); + check_cache_t cc; + if (CheckSZSCache(&cc,szs,szs->data,szs->size,fform,ext)) + { + FREE(szs->cdata); + szs->cdata = cc.cache.cdata; + szs->csize = cc.cache.csize; + szs->cache_used = true; + cc.cache.cdata = 0; + cc.cache.csize = 0; + ResetCheckCache(&cc); + if (rm_uncompressed) + ClearUncompressedSZS(szs); + return true; + } + + ResetCheckCache(&cc); + return false; +} + // /////////////////////////////////////////////////////////////////////////////// /////////////// END /////////////// |