TDE
/
extra-dependencies
mirror of https://mirror.git.trinitydesktop.org/gitea/TDE/extra-dependencies.git
0
0
Fork 0
Code Issues Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from '1829166fcc'
${ noResults }
extra-dependencies/debian/htdig/htdig-3.2.0b6/db/mp_cmpr.c

1137 lines
31 KiB
Raw Blame History

/*-
* See the file LICENSE for redistribution information.
*
* Copyright (c) 1999, 2000
* Loic Dachary. All rights reserved.
*
* Overview of the code (by Lachlan Andrew, lha@users.sourceforge.net):
*
* This code compresses pages on-the-fly, either using a built-in algorithm,
* or using the zlib library. The compressed page is stored in pages of
* size CMPR_MULTIPLY(db_io->pagesize) -- a fixed multiple of the true
* page size, db_io->pagesize. If the compressed page requires multiple
* pages, extra pages are allocated at the end of the file, and "chained"
* on to the original page. The chain is specified as an array in the first
* page (not a linked list). If a subsequent write of the page requires
* a shorter chain, the spare pages are recorded as "free" and listed in
* the weak-compression database (with suffix given by DB_CMPR_SUFFIX).
*
* When writing a compressed page, extra memory may need to be allocated if
* chaining occurs. This can cause recursive calls to CDB___memp_alloc(),
* since the latter may write dirty cache pages to satisfy the request.
* There is currently an explicit check for recursive calls, both in
* CDB___memp_alloc() and CDB___memp_cmpr_write(), but a more elegant
* solution would be nice.
*
* There also seems to be an issue with the memory allocation for the chain
* array. The small allocations seem to cause fragmentation in the memory
* pool (seen as very many small clean blocks, which don't go away).
*
*
* TODO:
* Keith Bostic says:
* The only change I'd probably think about is if
* we should merge the call to CDB___memp_pg and CDB___memp_cmpr -- kind
* of a stack of page modification routines, that sits on top of
* CDB___os_io. That's a bigger change, but it's probably cleaner
* in the long-run.
*
* Pending questions:
*
* The CMPR structure contains binary data. Should we store them in network order ?
* How is this related to DB_AM_SWAP ?
*
* The calls to cmpr_open/cmpr_close in memp_pgread/pgwrite are probably not
* at the right place / most logical place in the function. I have troubles
* finding out where to put them. They work, that's not the problem. Just don't
* know if they should be before or after.
*
* When opening weakcmpr DB, the DB_THREAD flag is not available. Should it be
* set if the main DB has this flag set ?
*
* In CDB___memp_cmpr, niop is always multiplied by compression factor for page 0.
* I see no problems with this but it's a bit awkward.
*
* In CDB___memp_cmpr_page, the page built fills some fields of the PAGE structure
* others are set to 0. I'm not 100% sure this is enough. It should only impact
* utilities that read pages by incrementing pgno. Only stat does this an it's
* enough for it. I've not found any other context where these fake pages are
* used.
*
*/
#include "db_config.h"
#ifndef lint
static const char sccsid[] = "@(#)mp_cmpr.c 1.1 (Senga) 01/08/99";
#endif /* not lint */
#ifndef NO_SYSTEM_INCLUDES
#include <sys/types.h>
#include <errno.h>
#include <string.h>
#endif
#include "db_int.h"
#include "db_page.h"
#include "shqueue.h"
#include "db_shash.h"
#include "mp.h"
#include "db_page.h"
#include "common_ext.h"
#ifdef DEBUG
#include "WordMonitor.h"
#endif /* DEBUG */
#if 0
#define DEBUG_CMPR 1
#endif
#if 0
#define DEBUG_CMPR_ALLOC 1
#endif
#ifdef HAVE_LIBZ
#include "zlib.h"
#endif /* HAVE_LIBZ */
static int memp_cmpr_zlib_level = -1;
/*
* Helpers declarations.
*/
static int CDB___memp_cmpr_page __P((DB_MPOOLFILE *, CMPR *, DB_IO *, ssize_t *));
/*
* Maximum chain length
* Cast to signed, as -1 used as a flag, which compares bigger on some systems
*/
#define CMPR_MAX (int)(dbenv->mp_cmpr_info->max_npages)
#define CMPR_MULTIPLY(n) ((n) << (dbenv->mp_cmpr_info->coefficient))
#define CMPR_DIVIDE(n) ((n) >> (dbenv->mp_cmpr_info->coefficient))
#ifdef HAVE_LIBZ
static DB_CMPR_INFO default_cmpr_info = {
CDB___memp_cmpr_deflate,
CDB___memp_cmpr_inflate,
3, /* reduce page size by factor of 1<<3 = 8 */
9, /* allow 9 reduced pages, in case "compression" expands data */
6, /* zlib compression level */
NULL
};
#else /* HAVE_LIBZ */
static DB_CMPR_INFO default_cmpr_info = {
0,
0,
0,
0,
0,
NULL
};
#endif /* HAVE_LIBZ */
/*
* Entry point. Functionaly equivalent to CDB___os_io.
* Compress/uncompress pages before returning them or writing them to disk.
*/
/*
* CDB___memp_cmpr --
* Transparent compression read/write
*
* PUBLIC: int CDB___memp_cmpr __P((DB_MPOOLFILE *, BH *, DB_IO *, int, ssize_t *));
*/
int
CDB___memp_cmpr(dbmfp, bhp, db_io, flag, niop)
DB_MPOOLFILE *dbmfp;
BH *bhp;
DB_IO *db_io;
int flag;
ssize_t *niop;
{
size_t orig_pagesize = db_io->pagesize;
db_pgno_t orig_pgno = db_io->pgno;
size_t orig_bytes = db_io->bytes;
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info;
int ret = 0;
db_io->pagesize = CMPR_DIVIDE(db_io->pagesize);
db_io->bytes = CMPR_DIVIDE(db_io->bytes);
#ifdef HAVE_LIBZ
if(memp_cmpr_zlib_level == -1)
{
memp_cmpr_zlib_level = cmpr_info->zlib_flags;
if(memp_cmpr_zlib_level == -1)
memp_cmpr_zlib_level = Z_DEFAULT_COMPRESSION;
}
#endif
/*
* Page 0 is a special case. It contains the metadata information (at most 512 bytes)
* and must not be compressed because it is read with CDB___os_read and not CDB___os_io.
*/
switch (flag) {
case DB_IO_READ:
if(db_io->pgno == 0) {
ret = CDB___os_io(db_io, DB_IO_READ, niop);
*niop = CMPR_MULTIPLY(*niop);
} else
ret = CDB___memp_cmpr_read(dbmfp, bhp, db_io, niop);
break;
case DB_IO_WRITE:
if(db_io->pgno == 0) {
ret = CDB___os_io(db_io, DB_IO_WRITE, niop);
*niop = CMPR_MULTIPLY(*niop);
} else
ret = CDB___memp_cmpr_write(dbmfp, bhp, db_io, niop);
break;
}
db_io->pgno = orig_pgno;
db_io->pagesize = orig_pagesize;
db_io->bytes = orig_bytes;
return ret;
}
/*
* CDB___memp_cmpr_read --
* Transparent compression read
*
* PUBLIC: int CDB___memp_cmpr_read __P((DB_MPOOLFILE *, BH *, DB_IO *, ssize_t *));
*/
int
CDB___memp_cmpr_read(dbmfp, bhp, db_io, niop)
DB_MPOOLFILE *dbmfp;
BH *bhp;
DB_IO *db_io;
ssize_t *niop;
{
CMPR cmpr;
int ret;
int chain = 0;
u_int8_t *buffcmpr = 0;
int buffcmpr_length = 0;
int chain_length = 0;
db_pgno_t first_pgno = db_io->pgno;
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info;
/*
* By default the compression does not use too much space,
* hence the chain is empty.
*/
F_CLR(bhp, BH_CMPR);
/*
* Read first page (if no overflow, this is the only one)
*/
ret = CDB___os_io(db_io, DB_IO_READ, niop);
/*
* An error or partial read on the first page means that we're not
* going anywhere.
*/
if(ret || *niop < db_io->pagesize)
goto err;
/*
* Read the cmpr header from page.
*/
memcpy(&cmpr, db_io->buf, sizeof(CMPR));
/*
* If not at the beginning of compressed page chain, build
* a fake page.
*/
if(F_ISSET(&cmpr, DB_CMPR_FREE) || F_ISSET(&cmpr, DB_CMPR_INTERNAL)) {
ret = CDB___memp_cmpr_page(dbmfp, &cmpr, db_io, niop);
goto err;
}
/*
* Sanity check. Happens if file corrupted.
*/
if(!F_ISSET(&cmpr, DB_CMPR_FIRST)) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: expected DB_CMPR_FIRST flag set at pgno = %ld", db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL);
goto err;
}
if((ret = CDB___os_malloc(db_io->pagesize * CMPR_MAX, NULL, &buffcmpr)) != 0)
goto err;
do {
/*
* Read the first part of the compressed data from page.
*/
memcpy(buffcmpr + buffcmpr_length, DB_CMPR_DATA(db_io), DB_CMPR_PAGESIZE(db_io));
buffcmpr_length += DB_CMPR_PAGESIZE(db_io);
/*
* Flag must only contain FIRST|INTERNAL and/or CHAIN. If other bits are
* set, the data is corrupted. Removing the FIRST|INTERNAL bits and checking
* the CHAIN bit with == instead of F_ISSET verify this.
*/
F_CLR(&cmpr, DB_CMPR_FIRST | DB_CMPR_INTERNAL);
chain = cmpr.flags;
if(chain == DB_CMPR_CHAIN) {
/*
* Overflow Case. Continue reading data from extra pages.
*/
chain_length++;
if(chain_length >= CMPR_MAX) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: compression chain too long at pgno = %ld", db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL);
goto err;
}
if(cmpr.next == 0) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: cmpr.next is null at pgno = %ld", chain, db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL);
goto err;
}
/*
* Keep the chain in buffer header.
* Freed when bhp freed in CDB___memp_bhfree().
*/
CDB___memp_cmpr_alloc_chain(dbmfp->dbmp, bhp, BH_CMPR_POOL);
bhp->chain[chain_length - 1] = cmpr.next;
db_io->pgno = cmpr.next;
/*
* Read data from extra page.
*/
if((ret = CDB___os_io(db_io, DB_IO_READ, niop)) != 0 ||
*niop != db_io->pagesize) {
ret = EIO;
goto err;
}
/*
* Read the cmpr header from this extra page
*/
memcpy(&cmpr, db_io->buf, sizeof(CMPR));
} else if(chain != 0) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unexpected compression flag value 0x%x at pgno = %ld", chain, db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret);
goto err;
} else if(cmpr.next != 0) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: cmpr.next is not null at pgno = %ld", chain, db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret);
goto err;
}
} while(chain);
/*
* We gathered all the compressed data in buffcmpr, inflate it.
*/
if(cmpr_info->zlib_flags != 0)
ret = CDB___memp_cmpr_inflate(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data);
else
ret = (*cmpr_info->uncompress)(buffcmpr, buffcmpr_length, db_io->buf, CMPR_MULTIPLY(db_io->pagesize), cmpr_info->user_data);
if(ret != 0)
{
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_read: unable to uncompress page at pgno = %ld", first_pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret);
goto err;
}
#ifdef DEBUG
{
int ratio = buffcmpr_length > 0 ? (CMPR_MULTIPLY(db_io->pagesize) / buffcmpr_length) : 0;
if(ratio > 10) ratio = 10;
word_monitor_add(WORD_MONITOR_COMPRESS_01 + ratio, 1);
}
#endif /* DEBUG */
*niop = CMPR_MULTIPLY(db_io->pagesize);
err:
#ifdef DEBUG_CMPR
if(chain_length > 0) {
int i;
fprintf(stderr,"CDB___memp_cmpr_read:: chain_length (number of overflow pages):%2d\n",chain_length);
fprintf(stderr,"CDB___memp_cmpr_read:: chain ");
for(i = 0; i < chain_length; i++)
fprintf(stderr, "%d, ", bhp->chain[i]);
fprintf(stderr, "\n");
}
#endif
if(buffcmpr) CDB___os_free(buffcmpr, 0);
return ret;
}
/*
* CDB___memp_cmpr_write --
* Transparent compression write
*
* PUBLIC: int CDB___memp_cmpr_write __P((DB_MPOOLFILE *, BH *, DB_IO *, ssize_t *));
*/
int
CDB___memp_cmpr_write(dbmfp, bhp, db_io, niop)
DB_MPOOLFILE *dbmfp;
BH *bhp;
DB_IO *db_io;
ssize_t *niop;
{
CMPR cmpr;
int chain_length = 0;
int first_nonreused_chain_pos = 0;
int ret;
u_int8_t *buffcmpr = 0;
u_int8_t *buffp;
unsigned int buffcmpr_length;
u_int8_t *orig_buff = db_io->buf;
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
DB_CMPR_INFO *cmpr_info = dbenv->mp_cmpr_info;
if((ret = CDB___os_malloc(CMPR_MULTIPLY(db_io->bytes), NULL, &db_io->buf)) != 0)
goto err;
if(cmpr_info->zlib_flags != 0)
ret = CDB___memp_cmpr_deflate(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data);
else
ret = (*cmpr_info->compress)(orig_buff, CMPR_MULTIPLY(db_io->pagesize), &buffcmpr, &buffcmpr_length, cmpr_info->user_data);
if(ret != 0)
{
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: unable to compress page at pgno = %ld", db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, ret);
goto err;
}
#ifdef DEBUG
{
int ratio = buffcmpr_length > 0 ? (CMPR_MULTIPLY(db_io->pagesize) / buffcmpr_length) : 0;
if(ratio > 10) ratio = 10;
word_monitor_add(WORD_MONITOR_COMPRESS_01 + ratio, 1);
}
#endif /* DEBUG */
/*
* This can never happen.
*/
if(buffcmpr_length > DB_CMPR_PAGESIZE(db_io) * CMPR_MAX) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: compressed data is too big at pgno = %ld", db_io->pgno);
ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL);
goto err;
}
buffp = buffcmpr;
cmpr.flags = DB_CMPR_FIRST;
cmpr.next = 0;
/* write pages until the whole compressed data is written */
do {
unsigned int length = buffcmpr_length - (buffp - buffcmpr);
unsigned int copy_length = length > DB_CMPR_PAGESIZE(db_io) ? DB_CMPR_PAGESIZE(db_io) : length;
/*
* We handle serious compression stuff only if we need to.
* overflow! the compressed buffer is too big -> get extra page
*/
if(length > copy_length) {
if (dbmfp->dbmp->recursion_level >= 2 ) {
fprintf(stderr,"CDB___memp_cmpr_write: Wanted %d > %d bytes\n", length, copy_length);
fprintf(stderr,"Reducing wordlist_cache_dirty_level may help.\n");
ret = EBUSY;
goto err;
}
chain_length++;
if(chain_length >= CMPR_MAX) {
CDB___db_err(dbmfp->dbmp->dbenv, "CDB___memp_cmpr_write: chain_length overflow");
ret = CDB___db_panic(dbmfp->dbmp->dbenv, EINVAL);
goto err;
}
F_SET(&cmpr, DB_CMPR_CHAIN);
if((ret = CDB___memp_cmpr_alloc(dbmfp, &cmpr.next, bhp, &first_nonreused_chain_pos)) != 0)
goto err;
CDB___memp_cmpr_alloc_chain(dbmfp->dbmp, bhp, BH_CMPR_OS);
bhp->chain[chain_length - 1] = cmpr.next;
}
/* write in the cmpr header */
memcpy(db_io->buf, &cmpr, DB_CMPR_OVERHEAD);
/* write in what's left of the compressed buffer (and that also fits in) */
memcpy(db_io->buf + DB_CMPR_OVERHEAD, buffp, copy_length);
buffp += copy_length;
/* actual output */
if((ret = CDB___os_io(db_io, DB_IO_WRITE, niop)) != 0 ||
*niop != db_io->pagesize) {
ret = EIO;
goto err;
}
db_io->pgno = cmpr.next;
cmpr.flags = DB_CMPR_INTERNAL;
cmpr.next = 0;
} while((unsigned int)(buffp - buffcmpr) < buffcmpr_length);
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_write:: chain_length (number of overflow pages):%2d\n",chain_length);
if(chain_length > 0) {
int i;
fprintf(stderr,"CDB___memp_cmpr_write:: chain ");
for(i = 0; i < chain_length; i++)
fprintf(stderr, "%d, ", bhp->chain[i]);
fprintf(stderr, "\n");
}
#endif
/*
* If the chain was not completely reused, free the remaining pages (the page compression
* rate is better).
*/
if(F_ISSET(bhp, BH_CMPR) && first_nonreused_chain_pos >= 0) {
int i;
CMPR cmpr;
cmpr.flags = DB_CMPR_FREE;
cmpr.next = 0;
memcpy(db_io->buf, &cmpr, sizeof(CMPR));
for(i = first_nonreused_chain_pos; i < (CMPR_MAX - 1) && bhp->chain[i]; i++) {
if((ret = CDB___memp_cmpr_free(dbmfp, bhp->chain[i])) != 0)
goto err;
/*
* Mark the page as free for recovery.
*/
db_io->pgno = bhp->chain[i];
if((ret = CDB___os_io(db_io, DB_IO_WRITE, niop)) != 0 ||
*niop != db_io->pagesize) {
ret = EIO;
goto err;
}
bhp->chain[i] = 0;
}
}
CDB___memp_cmpr_free_chain(dbmfp->dbmp, bhp);
/*
* In case of success, always pretend that we exactly wrote the
* all bytes of the original pagesize.
*/
*niop = CMPR_MULTIPLY(db_io->pagesize);
err:
CDB___os_free(db_io->buf, 0);
db_io->buf = orig_buff;
if(buffcmpr) CDB___os_free(buffcmpr, 0);
return ret;
}
/*
* Helpers
*/
/*
* CDB___memp_cmpr_page --
* Build a fake page. This function is a CDB___memp_cmpr_read helper.
*
*/
static int
CDB___memp_cmpr_page(dbmfp, cmpr, db_io, niop)
DB_MPOOLFILE *dbmfp;
CMPR *cmpr;
DB_IO *db_io;
ssize_t *niop;
{
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
int ret = 0;
PAGE page;
memset((char*)&page, '\0', sizeof(PAGE));
page.pgno = db_io->pgno;
page.type = F_ISSET(cmpr, DB_CMPR_FREE) ? P_CMPR_FREE : P_CMPR_INTERNAL;
/*
* Sanity check
*/
if(db_io->pagesize < sizeof(PAGE)) {
ret = ENOMEM;
goto err;
}
memcpy(db_io->buf, (char*)&page, sizeof(PAGE));
*niop = CMPR_MULTIPLY(db_io->pagesize);
err:
return ret;
}
/*
* CDB___memp_cmpr_inflate --
* Decompress buffer
*
* PUBLIC: int CDB___memp_cmpr_inflate __P((const u_int8_t *, int, u_int8_t *, int, void *));
*/
int
CDB___memp_cmpr_inflate(inbuff, inbuff_length, outbuff, outbuff_length, user_data)
const u_int8_t* inbuff;
int inbuff_length;
u_int8_t* outbuff;
int outbuff_length;
void *user_data;
{
#ifdef HAVE_LIBZ
int ret = 0;
z_stream c_stream;
c_stream.zalloc=(alloc_func)0;
c_stream.zfree=(free_func)0;
c_stream.opaque=(voidpf)0;
c_stream.next_in = (Bytef*)inbuff;
c_stream.avail_in = inbuff_length;
c_stream.next_out = outbuff;
c_stream.avail_out = outbuff_length;
if(inflateInit(&c_stream) != Z_OK ||
inflate(&c_stream, Z_FINISH) != Z_STREAM_END ||
inflateEnd(&c_stream) != Z_OK)
ret = EIO;
/*
* The uncompressed data must *exactly* fill outbuff_length.
*/
if(c_stream.avail_out != 0)
ret = EIO;
return ret;
#else /* HAVE_LIBZ */
return EINVAL;
#endif /* HAVE_LIBZ */
}
/*
* CDB___memp_cmpr_deflate --
* Compress buffer
*
* PUBLIC: int CDB___memp_cmpr_deflate __P((const u_int8_t *, int, u_int8_t **, int*, void *));
*/
int
CDB___memp_cmpr_deflate(inbuff, inbuff_length, outbuffp, outbuff_lengthp, user_data)
const u_int8_t* inbuff;
int inbuff_length;
u_int8_t** outbuffp;
int* outbuff_lengthp;
void *user_data;
{
#ifdef HAVE_LIBZ
int ret = 0;
int r;
int off = 0;
int freesp = 0;
z_stream c_stream;
u_int8_t* outbuff;
/*
* Z_FINISH can be used immediately after deflateInit if all the compression
* is to be done in a single step. In this case, avail_out must be at least
* 0.1% larger than avail_in plus 12 bytes. If deflate does not return
* Z_STREAM_END, then it must be called again as described above.
*
* !!!
* In order to avoid division by 1000, divide by 512 (2^9) using shift.
* That is, make the buffer 0.2% larger.
*/
int outbuff_length = inbuff_length + (inbuff_length >> 9) + 12;
*outbuffp = 0;
*outbuff_lengthp = 0;
if(CDB___os_malloc(outbuff_length, NULL, &outbuff) != 0) {
ret = ENOMEM;
goto err;
}
/*
* Clear possible garbage in the page
*/
{
PAGE* pg = (PAGE*)inbuff;
switch(TYPE(pg)) {
case P_IBTREE:
case P_LBTREE:
off = LOFFSET(pg);
freesp = P_FREESPACE(pg);
memset((char*)(inbuff + off), 0, freesp);
break;
}
}
c_stream.zalloc=(alloc_func)0;
c_stream.zfree=(free_func)0;
c_stream.opaque=(voidpf)0;
if(deflateInit(&c_stream, memp_cmpr_zlib_level) != Z_OK)
{
ret = EIO;
goto err;
}
c_stream.next_in = (Bytef*)inbuff;
c_stream.avail_in = inbuff_length;
c_stream.next_out = outbuff;
c_stream.avail_out = outbuff_length;
while((r = deflate(&c_stream, Z_FINISH)) != Z_STREAM_END && r == Z_OK)
;
if(r != Z_STREAM_END)
ret = EIO;
if(deflateEnd(&c_stream) != Z_OK)
ret = EIO;
if(ret == 0) {
*outbuffp = outbuff;
*outbuff_lengthp = outbuff_length - c_stream.avail_out;
} else {
CDB___os_free(outbuff, outbuff_length);
}
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_deflate:: compress %d bytes to %d \n", inbuff_length, *outbuff_lengthp);
#endif
err:
return ret;
#else /* HAVE_LIBZ */
return EINVAL;
#endif /* HAVE_LIBZ */
}
/*
* CDB___memp_cmpr_info_valid --
* Compute compressed page size
*/
static int
CDB___memp_cmpr_info_valid(dbenv,cmpr_info)
DB_ENV *dbenv;
DB_CMPR_INFO *cmpr_info;
{
int ret = 0;
if(!cmpr_info ) {
CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: cmpr_info == NULL");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
if(!cmpr_info->compress ) {
CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: compress == NULL!");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
if(!cmpr_info->uncompress ) {
CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: uncompress == NULL!");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
if(cmpr_info->coefficient == 0 || cmpr_info->coefficient > 5 ) {
CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: coefficient should be > 0 and < 5 coefficient=%d ", cmpr_info->coefficient);
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
if(cmpr_info->max_npages == 0 || cmpr_info->max_npages > 128 ) {
CDB___db_err(dbenv, "CDB___memp_cmpr_info_valid: max_npages should be > 0 and < 128 max_npages=%d ", cmpr_info->max_npages);
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
err:
return ret;
}
/*
* __memp_cmpr_pagesize --
* Compute compressed page size
*
* PUBLIC: u_int8_t CDB___memp_cmpr_coefficient __P((DB_ENV *dbenv));
*/
u_int8_t
CDB___memp_cmpr_coefficient(dbenv)
DB_ENV *dbenv;
{
u_int8_t ret = 0;
if(!dbenv || !dbenv->mp_cmpr_info) {
ret = default_cmpr_info.coefficient;
} else {
CDB___memp_cmpr_info_valid(dbenv, dbenv->mp_cmpr_info);
ret = dbenv->mp_cmpr_info->coefficient;
}
return (ret);
}
/*
* Initialisation of page compression
*/
/*
* CDB___memp_cmpr_open --
* Open the db that contains the free compression pages.
*
* PUBLIC: int CDB___memp_cmpr_open __P((const char *, DB_ENV *, CMPR_CONTEXT *));
*/
int
CDB___memp_cmpr_open(dbenv, path, flags, mode, cmpr_context)
DB_ENV *dbenv;
const char *path;
int flags;
int mode;
CMPR_CONTEXT *cmpr_context;
{
int ret;
char* tmp = 0;
int tmp_length = strlen(path) + strlen(DB_CMPR_SUFFIX) + 1;
/*
* Management of pages containing data when the compression does not achieve
* the expected compression ratio.
*/
{
DB *dbp;
if((ret = CDB___os_malloc(tmp_length, NULL, &tmp)) != 0)
goto err;
sprintf(tmp, "%s%s", path, DB_CMPR_SUFFIX);
/* Use *standalone* database, to prevent recursion when writing pages */
/* from the cache, shared with other members of the environment */
if(CDB_db_create(&dbp, NULL, 0) != 0)
goto err;
cmpr_context->weakcmpr = dbp;
(dbp->set_flags)(dbp, DB_RECNUM);
LF_CLR(DB_COMPRESS);
if(!LF_ISSET(DB_RDONLY)) LF_SET(DB_CREATE);
if((ret = (dbp->open)(dbp, tmp, NULL, DB_BTREE, flags, mode)) != 0)
goto err;
}
/*
* Initialisation of cmpr_context
*/
if(!dbenv->mp_cmpr_info) {
if(default_cmpr_info.compress == 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_open: zlib compression not available, re-compile --with-zlib=DIR");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
dbenv->mp_cmpr_info = &default_cmpr_info;
}
/*
* Check if cmpr_info is sane
*/
if((ret = CDB___memp_cmpr_info_valid(dbenv, dbenv->mp_cmpr_info)))
goto err;
err:
if(tmp) CDB___os_free(tmp, tmp_length);
return ret;
}
/*
* CDB___memp_cmpr_close --
* Close the db that contains the free compression pages.
*
* PUBLIC: int CDB___memp_cmpr_close __P((CMPR_CONTEXT *));
*/
int
CDB___memp_cmpr_close(cmpr_context)
CMPR_CONTEXT *cmpr_context;
{
int ret = 0;
if(cmpr_context->weakcmpr == 0) {
ret = EINVAL;
goto err;
}
if((ret = cmpr_context->weakcmpr->close(cmpr_context->weakcmpr, 0)) != 0)
goto err;
cmpr_context->weakcmpr = 0;
err:
return ret;
}
/*
* CDB___memp_cmpr_alloc --
* Get a new free page to store weak compression data.
*
* PUBLIC: int CDB___memp_cmpr_alloc __P((DB_MPOOLFILE *, db_pgno_t *, BH *, int *));
*/
int
CDB___memp_cmpr_alloc(dbmfp, pgnop, bhp, first_nonreused_chain_posp)
DB_MPOOLFILE *dbmfp;
db_pgno_t *pgnop;
BH *bhp;
int *first_nonreused_chain_posp;
{
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
int ret = 0;
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_alloc:: bhp:%8x bhp->chain:%8x first_nonreused_chain_posp:%2d\n", bhp, bhp->chain, *first_nonreused_chain_posp);
#endif
if(F_ISSET(bhp, BH_CMPR) && bhp->chain == NULL) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: BH_CMPR set and bhp->chain == NULL");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
if((*first_nonreused_chain_posp) >= (CMPR_MAX - 1)) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: first_nonreused_chain_pos >= (CMPR_MAX - 1)");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
/*
* If possible reuse an existing chain.
*/
if((*first_nonreused_chain_posp) >= 0 && F_ISSET(bhp, BH_CMPR) && bhp->chain[*first_nonreused_chain_posp]) {
*pgnop = bhp->chain[*first_nonreused_chain_posp];
(*first_nonreused_chain_posp)++;
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_alloc:: reusing page in chain \n");
#endif
} else {
DB *db = dbmfp->cmpr_context.weakcmpr;
DBT key;
DBT data;
db_recno_t recno = 1;
/* all pages in bhp->chain are now reused */
(*first_nonreused_chain_posp) = -1;
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_alloc:: no more reusable pages in chain\n");
#endif
if(db == 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: dbmfp->cmpr_context.weakcmpr is null");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
/*
* If the existing chain is too short, pop a free page from
* the free pages database.
*/
memset(&key, '\0', sizeof(DBT));
memset(&data, '\0', sizeof(DBT));
key.data = &recno;
key.size = sizeof(recno);
if((ret = db->get(db, NULL, &key, &data, DB_SET_RECNO)) != 0) {
/*
* If the free list is empty, create a new page.
*/
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_alloc:: weakcmpr free page pool empty, allocating\n");
#endif
if(ret == DB_NOTFOUND) {
DB_MPOOL *dbmp = dbmfp->dbmp;
ret = 0;
R_LOCK(dbenv, &dbmp->reginfo);
++dbmfp->mfp->last_pgno;
#ifdef DEBUG
word_monitor_set(WORD_MONITOR_PGNO, dbmfp->mfp->last_pgno);
#endif /* DEBUG */
*pgnop = dbmfp->mfp->last_pgno;
R_UNLOCK(dbenv, &dbmp->reginfo);
ret = 0;
} else {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected error from weakcmpr base");
ret = CDB___db_panic(dbenv, ret);
}
} else {
if(key.size != sizeof(db_pgno_t)) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected key size from weakcmpr base (%d instead of %d)", key.size, sizeof(db_pgno_t));
ret = CDB___db_panic(dbenv, ret);
} else {
memcpy((char*)pgnop, (char*)key.data, key.size);
if((ret = db->del(db, NULL, &key, 0)) != 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: del error, got pgno %d", *pgnop);
ret = CDB___db_panic(dbenv, ret);
}
if(*pgnop == 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc: unexpected pgno == 0");
ret = CDB___db_panic(dbenv, ret);
}
}
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_alloc:: reuse free page %d from weakcmpr\n", *pgnop);
#endif
}
}
err:
return ret;
}
/*
* CDB___memp_cmpr_free --
* Release a page used to store weak compression data.
*
* PUBLIC: int CDB___memp_cmpr_free __P((DB_MPOOLFILE *, db_pgno_t));
*/
int
CDB___memp_cmpr_free(dbmfp, pgno)
DB_MPOOLFILE *dbmfp;
db_pgno_t pgno;
{
int ret = 0;
DB_ENV *dbenv = dbmfp->dbmp->dbenv;
DB *db = dbmfp->cmpr_context.weakcmpr;
DBT key;
DBT data;
#ifdef DEBUG_CMPR
fprintf(stderr,"CDB___memp_cmpr_free:: freeing page (inserting into weakcmpr):%3d \n",pgno);
#endif
if(db == 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_free: dbmfp->cmpr_context.weakcmpr is null");
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
memset(&key, '\0', sizeof(DBT));
memset(&data, '\0', sizeof(DBT));
key.data = &pgno;
key.size = sizeof(db_pgno_t);
data.data = " ";
data.size = 1;
if((ret = db->put(db, 0, &key, &data, DB_NOOVERWRITE)) != 0) {
CDB___db_err(dbenv, "CDB___memp_cmpr_free: put failed for pgno = %d", pgno);
ret = CDB___db_panic(dbenv, ret);
goto err;
}
err:
return ret;
}
/*
* CDB___memp_cmpr_alloc_chain --
* Allocate chain entry in BH
*
* PUBLIC: int CDB___memp_cmpr_alloc_chain __P((DB_MPOOL *, BH *));
*/
int
CDB___memp_cmpr_alloc_chain(dbmp, bhp, alloc_type)
DB_MPOOL *dbmp;
BH *bhp;
int alloc_type;
{
DB_ENV *dbenv = dbmp->dbenv;
int ret = 0;
if(!bhp->chain) {
int alloc_ret;
int alloc_length = sizeof(db_pgno_t)*(CMPR_MAX-1);
switch(alloc_type) {
case BH_CMPR_POOL:
{
MPOOL *mp = dbmp->reginfo.primary;
int n_cache = NCACHE(mp, bhp->pgno);
alloc_ret = CDB___memp_alloc(dbmp, &dbmp->c_reginfo[n_cache], NULL, alloc_length, NULL, (void *)(&bhp->chain));
F_SET(bhp, BH_CMPR_POOL);
}
break;
case BH_CMPR_OS:
alloc_ret = CDB___os_malloc(alloc_length, NULL, &bhp->chain);
F_SET(bhp, BH_CMPR_OS);
break;
default:
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc_chain: unknown alloc type :%d", alloc_type);
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
break;
}
if(alloc_ret) {
CDB___db_err(dbenv, "CDB___memp_cmpr_alloc_chain: memp_alloc %d bytes failed:%d", alloc_length, alloc_ret);
ret = CDB___db_panic(dbenv, EINVAL);
goto err;
}
memset((void *)bhp->chain, 0, alloc_length);
#if defined(DEBUG_CMPR) || defined(DEBUG_CMPR_ALLOC)
fprintf(stderr, "CDB___memp_cmpr_alloc_chain:: allocate chain in %s :%8x\n", (alloc_type == BH_CMPR_OS ? "malloc" : "shalloc"), bhp->chain);
#endif
} else {
#ifdef DEBUG_CMPR
fprintf(stderr, "CDB___memp_cmpr_alloc_chain:: existing chain:%8x\n", bhp->chain);
#endif
}
F_SET(bhp, BH_CMPR);
err:
return ret;
}
/*
* CDB___memp_cmpr_free_chain --
* Free chain entry in BH
*
* PUBLIC: int CDB___memp_cmpr_free_chain __P((DB_MPOOL *, BH *));
*/
int
CDB___memp_cmpr_free_chain(dbmp, bhp)
DB_MPOOL *dbmp;
BH *bhp;
{
DB_ENV *dbenv = dbmp->dbenv;
if(F_ISSET(bhp, BH_CMPR)) {
if(bhp->chain) {
int alloc_length = sizeof(db_pgno_t)*(CMPR_MAX-1);
int alloc_type = bhp->flags & (BH_CMPR_POOL|BH_CMPR_OS);
switch(alloc_type) {
case BH_CMPR_POOL:
{
MPOOL *mp = dbmp->reginfo.primary;
int n_cache = NCACHE(mp, bhp->pgno);
CDB___db_shalloc_free(dbmp->c_reginfo[n_cache].addr, bhp->chain);
}
break;
case BH_CMPR_OS:
CDB___os_free(bhp->chain, alloc_length);
break;
default:
CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: unknown alloc type :%d", alloc_type);
return CDB___db_panic(dbenv, EINVAL);
break;
}
#if defined(DEBUG_CMPR) || defined(DEBUG_CMPR_ALLOC)
fprintf(stderr, "CDB___memp_cmpr_free_chain:: free chain in %s :%8x\n", (alloc_type == BH_CMPR_OS ? "malloc" : "shalloc"), bhp->chain);
#endif
bhp->chain = NULL;
} else {
CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: BH_CMPR set but null bhp->chain");
return CDB___db_panic(dbenv, EINVAL);
}
} else if(bhp->chain) {
CDB___db_err(dbenv, "CDB___memp_cmpr_free_chain: BH_CMPR not set but bhp->chain not null");
return CDB___db_panic(dbenv, EINVAL);
}
F_CLR(bhp, BH_CMPR | BH_CMPR_OS | BH_CMPR_POOL);
return 0;
}
View Git Blame Copy Permalink