You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3544 lines
89 KiB
3544 lines
89 KiB
//
|
|
// search.cc
|
|
//
|
|
// search: Sample implementation of search algorithms using
|
|
// a mifluz inverted index.
|
|
//
|
|
// Each class is documented in the class definition. Before
|
|
// each method declaration a comment explains the semantic of
|
|
// the method. In the method definition comments in the code
|
|
// may contain additional information.
|
|
//
|
|
// Each virtual function is documented in the base class, not
|
|
// in the derived classes except for semantic differences.
|
|
//
|
|
// The class tree is:
|
|
//
|
|
// WordKeySemantic
|
|
//
|
|
// WordExclude
|
|
// WordExcludeMask
|
|
// WordPermute
|
|
//
|
|
// WordSearch
|
|
//
|
|
// WordMatch
|
|
//
|
|
// WordTree
|
|
// WordTreeOperand
|
|
// WordTreeOptional
|
|
// WordTreeOr
|
|
// WordTreeAnd
|
|
// WordTreeNear
|
|
// WordTreeMandatory
|
|
// WordTreeNot
|
|
// WordTreeLiteral
|
|
//
|
|
// WordParser
|
|
//
|
|
// Part of the ht://Dig package <http://www.htdig.org/>
|
|
// Copyright (c) 1999-2004 The ht://Dig Group
|
|
// For copyright details, see the file COPYING in your distribution
|
|
// or the GNU Library General Public License (LGPL) version 2 or later
|
|
// <http://www.gnu.org/copyleft/lgpl.html>
|
|
//
|
|
// $Id: search.cc,v 1.9 2004/05/28 13:15:29 lha Exp $
|
|
//
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include <htconfig.h>
|
|
#endif /* HAVE_CONFIG_H */
|
|
|
|
#ifdef HAVE_UNISTD_H
|
|
#include <unistd.h>
|
|
#endif /* HAVE_UNISTD_H */
|
|
|
|
// If we have this, we probably want it.
|
|
#ifdef HAVE_GETOPT_H
|
|
#include <getopt.h>
|
|
#endif /* HAVE_GETOPT_H */
|
|
#ifdef HAVE_MALLOC_H
|
|
#include <malloc.h>
|
|
#endif /* HAVE_MALLOC_H */
|
|
#include <stdlib.h>
|
|
|
|
#include <htString.h>
|
|
#include <WordList.h>
|
|
#include <WordContext.h>
|
|
#include <WordCursor.h>
|
|
|
|
//
|
|
// Verbosity level set with -v (++)
|
|
//
|
|
static int verbose = 0;
|
|
|
|
// ************************* Document definition implementation ***********
|
|
|
|
#define TAG 1
|
|
#define SERVER 2
|
|
#define URL 3
|
|
#define LOCATION 4
|
|
|
|
// *********************** WordKeySemantic implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// encapsulate WordKey semantic for document and location
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordKeySemantic.h>
|
|
//
|
|
// #define SERVER 1
|
|
// #define URL 2
|
|
// #define LOCATION 3
|
|
//
|
|
// static int document[] = {
|
|
// SERVER,
|
|
// URL
|
|
// };
|
|
//
|
|
// WordKeySemantic semantic;
|
|
// semantic.Initialize(document, sizeof(document)/sizeof(int), LOCATION);
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// Encapsulate the semantic of a WordKey object fields. It defines
|
|
// what a document and a location are. It implements the set of
|
|
// operation that a search needs to perform given the fact that it
|
|
// implements a search whose purpose is to retrieve a document and
|
|
// wants to implement proximity search based on a word location.
|
|
//
|
|
//
|
|
// END
|
|
//
|
|
// A document is a set of fields in a given order.
|
|
// A location is a field.
|
|
// The actual fields used to implement WordKeySemantic methods are
|
|
// set with the Initialize method.
|
|
//
|
|
class WordKeySemantic {
|
|
public:
|
|
WordKeySemantic();
|
|
~WordKeySemantic();
|
|
|
|
//-
|
|
// Set the actual field numbers that define what a document is and
|
|
// what a location is. The <b>document_arg<b> is a list of WordKey field
|
|
// positions of length <b>document_length_arg</b> that must be adjacent.
|
|
// The <b>location_arg</b> is the WordKey field position of the word
|
|
// location within a document.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
int Initialize(int* document_arg, int document_length_arg, int location_arg);
|
|
|
|
//
|
|
// These functions and only these know what a document is.
|
|
// This should really be a class containing function pointers and be
|
|
// given as argument to the search algorithm.
|
|
//
|
|
//-
|
|
// Copy the document in <b>from</b> into <b>to.</b>
|
|
//
|
|
void DocumentSet(const WordKey& from, WordKey& to);
|
|
//-
|
|
// Increment the document in <b>key</b> using the <i>SetToFollowing</i>
|
|
// method of WordKey. <b>uniq</b> is the WordKey position at which the
|
|
// increment starts.
|
|
//
|
|
void DocumentNext(WordKey& key, int uniq);
|
|
//-
|
|
// Compare the document fields defined in both <b>a</b> and <b>b</b>
|
|
// and return the difference a - b, as in strcmp. If all document
|
|
// fields in <b>a</b> or <b>b</b> are undefined return 1.
|
|
//
|
|
int DocumentCompare(const WordKey& a, const WordKey& b);
|
|
//-
|
|
// Set all document fields to 0.
|
|
//
|
|
int DocumentClear(WordKey& key);
|
|
|
|
//
|
|
// These functions and only these know what a location is.
|
|
// This should really be a class containing function pointers and be
|
|
// given as argument to the search algorithm.
|
|
//
|
|
//-
|
|
// Copy the document and location in <b>from</b> into <b>to.</b>
|
|
//
|
|
void LocationSet(const WordKey& from, WordKey& to);
|
|
//-
|
|
// Increment the document and location in <b>key</b>
|
|
// using the <i>SetToFollowing</i>
|
|
// method of WordKey.
|
|
//
|
|
void LocationNext(WordKey& key);
|
|
//-
|
|
// Compare <b>expected</b> location to <b>actual</b> location. Compares equal
|
|
// as long as expected location is at a maximum distance of <b>proximity</b>
|
|
// of actual. If <b>actual</b> only has undefined field, return > 0.
|
|
// <b>expected</b> must always be the lowest possible bound.
|
|
// <b>actual</b> is tolerated if it is greater than <b>actual</b> but not
|
|
// greater than <b>proximity</b> if <b>proximity</b> > 0 or abs(<b>proximity</b>) * 2 if
|
|
// <b>proximity</b> < 0.
|
|
// Return the difference expected - actual.
|
|
//
|
|
int LocationCompare(const WordKey& expected, const WordKey& actual, int proximity = 0);
|
|
//-
|
|
// <b>key</b> is the expected location of a searched key.
|
|
// LocationNearLowest modifies <b>key</b> to add tolerance accroding to
|
|
// <b>proximity</b>.
|
|
//
|
|
// The idea is that <b>key</b> will be the lowest possible match for
|
|
// for the <b>proximity</b> range. If <proxmity> is positive, <b>key</b>
|
|
// is already the lowest possible match since we accept [0 proximity].
|
|
// If <b>proximity</b> is negative, substract it since we accept
|
|
// [-proximity proximity].
|
|
//
|
|
// For better understanding see the functions in which it is used.
|
|
//
|
|
void LocationNearLowest(WordKey& key, int proximity);
|
|
|
|
//-
|
|
// Undefined the location field in <b>key.</b>.
|
|
//
|
|
void Location2Document(WordKey& key);
|
|
|
|
protected:
|
|
int* document;
|
|
int document_length;
|
|
int location;
|
|
};
|
|
|
|
WordKeySemantic::WordKeySemantic()
|
|
{
|
|
int nfields = WordKey::NFields();
|
|
document = new int[nfields];
|
|
document_length = 0;
|
|
location = -1;
|
|
}
|
|
|
|
WordKeySemantic::~WordKeySemantic()
|
|
{
|
|
if(document) delete [] document;
|
|
}
|
|
|
|
int WordKeySemantic::Initialize(int* document_arg, int document_length_arg, int location_arg)
|
|
{
|
|
memcpy((char*)document, (char*)document_arg, document_length_arg * sizeof(int));
|
|
document_length = document_length_arg;
|
|
location = location_arg;
|
|
return OK;
|
|
}
|
|
|
|
void WordKeySemantic::DocumentSet(const WordKey& from, WordKey& to)
|
|
{
|
|
to.Clear();
|
|
for(int i = 0; i < document_length; i++)
|
|
to.Set(document[i], from.Get(document[i]));
|
|
}
|
|
|
|
int WordKeySemantic::DocumentCompare(const WordKey& a, const WordKey& b)
|
|
{
|
|
int ret = 1;
|
|
for(int i = 0; i < document_length; i++) {
|
|
int idx = document[i];
|
|
if((a.IsDefined(idx) && b.IsDefined(idx)) &&
|
|
(ret = a.Get(idx) - b.Get(idx)) != 0) return ret;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int WordKeySemantic::DocumentClear(WordKey& key)
|
|
{
|
|
for(int i = 0; i < document_length; i++)
|
|
key.Set(document[i], 0);
|
|
return 0;
|
|
}
|
|
|
|
void WordKeySemantic::DocumentNext(WordKey& key, int uniq)
|
|
{
|
|
if(uniq)
|
|
key.SetToFollowing(uniq);
|
|
else
|
|
key.SetToFollowing(document[document_length-1]);
|
|
}
|
|
|
|
|
|
void WordKeySemantic::LocationSet(const WordKey& from, WordKey& to)
|
|
{
|
|
DocumentSet(from, to);
|
|
to.Set(location, from.Get(location));
|
|
}
|
|
|
|
int WordKeySemantic::LocationCompare(const WordKey& expected, const WordKey& actual, int proximity)
|
|
{
|
|
int ret = 1;
|
|
if((ret = DocumentCompare(expected, actual)) != 0) return ret;
|
|
//
|
|
// Only compare location if defined.
|
|
//
|
|
if((expected.IsDefined(location) && actual.IsDefined(location)) &&
|
|
(ret = expected.Get(location) - actual.Get(location))) {
|
|
if(proximity < 0) {
|
|
//
|
|
// -N means ok if in range [-N +N]
|
|
//
|
|
proximity *= 2;
|
|
if(ret < 0 && ret >= proximity)
|
|
ret = 0;
|
|
} else {
|
|
//
|
|
// N means ok if in range [0 +N]
|
|
//
|
|
if(ret < 0 && ret >= -proximity)
|
|
ret = 0;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
void WordKeySemantic::LocationNext(WordKey& key)
|
|
{
|
|
key.SetToFollowing(location);
|
|
}
|
|
|
|
void WordKeySemantic::LocationNearLowest(WordKey& key, int proximity)
|
|
{
|
|
if(proximity < 0) {
|
|
if(key.Underflow(location, proximity))
|
|
key.Get(location) = 0;
|
|
else
|
|
key.Get(location) += proximity;
|
|
}
|
|
}
|
|
|
|
void WordKeySemantic::Location2Document(WordKey& key)
|
|
{
|
|
key.Undefined(location);
|
|
}
|
|
|
|
// ************************* WordExclude implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// permute bits in bit field
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordExclude.h>
|
|
//
|
|
// #define BITS 5
|
|
//
|
|
// WordExclude permute;
|
|
// permute.Initialize(BITS);
|
|
// while(permute.Next() == WORD_EXCLUDE_OK)
|
|
// ...
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// Count from 1 to the specified maximum. A variable++ loop does the same.
|
|
// The <b>WordExclude</b> class counts in a specific order.
|
|
// It first step thru all the permutations containing only 1 bit set, in
|
|
// increasing order. Then thru all the permutations containing 2 bits set,
|
|
// in increasing order. As so forth until the maximum number is reached.
|
|
// See the <b>Permute</b> method for more information.
|
|
//
|
|
//
|
|
// END
|
|
|
|
//
|
|
// Helper that displays an unsigned int in binary/hexa/decimal
|
|
//
|
|
static inline void show_bits(unsigned int result)
|
|
{
|
|
int i;
|
|
for(i = 0; i < 10; i++) {
|
|
fprintf(stderr, "%c", (result & (1 << i)) ? '1' : '0');
|
|
}
|
|
fprintf(stderr, " (0x%08x - %15d)\n", result, result);
|
|
}
|
|
|
|
//
|
|
// WordExclude methods return values
|
|
//
|
|
#define WORD_EXCLUDE_OK 1
|
|
#define WORD_EXCLUDE_END 2
|
|
|
|
//
|
|
// Maximum number of bits
|
|
//
|
|
#define WORD_EXCLUDE_MAX (sizeof(unsigned int) * 8)
|
|
|
|
//
|
|
// Convert a position <p> in a <l> bits mask into a bit offset (from 0)
|
|
//
|
|
#define WORD_EXCLUDE_POSITION2BIT(l,p) ((l) - (p) - 1)
|
|
|
|
class WordExclude {
|
|
public:
|
|
//-
|
|
// Reset the generator and prepare it for <b>length</b> bits generation.
|
|
// The <b>length</b> cannot be greater than <i>WORD_EXCLUDE_MAX.</i>
|
|
// Returns OK if no error occurs, NOTOK otherwise.
|
|
//
|
|
virtual int Initialize(unsigned int length);
|
|
//-
|
|
// Move to next exclude mask. Returns WORD_EXCLUDE_OK if successfull,
|
|
// WORD_EXCLUDE_END if at the end of the permutations. It starts by
|
|
// calling <i>Permute</i> with one bit set, then two and up to
|
|
// <i>Maxi()</i> included. The last permutation only generates one
|
|
// possibility since all the bits are set.
|
|
//
|
|
virtual int Next();
|
|
//-
|
|
// Exclude bit for <b>position</b> starts at most significant bit. That is
|
|
// position 0 exclude bit is most significant bit of the current mask.
|
|
// Returns true if position is excluded, false otherwise.
|
|
//
|
|
virtual inline unsigned int Excluded(int position) { return mask & (1 << WORD_EXCLUDE_POSITION2BIT(maxi, position)); }
|
|
//-
|
|
// Returns how many bits are not excluded with current mask.
|
|
//
|
|
virtual inline int NotExcludedCount() const { return maxi - bits; }
|
|
//-
|
|
// Returns how many bits are excluded with current mask.
|
|
//
|
|
virtual inline int ExcludedCount() const { return bits; }
|
|
//
|
|
// Save and restore in string
|
|
//
|
|
//-
|
|
// Write an ascii representation of the WordExclude object in <b>buffer.</b>
|
|
// Each bit is represented by the character 0 or 1. The most significant
|
|
// bit is the last character in the string. For instance
|
|
// 1000 is the string representation of a WordExclude object initialized
|
|
// with length = 4 after the first <i>Next</i> operation.
|
|
//
|
|
virtual void Get(String& buffer) const;
|
|
//-
|
|
// Initialize the object from the string representation in <b>buffer.</b>
|
|
// Returns OK on success, NOTOK on failure.
|
|
//
|
|
virtual int Set(const String& buffer);
|
|
|
|
//-
|
|
// Generate all the permutations
|
|
// containing <i>n</i> bits in a <b>bits</b> bit word in increasing order.
|
|
// The <b>mask</b> argument is originally filled by the caller
|
|
// with the <i>n</i> least significant bits set. A call to Permute
|
|
// generates the next permutation immediately greater (numerically)
|
|
// than the one contained in <b>mask</b>.
|
|
//
|
|
// Permute returns the next permutation or 0 if it reached the
|
|
// maximum.
|
|
//
|
|
// To understand the algorithm, imagine 1 is a ball and 0 a space.
|
|
//
|
|
// When playing the game you start with a rack of <b>bits</b> slots filled
|
|
// with <i>n</i> balls all on the left side. You end the game when all
|
|
// the balls are on the right side.
|
|
//
|
|
// Sarting from the left, search for the first ball that has an empty
|
|
// space to the right. While searching remove all the balls you find.
|
|
// Place a ball in the empty space you found, at the right of the last
|
|
// ball removed. Sarting from the left, fill all empty spaces with
|
|
// the removed balls. Repeat until all balls are to the right.
|
|
//
|
|
// Here is a sample generated by repeated calls to WordExclude::Permute:
|
|
// (left most bit is least significant)
|
|
// <pre>
|
|
// mask = 1111100000
|
|
// while(mask = WordExclude::Permute(mask, 7))
|
|
// show_bits(mask)
|
|
//
|
|
// 1111100000 (0x0000001f - 31)
|
|
// 1111010000 (0x0000002f - 47)
|
|
// 1110110000 (0x00000037 - 55)
|
|
// 1101110000 (0x0000003b - 59)
|
|
// 1011110000 (0x0000003d - 61)
|
|
// 0111110000 (0x0000003e - 62)
|
|
// 1111001000 (0x0000004f - 79)
|
|
// 1110101000 (0x00000057 - 87)
|
|
// 1101101000 (0x0000005b - 91)
|
|
// 1011101000 (0x0000005d - 93)
|
|
// 0111101000 (0x0000005e - 94)
|
|
// 1110011000 (0x00000067 - 103)
|
|
// 1101011000 (0x0000006b - 107)
|
|
// 1011011000 (0x0000006d - 109)
|
|
// 0111011000 (0x0000006e - 110)
|
|
// 1100111000 (0x00000073 - 115)
|
|
// 1010111000 (0x00000075 - 117)
|
|
// 0110111000 (0x00000076 - 118)
|
|
// 1001111000 (0x00000079 - 121)
|
|
// 0101111000 (0x0000007a - 122)
|
|
// 0011111000 (0x0000007c - 124)
|
|
// </pre>
|
|
// A recursive implementation would be:
|
|
// <pre>
|
|
// /* Recursive */
|
|
// void permute(unsigned int result, int bits_count, int bits_toset)
|
|
// {
|
|
// if(bits_toset <= 0 || bits_count <= 0) {
|
|
// if(bits_toset <= 0)
|
|
// do_something(result);
|
|
// } else {
|
|
// permute(result, bits_count - 1, bits_toset);
|
|
// permute(result | (1 << (bits_count - 1)), bits_count - 1, bits_toset - 1);
|
|
// }
|
|
// }
|
|
// </pre>
|
|
// Which is more elegant but not practical at all in our case.
|
|
//
|
|
inline unsigned int Permute(unsigned int mask, unsigned int bits);
|
|
|
|
//-
|
|
// Return the current bit field value.
|
|
//
|
|
virtual inline unsigned int& Mask() { return mask; }
|
|
virtual inline unsigned int Mask() const { return mask; }
|
|
|
|
virtual inline unsigned int& Maxi() { return maxi; }
|
|
virtual inline unsigned int Maxi() const { return maxi; }
|
|
|
|
virtual inline unsigned int& Bits() { return bits; }
|
|
virtual inline unsigned int Bits() const { return bits; }
|
|
|
|
private:
|
|
unsigned int mask;
|
|
unsigned int maxi;
|
|
unsigned int bits;
|
|
};
|
|
|
|
int WordExclude::Initialize(unsigned int length)
|
|
{
|
|
if(length > WORD_EXCLUDE_MAX) {
|
|
fprintf(stderr, "WordExclude::Initialize: length must be < %d\n", (int)WORD_EXCLUDE_MAX);
|
|
return NOTOK;
|
|
}
|
|
|
|
mask = 0;
|
|
bits = 0;
|
|
maxi = length;
|
|
|
|
return OK;
|
|
}
|
|
|
|
inline unsigned int WordExclude::Permute(unsigned int mask, unsigned int bits)
|
|
{
|
|
unsigned int bits_cleared = 0;
|
|
unsigned int j;
|
|
for(j = 0; j < bits; j++) {
|
|
if(mask & (1 << j)) {
|
|
bits_cleared++;
|
|
mask &= ~(1 << j);
|
|
} else {
|
|
if(bits_cleared) {
|
|
bits_cleared--;
|
|
mask |= (1 << j);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if(j >= bits)
|
|
return 0;
|
|
|
|
for(j = 0; j < bits_cleared; j++)
|
|
mask |= (1 << j);
|
|
|
|
return mask;
|
|
}
|
|
|
|
int WordExclude::Next()
|
|
{
|
|
mask = Permute(mask, maxi);
|
|
|
|
int ret = WORD_EXCLUDE_OK;
|
|
|
|
if(mask == 0) {
|
|
bits++;
|
|
if(bits > maxi)
|
|
ret = WORD_EXCLUDE_END;
|
|
else {
|
|
unsigned int i;
|
|
for(i = 0; i < bits; i++)
|
|
mask |= (1 << i);
|
|
ret = WORD_EXCLUDE_OK;
|
|
}
|
|
}
|
|
|
|
if(verbose > 2) show_bits(mask);
|
|
|
|
return ret;
|
|
}
|
|
|
|
void WordExclude::Get(String& buffer) const
|
|
{
|
|
buffer.trunc();
|
|
unsigned int i;
|
|
for(i = 0; i < maxi; i++) {
|
|
buffer << ((mask & (1 << i)) ? '1' : '0');
|
|
}
|
|
}
|
|
|
|
int WordExclude::Set(const String& buffer)
|
|
{
|
|
if(Initialize(buffer.length()) == NOTOK)
|
|
return NOTOK;
|
|
unsigned int i;
|
|
for(i = 0; i < maxi; i++) {
|
|
if(buffer[i] == '1') {
|
|
mask |= (1 << i);
|
|
bits++;
|
|
}
|
|
}
|
|
return OK;
|
|
}
|
|
|
|
// ************************* WordExcludeMask implementation *******************
|
|
//
|
|
// NAME
|
|
//
|
|
// WordExclude specialization that ignore some bits
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordExcludeMask.h>
|
|
//
|
|
// #define BITS 9
|
|
// #define IGNORE 0x0f0
|
|
// #define IGNORE_MASK 0x050
|
|
//
|
|
// WordExcludeMask permute;
|
|
// permute.Initialize(BITS, IGNORE, IGNORE_MASK);
|
|
// while(permute.Next() == WORD_EXCLUDE_OK)
|
|
// ...
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// Only perform WordExclude operations on the bits that are not set in
|
|
// <i>ignore.</i> The bits of <i>ignore_mask</i> that are set in
|
|
// <i>ignore</i> are untouched. In the synopsis section, for instance,
|
|
// bits 1,2,3,4 and 9 will be permuted and the bits 5,6,7,8 will be
|
|
// left untouched.
|
|
//
|
|
//
|
|
// END
|
|
//
|
|
#define WORD_EXCLUDE_IGNORED (-1)
|
|
|
|
class WordExcludeMask : public WordExclude {
|
|
public:
|
|
//-
|
|
// <b>ignore</b> gives the mask of bits to ignore. The actual WordExclude
|
|
// operations are made on a number of bits that is <b>length</b> - (the number
|
|
// of bits set in <b>ignore).</b>
|
|
// The <b>ignore_mask_arg</b> contains the actual values of the bits ignored by
|
|
// the <b>ignore</b> argument.
|
|
//
|
|
virtual inline int Initialize(unsigned int length, unsigned int ignore, unsigned int ignore_mask_arg) {
|
|
ignore_mask = ignore_mask_arg;
|
|
ignore_maxi = length;
|
|
unsigned int maxi = 0;
|
|
unsigned int i;
|
|
for(i = 0, ignore_bits = 0; i < length; i++) {
|
|
if(ignore & (1 << i)) {
|
|
bit2bit[i] = WORD_EXCLUDE_IGNORED;
|
|
if(ignore_mask & (1 << i)) ignore_bits++;
|
|
} else {
|
|
bit2bit[i] = maxi++;
|
|
}
|
|
}
|
|
|
|
return WordExclude::Initialize(maxi);
|
|
}
|
|
|
|
virtual inline unsigned int Excluded(int position) {
|
|
position = WORD_EXCLUDE_POSITION2BIT(ignore_maxi, position);
|
|
if(bit2bit[position] == WORD_EXCLUDE_IGNORED)
|
|
return ignore_mask & (1 << position);
|
|
else
|
|
return WordExclude::Mask() & (1 << bit2bit[position]);
|
|
}
|
|
|
|
virtual inline int NotExcludedCount() const {
|
|
return ignore_maxi - ignore_bits - WordExclude::Bits();
|
|
}
|
|
|
|
virtual inline int ExcludedCount() const {
|
|
return ignore_bits - WordExclude::Bits();
|
|
}
|
|
|
|
//-
|
|
// The semantic is the same as the Get method of Wordexclude
|
|
// except that ignored bits are assigned 3 and 2 instead of 1 and 0
|
|
// respectively.
|
|
//
|
|
virtual void Get(String& buffer) const;
|
|
//-
|
|
// The semantic is the same as the Get method of Wordexclude
|
|
// except that ignored bits are assigned 3 and 2 instead of 1 and 0
|
|
// respectively.
|
|
//
|
|
virtual int Set(const String& buffer);
|
|
|
|
virtual inline unsigned int Mask() const {
|
|
unsigned int ret = ignore_mask;
|
|
unsigned int i;
|
|
for(i = 0; i < ignore_maxi; i++) {
|
|
if(bit2bit[i] != WORD_EXCLUDE_IGNORED) {
|
|
if(WordExclude::Mask() & (1 << bit2bit[i]))
|
|
ret |= (1 << i);
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
virtual inline unsigned int Maxi() const { return ignore_maxi; }
|
|
|
|
virtual inline unsigned int Bits() const { return ignore_bits + WordExclude::Bits(); }
|
|
|
|
private:
|
|
unsigned int ignore_mask;
|
|
unsigned int ignore_maxi;
|
|
unsigned int ignore_bits;
|
|
int bit2bit[WORD_EXCLUDE_MAX];
|
|
};
|
|
|
|
void WordExcludeMask::Get(String& buffer) const
|
|
{
|
|
buffer.trunc();
|
|
unsigned int i;
|
|
for(i = 0; i < ignore_maxi; i++) {
|
|
if(bit2bit[i] == WORD_EXCLUDE_IGNORED)
|
|
buffer << ((ignore_mask & (1 << i)) ? '3' : '2');
|
|
else
|
|
buffer << ((WordExclude::Mask() & (1 << bit2bit[i])) ? '1' : '0');
|
|
}
|
|
}
|
|
|
|
int WordExcludeMask::Set(const String& buffer)
|
|
{
|
|
WordExclude::Initialize(0);
|
|
|
|
unsigned int& maxi = WordExclude::Maxi();
|
|
unsigned int& mask = WordExclude::Mask();
|
|
unsigned int& bits = WordExclude::Bits();
|
|
ignore_mask = 0;
|
|
ignore_bits = 0;
|
|
ignore_maxi = buffer.length();
|
|
|
|
unsigned int i;
|
|
for(i = 0; i < ignore_maxi; i++) {
|
|
if(buffer[i] == '1' || buffer[i] == '0') {
|
|
if(buffer[i] == '1') {
|
|
mask |= (1 << maxi);
|
|
bits++;
|
|
}
|
|
bit2bit[i] = maxi;
|
|
maxi++;
|
|
} else if(buffer[i] == '3' || buffer[i] == '2') {
|
|
if(buffer[i] == '3') {
|
|
ignore_mask |= (1 << i);
|
|
ignore_bits++;
|
|
}
|
|
bit2bit[i] = WORD_EXCLUDE_IGNORED;
|
|
}
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
// ************************* WordPermute implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// WordExclude specialization with proximity toggle
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordPermute.h>
|
|
//
|
|
// #define BITS 5
|
|
//
|
|
// WordPermute permute;
|
|
// permute.Initialize(BITS);
|
|
// while(permute.Next() == WORD_EXCLUDE_OK)
|
|
// if(permute.UseProximity()) ...
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// Each WordExclude permutation is used twice by Next. Once with
|
|
// the proximity flag set and once with the proximity flag cleared.
|
|
// If the length of the bit field (length argument of Initialize) is
|
|
// lower or equal to 1, then the proximity flag is always false.
|
|
//
|
|
//
|
|
// END
|
|
//
|
|
// WordPermute methods return values
|
|
//
|
|
#define WORD_PERMUTE_OK WORD_EXCLUDE_OK
|
|
#define WORD_PERMUTE_END WORD_EXCLUDE_END
|
|
|
|
//
|
|
// Use or don't use proximity flag
|
|
//
|
|
#define WORD_PERMUTE_PROXIMITY_NO 0
|
|
#define WORD_PERMUTE_PROXIMITY_TOGGLE 1
|
|
#define WORD_PERMUTE_PROXIMITY_ONLY 2
|
|
|
|
//
|
|
// Deals with word exclusion and proximity permutations for
|
|
// the implementation of the Optional retrieval model.
|
|
//
|
|
class WordPermute : public WordExcludeMask {
|
|
public:
|
|
//-
|
|
// The <b>nuse_proximity</b> may be set to the following:
|
|
//
|
|
// WORD_PERMUTE_PROXIMITY_NO so that the object behaves as
|
|
// WordExcludeMask and Proximity() always return false.
|
|
//
|
|
// WORD_PERMUTE_PROXIMITY_TOGGLE so that each permutation is issued twice:
|
|
// once with the proximity flag set (Proximity() method) and once with
|
|
// the proximity flag cleared.
|
|
//
|
|
// WORD_PERMUTE_PROXIMITY_ONLY so that the object behaves as
|
|
// WordExcludeMask and Proximity() always return true.
|
|
//
|
|
virtual inline int Initialize(unsigned int length, unsigned int ignore, unsigned int ignore_mask_arg, int nuse_proximity) {
|
|
use_proximity = nuse_proximity;
|
|
switch(use_proximity) {
|
|
case WORD_PERMUTE_PROXIMITY_NO:
|
|
proximity = 0;
|
|
break;
|
|
case WORD_PERMUTE_PROXIMITY_TOGGLE:
|
|
//
|
|
// Don't bother to try proximity search if only one word
|
|
// is involved.
|
|
//
|
|
proximity = length > 1;
|
|
break;
|
|
case WORD_PERMUTE_PROXIMITY_ONLY:
|
|
proximity = 1;
|
|
break;
|
|
default:
|
|
fprintf(stderr, "WordPermute::Initialize: unexpected use_proximity = %d\n", use_proximity);
|
|
return 0;
|
|
}
|
|
return WordExcludeMask::Initialize(length, ignore, ignore_mask_arg);
|
|
}
|
|
|
|
//-
|
|
// Return true if the proximity flag is set, false if it is
|
|
// cleared.
|
|
//
|
|
inline int Proximity() {
|
|
switch(use_proximity) {
|
|
case WORD_PERMUTE_PROXIMITY_NO:
|
|
return 0;
|
|
break;
|
|
case WORD_PERMUTE_PROXIMITY_TOGGLE:
|
|
return proximity;
|
|
break;
|
|
case WORD_PERMUTE_PROXIMITY_ONLY:
|
|
return 1;
|
|
break;
|
|
default:
|
|
fprintf(stderr, "WordPermute::Proximity: unexpected use_proximity = %d\n", use_proximity);
|
|
return 0;
|
|
break;
|
|
}
|
|
}
|
|
|
|
//-
|
|
// Return WORD_PERMUTE_PROXIMITY_NO, WORD_PERMUTE_PROXIMITY_TOGGLE or
|
|
// WORD_PERMUTE_PROXIMITY_ONLY.
|
|
//
|
|
inline int UseProximity() { return use_proximity; }
|
|
|
|
//-
|
|
// Find the next permutation. If <b>WORD_PERMUTE_PROXIMITY_TOGGLE<b> was
|
|
// specified in Initialize each permutation is issued twice (see
|
|
// Proximity() to differentiate them), except when the mask
|
|
// only contains one non exluded bit (NotExcludeCount() <= 1).
|
|
// In both case the last permutation with all bits excluded
|
|
// (i.e. when NotExcludedCount() <= 0) is never returned because
|
|
// it is useless.
|
|
//
|
|
virtual int Next() {
|
|
if(Maxi() <= 0)
|
|
return WORD_PERMUTE_END;
|
|
|
|
int ret = WORD_PERMUTE_OK;
|
|
int check_useless = 0;
|
|
if(use_proximity == WORD_PERMUTE_PROXIMITY_TOGGLE) {
|
|
//
|
|
// Move to next permutation as follows:
|
|
// exclude mask 1 + use proximity
|
|
// exclude mask 1 + don't use proximity
|
|
// exclude mask 2 + use proximity
|
|
// exclude mask 2 + don't use proximity
|
|
// and so on.
|
|
// If only one word is involved never use proximity.
|
|
//
|
|
if(proximity) {
|
|
proximity = 0;
|
|
} else {
|
|
proximity = 1;
|
|
if((ret = WordExcludeMask::Next()) == WORD_PERMUTE_OK) {
|
|
//
|
|
// Do not toggle proximity for only one non excluded word
|
|
//
|
|
if(NotExcludedCount() <= 1)
|
|
proximity = 0;
|
|
check_useless = 1;
|
|
} else if(ret == WORD_PERMUTE_END)
|
|
proximity = 0;
|
|
}
|
|
} else {
|
|
ret = WordExcludeMask::Next();
|
|
check_useless = 1;
|
|
}
|
|
|
|
if(check_useless && ret == WORD_PERMUTE_OK) {
|
|
//
|
|
// If no bits are ignored or all ignore_mask bits are set to
|
|
// one, the last permutation has all exclude bits set, which
|
|
// is useless. Just skip it and expect to be at the end of
|
|
// all permutations.
|
|
//
|
|
if(NotExcludedCount() <= 0) {
|
|
ret = WordExcludeMask::Next();
|
|
if(ret != WORD_PERMUTE_END) {
|
|
fprintf(stderr, "WordPermute::Next: expected WORD_PERMUTE_END\n");
|
|
ret = NOTOK;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
//-
|
|
// The semantic is the same as the Get method of Wordexclude
|
|
// but a letter T is appended to the string if the proximity
|
|
// flag is set, or F is appended to the string if the proximity
|
|
// is clear.
|
|
//
|
|
virtual inline void Get(String& buffer) const {
|
|
WordExcludeMask::Get(buffer);
|
|
if(use_proximity == WORD_PERMUTE_PROXIMITY_TOGGLE)
|
|
buffer << (proximity ? 'T' : 'F');
|
|
}
|
|
|
|
//-
|
|
// The semantic is the same as the Get method of Wordexclude
|
|
// but if the string end with a T the proximity flag is set
|
|
// and if the string end with a F the proximity flag is cleared.
|
|
//
|
|
virtual inline int Set(const String& buffer) {
|
|
if(buffer.length() < 1) {
|
|
fprintf(stderr, "WordPermute::Set: buffer length < 1\n");
|
|
return NOTOK;
|
|
}
|
|
int ret = OK;
|
|
if(use_proximity == WORD_PERMUTE_PROXIMITY_TOGGLE) {
|
|
if((ret = WordExcludeMask::Set(buffer.sub(0, buffer.length() - 1))) == OK)
|
|
proximity = buffer.last() == 'T';
|
|
} else {
|
|
ret = WordExcludeMask::Set(buffer);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
protected:
|
|
int use_proximity;
|
|
int proximity;
|
|
};
|
|
|
|
// ************************* WordTree implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// Base class for query resolution nodes
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordTree.h>
|
|
//
|
|
// class WordTreeMethod : public WordTree {
|
|
// ...
|
|
// };
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// The WordTree class is derived from the WordCursor class and implement
|
|
// the basic operations and data structures needed for query resolution.
|
|
// It is the common base class of all the classes that actually implement
|
|
// a query resolution. The derived classes must be implemented to follow
|
|
// the WordCursor semantic for Walk* operations.
|
|
//
|
|
//
|
|
// END
|
|
//
|
|
|
|
#define WORD_WALK_REDO 0x1000
|
|
#define WORD_WALK_RESTART 0x2000
|
|
#define WORD_WALK_NEXT 0x4000
|
|
|
|
//
|
|
// Return values of CursorsObeyProximity method
|
|
//
|
|
#define WORD_SEARCH_NOPROXIMITY 1
|
|
|
|
//
|
|
// operand values
|
|
//
|
|
#define WORD_TREE_OR 1
|
|
#define WORD_TREE_AND 2
|
|
#define WORD_TREE_NEAR 3
|
|
#define WORD_TREE_OPTIONAL 4
|
|
#define WORD_TREE_LITERAL 5
|
|
#define WORD_TREE_MANDATORY 6
|
|
#define WORD_TREE_NOT 7
|
|
|
|
#define WORD_TREE_OP_SIZE 20
|
|
|
|
//
|
|
// Default proximity is to search for adjacent words in order
|
|
//
|
|
#ifndef WORD_SEARCH_DEFAULT_PROXIMITY
|
|
#define WORD_SEARCH_DEFAULT_PROXIMITY 1
|
|
#endif /* WORD_SEARCH_DEFAULT_PROXIMITY */
|
|
|
|
static char* operator_name[WORD_TREE_OP_SIZE] = {
|
|
"",
|
|
"or",
|
|
"and",
|
|
"near",
|
|
"optional",
|
|
"literal",
|
|
"mandatory",
|
|
"not",
|
|
0
|
|
};
|
|
|
|
class WordTree : public WordCursor {
|
|
public:
|
|
WordTree() {
|
|
proximity = 0;
|
|
uniq = 0;
|
|
}
|
|
|
|
virtual int ContextSaveList(StringList& list) const {
|
|
return OK;
|
|
}
|
|
|
|
virtual int ContextRestoreList(StringList& list) {
|
|
return OK;
|
|
}
|
|
|
|
//-
|
|
// Initialize the object. <b>words</b> is used to initialize the
|
|
// WordCursor base class, <b>document, document_length</b> and
|
|
// <b>location</b> are used to initialize the WordKeySemantic data
|
|
// member. The <b>nuniq</b> is the WordKey field position used by
|
|
// the WordKeySemantic::DocumentNext function. The <b>nproximity</b>
|
|
// is the proximity factor used by the WordKeySemantic::LocationCompare
|
|
// method.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
virtual int Prepare(WordList *words, int nuniq, int nproximity, int *document, int document_length, int location) {
|
|
int ret;
|
|
proximity = nproximity;
|
|
uniq = nuniq;
|
|
if((ret = key_semantic.Initialize(document, document_length, location)) != OK)
|
|
return ret;
|
|
WordKey key;
|
|
if(!scope.empty()) {
|
|
if(key.Set(scope) != OK) {
|
|
fprintf(stderr, "WordTree::Prepare: setting scope %s failed\n", (char*)scope);
|
|
return NOTOK;
|
|
}
|
|
}
|
|
key.SetWord(search);
|
|
return WordCursor::Initialize(words, key, 0, 0, HTDIG_WORDLIST_WALKER);
|
|
}
|
|
|
|
//-
|
|
// Return a copy of the last document found.
|
|
//
|
|
WordKey GetDocument() {
|
|
WordKey found;
|
|
key_semantic.DocumentSet(GetFound().Key(), found);
|
|
return found;
|
|
}
|
|
|
|
//-
|
|
// Store in the <i>info</i> data member textual information about
|
|
// the latest match found.
|
|
//
|
|
virtual void SetInfo() { info = GetFound().Key().GetWord(); }
|
|
|
|
//-
|
|
// Return a copy of the <i>info</i> data member. Should be
|
|
// called after SetInfo().
|
|
//
|
|
String GetInfo() { return info; }
|
|
|
|
//-
|
|
// Sort WordTree data members (if any) in ascending frequency order.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
virtual int AscendingFrequency() { return OK; }
|
|
|
|
//-
|
|
// Delete WordTree data members (if any) that have a zero frequency.
|
|
// The number of data members deleted is returned in <b>stripped</b>.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
virtual int StripNonExistent(unsigned int& stripped) {
|
|
stripped = 0;
|
|
return OK;
|
|
}
|
|
|
|
//
|
|
// Input
|
|
//
|
|
//-
|
|
// Proximity factor. See WordKeySemantic::LocationCompare.
|
|
//
|
|
int proximity;
|
|
//-
|
|
// Uniq WordKey field position. See WordKeySemantic::DocumentNext.
|
|
//
|
|
int uniq;
|
|
//-
|
|
// Semantic of the WordKey object.
|
|
//
|
|
WordKeySemantic key_semantic;
|
|
//-
|
|
// Textual representation of the search scope.
|
|
//
|
|
String scope;
|
|
//-
|
|
// Original search criterion that may be different from the
|
|
// WordCursor::searchKey data member.
|
|
//
|
|
String search;
|
|
|
|
//
|
|
// Internal state
|
|
//
|
|
//-
|
|
// Textual information about the latest match.
|
|
//
|
|
String info;
|
|
};
|
|
|
|
// ************************* WordTreeLiteral implementation ****************
|
|
|
|
class WordTreeLiteral : public WordTree {
|
|
public:
|
|
//-
|
|
// Constructor. The search criterion is <b>string</b> and the
|
|
// scope is <b>nscope.</b>.
|
|
//
|
|
WordTreeLiteral(const char* string, const char* nscope = "") {
|
|
search.set((char*)string);
|
|
scope.set((char*)nscope);
|
|
}
|
|
|
|
//-
|
|
// Returns WORD_TREE_LITERAL.
|
|
//
|
|
int IsA() const { return WORD_TREE_LITERAL; }
|
|
|
|
virtual int WalkRewind();
|
|
//-
|
|
// Only return a match for each distinct document.
|
|
//
|
|
virtual int WalkNext();
|
|
virtual int Seek(const WordKey& patch);
|
|
|
|
//-
|
|
// If scope is set the <b>bufferout</b> is filled with
|
|
// <pre>
|
|
// ( word "scope" )
|
|
// </pre>
|
|
// otherwise the <b>bufferout</b> only contains the word.
|
|
//
|
|
virtual int Get(String& bufferout) const {
|
|
if(scope.empty())
|
|
bufferout << search;
|
|
else
|
|
bufferout << "( " << operator_name[IsA()] << " \"" << scope << "\" " << search << " )";
|
|
return OK;
|
|
}
|
|
|
|
protected:
|
|
WordKey current_document;
|
|
};
|
|
|
|
int WordTreeLiteral::WalkRewind()
|
|
{
|
|
current_document.Clear();
|
|
return WordCursor::WalkRewind();
|
|
}
|
|
|
|
int WordTreeLiteral::WalkNext()
|
|
{
|
|
int ret;
|
|
do {
|
|
ret = WordCursor::WalkNext();
|
|
if(verbose > 3) fprintf(stderr, "WordTreeLiteral::WalkNext: reached %s\n", (char*)GetDocument().Get());
|
|
} while(ret == OK &&
|
|
key_semantic.DocumentCompare(current_document, GetDocument()) == 0);
|
|
|
|
if(ret == OK)
|
|
current_document = GetDocument();
|
|
else
|
|
current_document.Clear();
|
|
|
|
return ret;
|
|
}
|
|
|
|
int WordTreeLiteral::Seek(const WordKey& position)
|
|
{
|
|
current_document.Clear();
|
|
return WordCursor::Seek(position);
|
|
}
|
|
|
|
// ************************* WordTreeOperand implementation ****************
|
|
//
|
|
// NAME
|
|
//
|
|
// Base class for boolean query resolution nodes
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordTree.h>
|
|
//
|
|
// class WordTreeMethod : public WordTreeOperand {
|
|
// ...
|
|
// };
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// The WordTreeOperand class is derived from WordTree and implemet
|
|
// the basic operations and data structures needed for query resultion
|
|
// of boolean operators. It contains a list of WordTree objects (the
|
|
// operands or cursors) and redefine the basic WordCursor methods
|
|
// to operate on all of them according to the logic defined by the
|
|
// derived class.
|
|
//
|
|
//
|
|
// END
|
|
//
|
|
|
|
//
|
|
// Helper for debugging that returns the string representation
|
|
// of the return codes.
|
|
//
|
|
static char* ret2str(int ret)
|
|
{
|
|
if(ret == WORD_WALK_REDO)
|
|
return "REDO";
|
|
|
|
if(ret == WORD_WALK_RESTART)
|
|
return "RESTART";
|
|
|
|
if(ret == WORD_WALK_NEXT)
|
|
return "NEXT";
|
|
|
|
if(ret == OK)
|
|
return "OK";
|
|
|
|
if(ret == NOTOK)
|
|
return "NOTOK";
|
|
|
|
if(ret == WORD_WALK_ATEND)
|
|
return "ATEND";
|
|
|
|
return "???";
|
|
}
|
|
|
|
class WordTreeOperand : public WordTree
|
|
{
|
|
public:
|
|
//-
|
|
// Constructor. The scope is <b>nscope</b>.
|
|
//
|
|
WordTreeOperand(const char* nscope) {
|
|
scope.set((char*)nscope);
|
|
}
|
|
//-
|
|
// Free the objects pointed by <i>cursors</i> with delete as well
|
|
// as the <i>cursors</i> array itself with delete [].
|
|
//
|
|
virtual ~WordTreeOperand();
|
|
|
|
virtual void Clear() {
|
|
cursors = 0;
|
|
cursors_length = 0;
|
|
WordCursor::Clear();
|
|
}
|
|
|
|
//-
|
|
// Recursively call Optimize on each <i>cursors</i>.
|
|
//
|
|
virtual int Optimize();
|
|
|
|
//-
|
|
// Change the <i>permutation</i> data member ignore mask according
|
|
// to WORD_TREE_MANDATORY and WORD_TREE_NOT nodes found in
|
|
// <i>cursors</i>. MANDATORY and NOT nodes are reduced (replaced
|
|
// by their first child cursor. For each MANDATORY and NOT nodes
|
|
// the bit (see WordExcludeMask for information)
|
|
// corresponding to their position is ignored (set in the <b>ignore</b>
|
|
// argument of the WordExcludeMask::Initialize function. For NOT
|
|
// nodes, the bit corresponding to their position is set in
|
|
// the <b>ignore_mask</b> of the WordExcludeMask::Initialize function
|
|
// (i.e. implementing a <i>not</i> operation).
|
|
// The <b>proximity</b> argument may be WORD_PERMUTE_PROXIMITY_TOGGLE or
|
|
// WORD_PERMUTE_PROXIMITY_NO.
|
|
// Returns OK on success, NOTOK on failure.
|
|
//
|
|
int OptimizeOr(int proximity);
|
|
|
|
virtual int ContextSave(String& buffer) const {
|
|
StringList list;
|
|
int ret;
|
|
if((ret = ContextSaveList(list)) != OK)
|
|
return ret;
|
|
|
|
buffer.trunc();
|
|
String* element;
|
|
list.Start_Get();
|
|
while((element = (String*)list.Get_Next())) {
|
|
buffer << (*element) << ';';
|
|
}
|
|
//
|
|
// Trim last ;
|
|
//
|
|
buffer.chop(1);
|
|
|
|
return OK;
|
|
}
|
|
|
|
virtual int ContextSaveList(StringList& list) const {
|
|
//
|
|
// Apply to each cursor
|
|
//
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if(cursors[i]->ContextSaveList(list) == NOTOK)
|
|
return NOTOK;
|
|
return OK;
|
|
}
|
|
|
|
virtual int ContextRestore(const String& buffer) {
|
|
if(!buffer.empty()) {
|
|
StringList list(buffer, ";");
|
|
return ContextRestoreList(list);
|
|
} else {
|
|
return OK;
|
|
}
|
|
}
|
|
|
|
virtual int ContextRestoreList(StringList& list) {
|
|
//
|
|
// Apply to each cursor
|
|
//
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if(cursors[i]->ContextRestoreList(list) == NOTOK)
|
|
return NOTOK;
|
|
return OK;
|
|
}
|
|
|
|
//-
|
|
// Recursively call WalkInit on each <i>cursors</i>.
|
|
//
|
|
virtual int WalkInit();
|
|
//-
|
|
// Recursively call WalkRewind on each <i>cursors</i>.
|
|
// Reset the <i>pos</i> data member with WordKeySemantic::DocumentClear.
|
|
//
|
|
virtual int WalkRewind();
|
|
//-
|
|
// Recursively call WalkFinish on each <i>cursors</i>.
|
|
//
|
|
virtual int WalkFinish();
|
|
//-
|
|
// Recursively call Seek on each <i>cursors</i>.
|
|
// Save the <b>patch</b> argument in the <i>pos</i> data
|
|
// member.
|
|
//
|
|
virtual int Seek(const WordKey& patch);
|
|
|
|
//-
|
|
// The number of occurrence of a WordTreeOperand is the sum of the
|
|
// number of occurrence of each term.
|
|
//
|
|
virtual int Noccurrence(unsigned int& noccurrence) const {
|
|
noccurrence = 0;
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++) {
|
|
unsigned int frequency;
|
|
if(cursors[i]->Noccurrence(frequency) != OK)
|
|
return NOTOK;
|
|
noccurrence += frequency;
|
|
}
|
|
return OK;
|
|
}
|
|
|
|
//-
|
|
// The <b>bufferout</b> argument is filled with a lisp like representation
|
|
// of the tree starting at this node.
|
|
//
|
|
virtual int Get(String& bufferout) const {
|
|
bufferout << "( " << operator_name[IsA()] << " \"" << scope << "\" ";
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
bufferout << cursors[i]->Get() << " ";
|
|
bufferout << " )";
|
|
return OK;
|
|
}
|
|
|
|
//-
|
|
// Call Prepare on each <i>cursors</i>. Set the <i>search</i> member
|
|
// with an textual representation of the tree starting at this node.
|
|
//
|
|
virtual int Prepare(WordList *words, int nuniq, int nproximity, int *document, int document_length, int location) {
|
|
int ret;
|
|
if((ret = WordTree::Prepare(words, nuniq, nproximity, document, document_length, location)) != OK)
|
|
return ret;
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++) {
|
|
if((ret = cursors[i]->Prepare(words, nuniq, nproximity, document, document_length, location)) != OK)
|
|
return ret;
|
|
}
|
|
return Get(GetSearch().GetWord());
|
|
}
|
|
|
|
//-
|
|
// The current cursor offset (set by Seek for instance). It
|
|
// duplicates the function of the WordCursor <i>key</i> data member
|
|
// because the data type is different (WordKey instead of String).
|
|
//
|
|
WordKey pos;
|
|
//-
|
|
// Sub nodes array.
|
|
//
|
|
WordTree** cursors;
|
|
//-
|
|
// Number of valid entries in the <i>cursors</i> member.
|
|
//
|
|
unsigned int cursors_length;
|
|
//-
|
|
// Permutation generator with proximity toggle
|
|
//
|
|
WordPermute permutation;
|
|
};
|
|
|
|
WordTreeOperand::~WordTreeOperand()
|
|
{
|
|
if(cursors) {
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
delete cursors[i];
|
|
free(cursors);
|
|
}
|
|
}
|
|
|
|
int
|
|
WordTreeOperand::Optimize()
|
|
{
|
|
//
|
|
// Apply to each cursor
|
|
//
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if(cursors[i]->Optimize() == NOTOK)
|
|
return NOTOK;
|
|
return OK;
|
|
}
|
|
|
|
int WordTreeOperand::OptimizeOr(int proximity)
|
|
{
|
|
unsigned int ignore = 0;
|
|
unsigned int ignore_mask = 0;
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++) {
|
|
int reduce;
|
|
//
|
|
// Set ignore & ignore_mask if cursor is NOT or MANDATORY
|
|
//
|
|
switch(cursors[i]->IsA()) {
|
|
case WORD_TREE_MANDATORY:
|
|
ignore |= (1 << WORD_EXCLUDE_POSITION2BIT(cursors_length, i));
|
|
reduce = 1;
|
|
break;
|
|
case WORD_TREE_NOT:
|
|
ignore |= (1 << WORD_EXCLUDE_POSITION2BIT(cursors_length, i));
|
|
ignore_mask |= (1 << WORD_EXCLUDE_POSITION2BIT(cursors_length, i));
|
|
reduce = 1;
|
|
break;
|
|
default:
|
|
reduce = 0;
|
|
break;
|
|
}
|
|
//
|
|
// Replace the NOT or MANDATORY node by its only child
|
|
//
|
|
if(reduce) {
|
|
WordTreeOperand* old = (WordTreeOperand*)cursors[i];
|
|
cursors[i] = old->cursors[0];
|
|
old->cursors[0] = 0;
|
|
old->cursors_length--;
|
|
if(old->cursors_length > 0) {
|
|
fprintf(stderr, "WordTreeOptional::OptimizeOr: too many cursors\n");
|
|
return NOTOK;
|
|
}
|
|
delete old;
|
|
}
|
|
}
|
|
return permutation.Initialize(cursors_length, ignore, ignore_mask, proximity);
|
|
}
|
|
|
|
int
|
|
WordTreeOperand::WalkInit()
|
|
{
|
|
unsigned int i;
|
|
int ret = WORD_WALK_ATEND;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if((ret = cursors[i]->WalkInit()) != OK)
|
|
return ret;
|
|
return (status = ret);
|
|
}
|
|
|
|
int
|
|
WordTreeOperand::WalkRewind()
|
|
{
|
|
unsigned int i;
|
|
int ret = OK;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if((ret = cursors[i]->WalkRewind()) != OK)
|
|
return ret;
|
|
status = OK;
|
|
key_semantic.DocumentClear(pos);
|
|
cursor_get_flags = DB_SET_RANGE;
|
|
found.Clear();
|
|
return ret;
|
|
}
|
|
|
|
int
|
|
WordTreeOperand::WalkFinish()
|
|
{
|
|
unsigned int i;
|
|
int ret = OK;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if((ret = cursors[i]->WalkFinish()) != OK)
|
|
return ret;
|
|
return ret;
|
|
}
|
|
|
|
int
|
|
WordTreeOperand::Seek(const WordKey& patch)
|
|
{
|
|
pos.CopyFrom(patch);
|
|
cursor_get_flags = DB_SET_RANGE;
|
|
|
|
unsigned int i;
|
|
int ret = OK;
|
|
for(i = 0; i < cursors_length; i++)
|
|
if((ret = cursors[i]->Seek(patch)) != OK &&
|
|
ret != WORD_WALK_ATEND)
|
|
return ret;
|
|
status = OK;
|
|
return OK;
|
|
}
|
|
|
|
// ************************* WordTreeOptional implementation ****************
|
|
|
|
class WordTreeOptional : public WordTreeOperand {
|
|
public:
|
|
WordTreeOptional(const char* nscope) : WordTreeOperand(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_OPTIONAL
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_OPTIONAL; }
|
|
|
|
virtual int Optimize();
|
|
|
|
virtual int ContextSaveList(StringList& list) const;
|
|
|
|
virtual int ContextRestoreList(StringList& list);
|
|
|
|
//-
|
|
// Multipass walk of the occurrences according to the <i>permutation</i>
|
|
// data member specifications. First search for documents containing
|
|
// all occurrences near to each other. Then documents that
|
|
// contain all occurrences far appart. Then ignore the most frequent
|
|
// search criterion and search for documents that contain all the others
|
|
// near to each other. The logic goes on until there only remains the
|
|
// most frequent word.
|
|
//
|
|
virtual int WalkNext();
|
|
//-
|
|
// Only seek the first non excluded cursor. The implementation
|
|
// of WalkNext makes it useless to seek the others.
|
|
//
|
|
virtual int Seek(const WordKey& position);
|
|
|
|
virtual int Prepare(WordList *words, int nuniq, int nproximity, int *document, int document_length, int location) {
|
|
int ret;
|
|
if((ret = permutation.Initialize(cursors_length, 0, 0, WORD_PERMUTE_PROXIMITY_TOGGLE)) != OK)
|
|
return ret;
|
|
return WordTreeOperand::Prepare(words, nuniq, nproximity, document, document_length, location);
|
|
}
|
|
|
|
virtual void SetInfo();
|
|
|
|
virtual int UseProximity() const { return WORD_PERMUTE_PROXIMITY_TOGGLE; }
|
|
|
|
virtual int UsePermutation() const { return 1; }
|
|
|
|
//-
|
|
// Returns true if all cursors must have a frequency > 0, false otherwise.
|
|
//
|
|
virtual int AllOrNothing() const { return 0; }
|
|
|
|
//-
|
|
// Comparison between <b>cursor</b> and <b>constraint</b> is made
|
|
// with WordKeySemantic::LocationCompare using the <b>proximity</b>
|
|
// argument. If <b>master</b> is NULL it is set to point to <b>
|
|
// <b>cursor</b>.
|
|
//
|
|
// Return WORD_WALK_NEXT if <b>cursor</b> is at <b>constraint</b> and
|
|
// set <b>constraint</b> if <b>cursor</b> is <b>master</b>.
|
|
//
|
|
// Return WORD_WALK_REDO if <b>cursor</b> is above <b>constraint</b> and
|
|
// call cursor.WalkNext().
|
|
//
|
|
// Return WORD_WALK_RESTART if <b>cursor</b> is below <b>constraint</b> and
|
|
// set <b>constraint</b> from <b>cursor</b> using
|
|
// WordKeySemantic::DocumentSet if <b>cursor</b> is not <b>master</b>
|
|
// otherwise also set location of <b>constraint</b> using
|
|
// WordKeySemantic::LocationSet and call WordKeySemantic::LocationNext
|
|
// on <b>constraint.</b>
|
|
//
|
|
// Return WORD_WALK_ATEND if no more match possible.
|
|
//
|
|
// Return NOTOK on failure.
|
|
//
|
|
int SearchCursorNear(WordTree& cursor, WordTree*& master, WordKey& constraint, int proximity);
|
|
//-
|
|
// Comparison between <b>cursor</b> and <b>document</b> is made
|
|
// with WordKeySemantic::DocumentCompare.
|
|
//
|
|
// Return WORD_WALK_NEXT if <b>cursor</b> is above <b>document.</b>
|
|
//
|
|
// Return WORD_WALK_REDO if <b>cursor</b> is below <b>document</b>
|
|
// and call cursor.WalkNext().
|
|
//
|
|
// Return WORD_WALK_RESTART if <b>cursor</b> is at <b>document</b>
|
|
// and call WordKeySemantic::DocumentNext method on <b>document.</b>
|
|
//
|
|
// Return WORD_WALK_ATEND if no more match possible.
|
|
//
|
|
// Return NOTOK on failure.
|
|
//
|
|
int SearchCursorNot(WordTree& cursor, WordKey& document);
|
|
//-
|
|
// Comparison between <b>cursor</b> and <b>document</b> is made
|
|
// with WordKeySemantic::DocumentCompare.
|
|
//
|
|
// Return WORD_WALK_NEXT if <b>cursor</b> is at <b>document.</b>.
|
|
//
|
|
// Return WORD_WALK_REDO if <b>cursor</b> is below <b>document</b>
|
|
//
|
|
// Return WORD_WALK_RESTART if <b>cursor</b> is above <b>document</b>
|
|
// and call WordKeySemantic::DocumentNext method on <b>document.</b>
|
|
//
|
|
// Return WORD_WALK_ATEND if no more match possible.
|
|
//
|
|
// Return NOTOK on failure.
|
|
//
|
|
//
|
|
int SearchCursorAnd(WordTree& cursor, WordKey& document, WordExclude& permutation);
|
|
//
|
|
// We know that :
|
|
// 1) document does not contain any excluded words.
|
|
// 2) contains at least one occurrence of each non excluded word.
|
|
// The logic, although very similar to WordSearchNear::SearchOne
|
|
// is therefore simpler. We ignore all excluded cursors and
|
|
// return WORD_SEARCH_NOPROXIMITY as soon as a cursor move outside
|
|
// <document>.
|
|
//
|
|
//-
|
|
// If <b>document</b> contains words that match proximity
|
|
// requirement, return OK. Return WORD_SEARCH_NOPROXIMITY if proximity
|
|
// requirement cannot be matched for <document>.
|
|
//
|
|
int CursorsObeyProximity(WordKey& document);
|
|
|
|
//-
|
|
// Sort the <i>cursors</i> in ascending frequency order using the
|
|
// Noccurrence method on each cursor.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
virtual int AscendingFrequency();
|
|
//-
|
|
// Delete all elements of the <i>cursors</i> array that have a
|
|
// zero frequency. The <i>cursors</i> array is shrinked and the
|
|
// <i>cursors_length</i> set accordingly. Returns the number of
|
|
// deletions in the <b>stripped</i> argument.
|
|
// Return OK on success, NOTOK on failure.
|
|
//
|
|
virtual int StripNonExistent(unsigned int& stripped);
|
|
};
|
|
|
|
int WordTreeOptional::Optimize()
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::Optimize()) != OK)
|
|
return ret;
|
|
|
|
if(UseProximity() != WORD_PERMUTE_PROXIMITY_ONLY) {
|
|
if((ret = AscendingFrequency()) != OK)
|
|
return ret;
|
|
}
|
|
|
|
unsigned int stripped;
|
|
if((ret = StripNonExistent(stripped)) != OK)
|
|
return ret;
|
|
|
|
if(AllOrNothing() && stripped) {
|
|
//
|
|
// One word is missing and everything is lost,
|
|
// Just kill the remaining cursors.
|
|
//
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
delete cursors[i];
|
|
cursors_length = 0;
|
|
|
|
return OK;
|
|
} else {
|
|
return OptimizeOr(UseProximity());
|
|
}
|
|
}
|
|
|
|
int WordTreeOptional::ContextSaveList(StringList& list) const
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::ContextSaveList(list)) != OK)
|
|
return ret;
|
|
|
|
if(UsePermutation()) {
|
|
String* buffer = new String();
|
|
permutation.Get(*buffer);
|
|
|
|
list.Add(buffer);
|
|
}
|
|
|
|
{
|
|
String* buffer = new String();
|
|
if((ret = WordCursor::ContextSave(*buffer)) != OK)
|
|
return ret;
|
|
|
|
list.Add(buffer);
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
int WordTreeOptional::ContextRestoreList(StringList& list)
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::ContextRestoreList(list)) != OK)
|
|
return ret;
|
|
|
|
if(UsePermutation()) {
|
|
char* buffer = list[0];
|
|
if((ret = permutation.Set(buffer)) != OK)
|
|
return ret;
|
|
list.Remove(0);
|
|
}
|
|
|
|
{
|
|
char* buffer = list[0];
|
|
if(!buffer) return NOTOK;
|
|
WordKey key(buffer);
|
|
if((ret = Seek(key)) != OK)
|
|
return ret;
|
|
cursor_get_flags = DB_NEXT;
|
|
|
|
list.Remove(0);
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
int WordTreeOptional::WalkNext()
|
|
{
|
|
WordKey& constraint = pos;
|
|
//
|
|
// Set constraint with all 0
|
|
//
|
|
if(constraint.Empty())
|
|
key_semantic.DocumentClear(constraint);
|
|
|
|
//
|
|
// Advance cursors to next constraint, if not at the
|
|
// beginning of the search.
|
|
//
|
|
int ret = OK;
|
|
int match_ok = 0;
|
|
do {
|
|
//
|
|
// Advance cursors so that next call fetches another constraint
|
|
//
|
|
if(cursor_get_flags == DB_NEXT)
|
|
key_semantic.DocumentNext(constraint, uniq);
|
|
|
|
if((ret = Seek(constraint)) != OK)
|
|
return ret;
|
|
|
|
int near = permutation.Proximity();
|
|
WordTree* first = 0;
|
|
for(unsigned int i = 0; i < cursors_length;) {
|
|
WordTree& cursor = *(cursors[i]);
|
|
near = permutation.Proximity();
|
|
int excluded = permutation.Excluded(i);
|
|
if(verbose) fprintf(stderr, "WordTreeOptional::WalkNext: %s excluded = %s, proximity = %s\n", (char*)cursor.GetSearch().GetWord(), (excluded ? "yes" : "no"), (near ? "yes" : "no" ));
|
|
|
|
int ret;
|
|
if(excluded) {
|
|
ret = SearchCursorNot(cursor, constraint);
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::WalkNext: Not -> %s\n", ret2str(ret));
|
|
} else {
|
|
if(near) {
|
|
ret = SearchCursorNear(cursor, first, constraint, proximity);
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::WalkNext: Near -> %s\n", ret2str(ret));
|
|
} else {
|
|
ret = SearchCursorAnd(cursor, constraint, permutation);
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::WalkNext: And -> %s\n", ret2str(ret));
|
|
}
|
|
}
|
|
|
|
switch(ret) {
|
|
case WORD_WALK_ATEND:
|
|
if(UsePermutation()) {
|
|
//
|
|
// The search is over with this permutation, try another one.
|
|
//
|
|
switch(permutation.Next()) {
|
|
//
|
|
// No permutations left, the end
|
|
//
|
|
case WORD_PERMUTE_END:
|
|
return (status = WORD_WALK_ATEND);
|
|
break;
|
|
|
|
//
|
|
// Sart over with this permutation
|
|
//
|
|
case WORD_PERMUTE_OK:
|
|
if(WalkRewind() != OK)
|
|
return NOTOK;
|
|
break;
|
|
}
|
|
first = 0;
|
|
i = 0;
|
|
} else {
|
|
return (status = WORD_WALK_ATEND);
|
|
}
|
|
break;
|
|
case WORD_WALK_REDO:
|
|
break;
|
|
case WORD_WALK_RESTART:
|
|
first = 0;
|
|
i = 0;
|
|
break;
|
|
case WORD_WALK_NEXT:
|
|
i++;
|
|
break;
|
|
case NOTOK:
|
|
default:
|
|
return ret;
|
|
break;
|
|
}
|
|
}
|
|
|
|
cursor_get_flags = DB_NEXT;
|
|
|
|
SetInfo();
|
|
|
|
//
|
|
// Save possible result, i.e. first non excluded cursor
|
|
//
|
|
for(unsigned int i = 0; i < cursors_length; i++) {
|
|
WordTree& cursor = *(cursors[i]);
|
|
if(!permutation.Excluded(i)) {
|
|
found.Key().CopyFrom(cursor.GetFound().Key());
|
|
break;
|
|
}
|
|
}
|
|
|
|
match_ok = 1;
|
|
//
|
|
// Only bother if near and non near search are involved
|
|
//
|
|
if(UseProximity() == WORD_PERMUTE_PROXIMITY_TOGGLE) {
|
|
//
|
|
// If we reach this point in the function and
|
|
// either proximity search is active or there is
|
|
// only one word involved, the match is valid.
|
|
// Otherwise it may be excluded, see below.
|
|
//
|
|
if(!near && permutation.NotExcludedCount() > 1) {
|
|
//
|
|
// If not using proximity, a match that fits the proximity
|
|
// requirements must be skipped because it was matched by
|
|
// the previous permutation (see WordPermute).
|
|
//
|
|
switch(CursorsObeyProximity(constraint)) {
|
|
case OK:
|
|
match_ok = 0;
|
|
break;
|
|
case WORD_SEARCH_NOPROXIMITY:
|
|
match_ok = 1;
|
|
break;
|
|
default:
|
|
case NOTOK:
|
|
return NOTOK;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} while(!match_ok && ret == OK);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int WordTreeOptional::Seek(const WordKey& position)
|
|
{
|
|
pos.CopyFrom(position);
|
|
cursor_get_flags = DB_SET_RANGE;
|
|
status = OK;
|
|
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++) {
|
|
if(!permutation.Excluded(i)) {
|
|
WordTree& cursor = *(cursors[i]);
|
|
return cursor.Seek(position);
|
|
}
|
|
}
|
|
|
|
fprintf(stderr, "WordTreeOptional::Seek: failed\n");
|
|
return NOTOK;
|
|
}
|
|
|
|
|
|
void WordTreeOptional::SetInfo()
|
|
{
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
cursors[i]->SetInfo();
|
|
|
|
info.trunc();
|
|
|
|
for(i = 0; i < cursors_length; i++) {
|
|
WordTree& cursor = *(cursors[i]);
|
|
|
|
if(!permutation.Excluded(i))
|
|
info << cursor.info << " ";
|
|
}
|
|
|
|
info << (permutation.Proximity() ? "proximity" : "");
|
|
}
|
|
|
|
int WordTreeOptional::SearchCursorNear(WordTree& cursor, WordTree*& master, WordKey& constraint, int proximity)
|
|
{
|
|
int is_master = master == 0 || master == &cursor;
|
|
if(master == 0) master = &cursor;
|
|
const WordKey& masterKey = master->GetFound().Key();
|
|
|
|
int direction = key_semantic.LocationCompare(constraint, cursor.GetFound().Key(), proximity);
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::SearchCursorNear: LocationCompare(\n\t%s,\n\t%s)\n\t = %d\n", (char*)(constraint.Get()), (char*)(cursor.GetFound().Key().Get()), direction);
|
|
|
|
//
|
|
// If the cursor is in the authorized locations, consider
|
|
// next cursor
|
|
//
|
|
if(direction == 0) {
|
|
//
|
|
// master cursor makes the rules for location : its location
|
|
// is the base to calculate other words mandatory loacations.
|
|
//
|
|
if(is_master)
|
|
key_semantic.LocationSet(cursor.GetFound().Key(), constraint);
|
|
//
|
|
// Fix location constraint to accomodate proximity tolerance.
|
|
//
|
|
key_semantic.LocationNearLowest(constraint, proximity);
|
|
return WORD_WALK_NEXT;
|
|
|
|
//
|
|
// If current location is above cursor location
|
|
//
|
|
} else if(direction > 0) {
|
|
//
|
|
// Move the cursor up to the location.
|
|
//
|
|
cursor.Seek(constraint);
|
|
if(verbose > 1) fprintf(stderr, "WordTreeOptional::SearchCursorNear: leap to %s\n", (char*)constraint.Get());
|
|
int ret;
|
|
if((ret = cursor.WalkNext()) == OK) {
|
|
//
|
|
// Remove the location constraint for the master word
|
|
// so that it matches and then enforce location for other
|
|
// keys.
|
|
//
|
|
if(is_master)
|
|
key_semantic.Location2Document(constraint);
|
|
//
|
|
// Reconsider the situation for this cursor
|
|
//
|
|
return WORD_WALK_REDO;
|
|
} else {
|
|
return ret;
|
|
}
|
|
|
|
//
|
|
// If current location is lower than cursor location,
|
|
// meaning that the cursor found no match for the current
|
|
// location.
|
|
//
|
|
} else if(direction < 0) {
|
|
//
|
|
// The cursor document becomes the current document.
|
|
// The master cursor is forced to catch up.
|
|
//
|
|
key_semantic.DocumentSet(cursor.GetDocument(), constraint);
|
|
//
|
|
// It is possible that this cursor document is the same
|
|
// as the master cursor document (if this cursor hit in the
|
|
// same document but a higher location). In this case we must
|
|
// increase the location of the master cursor otherwise it will
|
|
// match without moving and loop forever.
|
|
//
|
|
if(!is_master && key_semantic.DocumentCompare(masterKey, constraint) == 0) {
|
|
key_semantic.LocationSet(masterKey, constraint);
|
|
key_semantic.LocationNext(constraint);
|
|
}
|
|
//
|
|
// Since the current location changed, start over.
|
|
//
|
|
return WORD_WALK_RESTART;
|
|
} else {
|
|
fprintf(stderr, "WordTreeOptional::WordCursorNear: reached unreachable statement\n");
|
|
return NOTOK;
|
|
}
|
|
return NOTOK;
|
|
}
|
|
|
|
int WordTreeOptional::SearchCursorNot(WordTree& cursor, WordKey& document)
|
|
{
|
|
int direction = key_semantic.DocumentCompare(document, cursor.GetFound().Key());
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::SearchCursorNot: DocumentCompare(\n\t%s,\n\t%s)\n\t = %d\n", (char*)(document.Get()), (char*)(cursor.GetFound().Key().Get()), direction);
|
|
|
|
//
|
|
// If the cursor is above the current document
|
|
// (being at the end of walk is being above all documents).
|
|
//
|
|
// Means that the cursor is positioned in an acceptable document
|
|
// and proceed to the next cursor.
|
|
//
|
|
if(direction < 0 || cursor.IsAtEnd()) {
|
|
return WORD_WALK_NEXT;
|
|
|
|
//
|
|
// If the cursor is below current document
|
|
//
|
|
} else if(direction > 0) {
|
|
//
|
|
// Move the cursor up to the document
|
|
//
|
|
cursor.Seek(document);
|
|
if(verbose > 1) fprintf(stderr, "WordTreeOptional::SearchCursorNot: leap to %s\n", (char*)document.Get());
|
|
int ret;
|
|
if((ret = cursor.WalkNext()) != OK && ret != WORD_WALK_ATEND)
|
|
return NOTOK;
|
|
//
|
|
// It is expected in this case that the cursor has moved above
|
|
// the current document and another visit in the loop will
|
|
// tell us.
|
|
//
|
|
return WORD_WALK_REDO;
|
|
|
|
//
|
|
// If the cursor matches the current document.
|
|
//
|
|
// Means that the current document is not a possible match
|
|
// since it is pointed by this cursor.
|
|
//
|
|
} else if(direction == 0) {
|
|
//
|
|
// The cursor does not give any hint on a possible
|
|
// next document, just go to the next possible one.
|
|
//
|
|
key_semantic.DocumentNext(document, uniq);
|
|
//
|
|
// Since the current document changed, start over.
|
|
//
|
|
return WORD_WALK_RESTART;
|
|
} else {
|
|
fprintf(stderr, "WordTreeOptional::WordCursorNot: reached unreachable statement\n");
|
|
return NOTOK;
|
|
}
|
|
return NOTOK;
|
|
}
|
|
|
|
int WordTreeOptional::SearchCursorAnd(WordTree& cursor, WordKey& document, WordExclude& permutation)
|
|
{
|
|
int direction = key_semantic.DocumentCompare(document, cursor.GetFound().Key());
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::SearchCursorAnd: DocumentCompare(\n\t%s,\n\t%s)\n\t = %d\n", (char*)(document.Get()), (char*)(cursor.GetFound().Key().Get()), direction);
|
|
|
|
//
|
|
// If the cursor is in the current document.
|
|
//
|
|
// Means that the cursor is positioned in an acceptable document
|
|
// and proceed to the next cursor.
|
|
//
|
|
if(direction == 0) {
|
|
return WORD_WALK_NEXT;
|
|
|
|
//
|
|
// If the cursor is below current document
|
|
//
|
|
} else if(direction > 0) {
|
|
//
|
|
// Move the cursor up to the document
|
|
//
|
|
cursor.Seek(document);
|
|
if(verbose > 1) fprintf(stderr, "WordTreeOptional::SearchCursorAnd: leap to %s\n", (char*)document.Get());
|
|
int ret;
|
|
if((ret = cursor.WalkNext()) == OK)
|
|
return WORD_WALK_REDO;
|
|
else
|
|
return ret;
|
|
|
|
//
|
|
// If the cursor is above current document.
|
|
//
|
|
// Means the the current document is not a possible match
|
|
// since it will never reach it because it's already
|
|
// above it.
|
|
//
|
|
} else if(direction < 0) {
|
|
//
|
|
// The cursor document becomes the current document.
|
|
//
|
|
key_semantic.DocumentSet(cursor.GetDocument(), document);
|
|
|
|
//
|
|
// Since the current document changed, start over.
|
|
//
|
|
return WORD_WALK_RESTART;
|
|
} else {
|
|
fprintf(stderr, "WordTreeOptional::WordCursorAnd: reached unreachable statement\n");
|
|
return NOTOK;
|
|
}
|
|
return NOTOK;
|
|
}
|
|
|
|
int WordTreeOptional::CursorsObeyProximity(WordKey& document)
|
|
{
|
|
//
|
|
// Run if more than one word is involved, proximity
|
|
// is always true if there is only one word.
|
|
//
|
|
if(permutation.NotExcludedCount() <= 1) return OK;
|
|
|
|
WordKey location;
|
|
|
|
//
|
|
// The first non excluded cursor contains anchor location.
|
|
//
|
|
unsigned int master_index = 0;
|
|
for(unsigned int i = 0; i < cursors_length; i++) {
|
|
if(!permutation.Excluded(i)) {
|
|
master_index = i;
|
|
break;
|
|
}
|
|
}
|
|
const WordKey& masterKey = cursors[master_index]->GetFound().Key();
|
|
key_semantic.DocumentSet(masterKey, location);
|
|
|
|
for(unsigned int i = 0; i < cursors_length;) {
|
|
if(permutation.Excluded(i)) {
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
WordTree& cursor = *(cursors[i]);
|
|
if(cursor.IsAtEnd()) return WORD_SEARCH_NOPROXIMITY;
|
|
// if(cursor.status & WORD_WALK_FAILED) return NOTOK;
|
|
|
|
//
|
|
// If the cursor moved outside of the tested document,
|
|
// no proximity match is possible.
|
|
//
|
|
if(key_semantic.DocumentCompare(cursor.GetFound().Key(), document) != 0)
|
|
return WORD_SEARCH_NOPROXIMITY;
|
|
|
|
int direction = key_semantic.LocationCompare(location, cursor.GetFound().Key(), proximity);
|
|
|
|
//
|
|
// If the cursor is in the authorized locations, consider
|
|
// next cursor
|
|
//
|
|
if(direction == 0) {
|
|
//
|
|
// master cursor makes the rules for location : its location
|
|
// is the base to calculate other words mandatory loacations.
|
|
//
|
|
if(i == master_index)
|
|
key_semantic.LocationSet(cursor.GetFound().Key(), location);
|
|
//
|
|
// Fix location constraint to accomodate proximity tolerance.
|
|
//
|
|
key_semantic.LocationNearLowest(location, proximity);
|
|
i++;
|
|
|
|
//
|
|
// If current location is greater than cursor location
|
|
//
|
|
} else if(direction > 0) {
|
|
//
|
|
// Move the cursor up to the location.
|
|
//
|
|
cursor.Seek(location);
|
|
if(verbose > 1) fprintf(stderr, "WordTreeOptional::CursorsObeyProximity: leap to %s\n", (char*)location.Get());
|
|
int ret;
|
|
if((ret = cursor.WalkNext()) != OK) {
|
|
if(ret == WORD_WALK_ATEND) {
|
|
return WORD_SEARCH_NOPROXIMITY;
|
|
} else {
|
|
return NOTOK;
|
|
}
|
|
}
|
|
//
|
|
// Remove the location constraint for the master word
|
|
// so that it matches and then enforce location for other
|
|
// keys.
|
|
//
|
|
if(i == master_index)
|
|
key_semantic.Location2Document(location);
|
|
//
|
|
// Reconsider the situation for this cursor
|
|
//
|
|
|
|
//
|
|
// If current location is lower than cursor location,
|
|
// meaning that the cursor found no match in the current
|
|
// document.
|
|
//
|
|
} else if(direction < 0) {
|
|
//
|
|
// Move to next master key, if possible.
|
|
//
|
|
if(i != master_index) {
|
|
key_semantic.LocationSet(masterKey, location);
|
|
key_semantic.LocationNext(location);
|
|
}
|
|
//
|
|
// Since the current location changed, start over.
|
|
//
|
|
i = 0;
|
|
}
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
//
|
|
// Helper class for AscendingFrequency method
|
|
//
|
|
class WordSort {
|
|
public:
|
|
unsigned int frequency;
|
|
WordTree *cursor;
|
|
};
|
|
|
|
//
|
|
// Helper function for AscendingFrequency method
|
|
//
|
|
static int ascending_frequency(const void *a, const void *b)
|
|
{
|
|
const WordSort& a_cursor = *(WordSort*)a;
|
|
const WordSort& b_cursor = *(WordSort*)b;
|
|
|
|
return a_cursor.frequency - b_cursor.frequency;
|
|
}
|
|
|
|
int WordTreeOptional::AscendingFrequency()
|
|
{
|
|
//
|
|
// Reorder cursors
|
|
//
|
|
WordSort *tmp = new WordSort[cursors_length];
|
|
|
|
memset((char*)tmp, '\0', cursors_length * sizeof(WordSort));
|
|
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++) {
|
|
unsigned int frequency;
|
|
if(cursors[i]->Noccurrence(frequency) != OK) {
|
|
delete [] tmp;
|
|
return NOTOK;
|
|
}
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::AscendingFrequency: %s occurs %d times\n", (char*)cursors[i]->GetSearch().Get(), frequency);
|
|
tmp[i].frequency = frequency;
|
|
tmp[i].cursor = cursors[i];
|
|
}
|
|
|
|
memset((char*)cursors, '\0', cursors_length * sizeof(WordTree*));
|
|
|
|
qsort((void *)tmp, cursors_length, sizeof(WordSort), &ascending_frequency);
|
|
|
|
for(i = 0; i < cursors_length; i++)
|
|
cursors[i] = tmp[i].cursor;
|
|
|
|
delete [] tmp;
|
|
return OK;
|
|
}
|
|
|
|
int WordTreeOptional::StripNonExistent(unsigned int& stripped)
|
|
{
|
|
stripped = 0;
|
|
|
|
WordTree** tmp = new WordTree*[cursors_length];
|
|
memset((char*)tmp, '\0', cursors_length * sizeof(WordTree*));
|
|
|
|
unsigned int from;
|
|
unsigned int to;
|
|
|
|
for(to = from = 0; from < cursors_length; from++) {
|
|
unsigned int frequency;
|
|
if(cursors[from]->Noccurrence(frequency) != OK) {
|
|
delete [] tmp;
|
|
return NOTOK;
|
|
}
|
|
|
|
if(verbose > 2) fprintf(stderr, "WordTreeOptional::StripNonExistent: %s occurs %d times\n", (char*)cursors[from]->GetSearch().Get(), frequency);
|
|
if(frequency > 0) {
|
|
tmp[to++] = cursors[from];
|
|
} else {
|
|
delete cursors[from];
|
|
stripped++;
|
|
}
|
|
}
|
|
|
|
memset((char*)cursors, '\0', cursors_length * sizeof(WordTree*));
|
|
|
|
cursors_length = to;
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
cursors[i] = tmp[i];
|
|
|
|
delete [] tmp;
|
|
|
|
return OK;
|
|
}
|
|
|
|
// ************************* WordTreeOr implementation ********************
|
|
|
|
class WordTreeOr : public WordTreeOperand {
|
|
public:
|
|
WordTreeOr(const char* nscope) : WordTreeOperand(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_OR
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_OR; }
|
|
|
|
virtual int Optimize();
|
|
|
|
virtual int ContextSaveList(StringList& list) const;
|
|
|
|
virtual int ContextRestoreList(StringList& list);
|
|
|
|
virtual void SetInfo();
|
|
|
|
virtual int WalkNext();
|
|
|
|
virtual int UsePermutation() const { return 0; }
|
|
|
|
virtual int UseProximity() const { return WORD_PERMUTE_PROXIMITY_NO; }
|
|
};
|
|
|
|
int WordTreeOr::Optimize()
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::Optimize()) != OK)
|
|
return ret;
|
|
|
|
if((ret = AscendingFrequency()) != OK)
|
|
return ret;
|
|
|
|
unsigned int stripped;
|
|
if((ret = StripNonExistent(stripped)) != OK)
|
|
return ret;
|
|
|
|
return OptimizeOr(WORD_PERMUTE_PROXIMITY_NO);
|
|
}
|
|
|
|
int WordTreeOr::ContextSaveList(StringList& list) const
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::ContextSaveList(list)) != OK)
|
|
return ret;
|
|
|
|
{
|
|
String* buffer = new String();
|
|
permutation.Get(*buffer);
|
|
|
|
list.Add(buffer);
|
|
}
|
|
|
|
{
|
|
String* buffer = new String();
|
|
if((ret = WordCursor::ContextSave(*buffer)) != OK)
|
|
return ret;
|
|
|
|
list.Add(buffer);
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
int WordTreeOr::ContextRestoreList(StringList& list)
|
|
{
|
|
int ret;
|
|
if((ret = WordTreeOperand::ContextRestoreList(list)) != OK)
|
|
return ret;
|
|
|
|
{
|
|
char* buffer = list[0];
|
|
if((ret = permutation.Set(buffer)) != OK)
|
|
return ret;
|
|
list.Remove(0);
|
|
}
|
|
|
|
{
|
|
char* buffer = list[0];
|
|
if(!buffer) return NOTOK;
|
|
WordKey key(buffer);
|
|
if((ret = Seek(key)) != OK)
|
|
return ret;
|
|
cursor_get_flags = DB_NEXT;
|
|
|
|
list.Remove(0);
|
|
}
|
|
|
|
return OK;
|
|
}
|
|
|
|
void WordTreeOr::SetInfo()
|
|
{
|
|
unsigned int i;
|
|
for(i = 0; i < cursors_length; i++)
|
|
cursors[i]->SetInfo();
|
|
|
|
info.trunc();
|
|
|
|
for(i = 0; i < cursors_length; i++) {
|
|
WordTree& cursor = *(cursors[i]);
|
|
|
|
if(!permutation.Excluded(i) &&
|
|
!cursor.IsAtEnd() &&
|
|
key_semantic.DocumentCompare(cursor.GetFound().Key(), GetFound().Key()) == 0) {
|
|
info << cursor.info << " ";
|
|
}
|
|
}
|
|
}
|
|
|
|
int WordTreeOr::WalkNext()
|
|
{
|
|
WordKey& constraint = pos;
|
|
//
|
|
// Set constraint with all 0
|
|
//
|
|
if(constraint.Empty())
|
|
key_semantic.DocumentClear(constraint);
|
|
|
|
WordKey candidate;
|
|
int match_ok;
|
|
do {
|
|
int ret;
|
|
unsigned int i;
|
|
candidate.Clear();
|
|
//
|
|
// Advance cursors so that next call fetches another constraint
|
|
//
|
|
if(cursor_get_flags == DB_NEXT)
|
|
key_semantic.DocumentNext(constraint, uniq);
|
|
|
|
if((ret = Seek(constraint)) != OK)
|
|
return ret;
|
|
|
|
match_ok = 1;
|
|
//
|
|
// All non excluded cursors are about to move
|
|
// at or beyond constraint. Search for the one (candidate) that
|
|
// is located at the lowest location beyond the constraint.
|
|
//
|
|
for(i = 0; i < cursors_length; i++) {
|
|
if(permutation.Excluded(i))
|
|
continue;
|
|
WordTree& cursor = *(cursors[i]);
|
|
|
|
switch((ret = cursor.WalkNext())) {
|
|
case WORD_WALK_ATEND:
|
|
//
|
|
// Constraint is above all matches for this cursor
|
|
//
|
|
break;
|
|
case OK:
|
|
//
|
|
// If candidate is not set or current cursor is below
|
|
// the current candidate, the curent cursor document becomes
|
|
// the candidate.
|
|
//
|
|
if(candidate.Empty() ||
|
|
key_semantic.DocumentCompare(candidate, cursor.GetFound().Key()) > 0) {
|
|
key_semantic.DocumentSet(cursor.GetDocument(), candidate);
|
|
}
|
|
break;
|
|
default:
|
|
return ret;
|
|
break;
|
|
}
|
|
}
|
|
|
|
//
|
|
// No candidate ? It's the end of the match list.
|
|
//
|
|
if(candidate.Empty())
|
|
return WORD_WALK_ATEND;
|
|
|
|
found.Key().CopyFrom(candidate);
|
|
|
|
SetInfo();
|
|
|
|
if(permutation.ExcludedCount() > 0) {
|
|
if((ret = Seek(candidate)) != OK)
|
|
return ret;
|
|
|
|
//
|
|
// Restart loop if candidate matches an excluded cursor.
|
|
//
|
|
for(i = 0; i < cursors_length && match_ok; i++) {
|
|
if(!permutation.Excluded(i))
|
|
continue;
|
|
WordTree& cursor = *(cursors[i]);
|
|
|
|
switch((ret = cursor.WalkNext())) {
|
|
case WORD_WALK_ATEND:
|
|
//
|
|
// This excluded cursor can't match the candidate, fine.
|
|
//
|
|
break;
|
|
case OK:
|
|
//
|
|
// This excluded cursor matches candidate therefore it's
|
|
// not a valid candidate. Restart search with this candidate
|
|
// as the constraint.
|
|
//
|
|
if(key_semantic.DocumentCompare(candidate, cursor.GetFound().Key()) == 0) {
|
|
constraint = candidate;
|
|
match_ok = 0;
|
|
}
|
|
break;
|
|
default:
|
|
return ret;
|
|
break;
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
cursor_get_flags = DB_NEXT;
|
|
|
|
} while(!match_ok);
|
|
|
|
constraint = candidate;
|
|
|
|
return OK;
|
|
}
|
|
|
|
// ************************* WordTreeAnd implementation ********************
|
|
|
|
class WordTreeAnd : public WordTreeOptional {
|
|
public:
|
|
WordTreeAnd(const char* nscope) : WordTreeOptional(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_AND
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_AND; }
|
|
|
|
virtual int UsePermutation() const { return 0; }
|
|
|
|
virtual int UseProximity() const { return WORD_PERMUTE_PROXIMITY_NO; }
|
|
|
|
virtual int AllOrNothing() const { return 1; }
|
|
};
|
|
|
|
// ************************* WordTreeNear implementation ********************
|
|
|
|
class WordTreeNear : public WordTreeOptional {
|
|
public:
|
|
WordTreeNear(const char* nscope) : WordTreeOptional(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_NEAR
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_NEAR; }
|
|
|
|
virtual int UsePermutation() const { return 0; }
|
|
|
|
virtual int UseProximity() const { return WORD_PERMUTE_PROXIMITY_ONLY; }
|
|
|
|
virtual int AllOrNothing() const { return 1; }
|
|
};
|
|
|
|
// ************************* WordTreeMandatory implementation ***************
|
|
|
|
class WordTreeMandatory : public WordTreeOperand {
|
|
public:
|
|
WordTreeMandatory(const char* nscope) : WordTreeOperand(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_MANDATORY
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_MANDATORY; }
|
|
};
|
|
|
|
// ************************* WordTreeNot implementation ***************
|
|
|
|
class WordTreeNot : public WordTreeOperand {
|
|
public:
|
|
WordTreeNot(const char* nscope) : WordTreeOperand(nscope) { }
|
|
|
|
//-
|
|
// Return WORD_TREE_NOT
|
|
//
|
|
virtual int IsA() const { return WORD_TREE_NOT; }
|
|
};
|
|
|
|
// ************************* WordMatch implementation ********************
|
|
|
|
//
|
|
// Return value of the Search method, tells us which document
|
|
// matched and why.
|
|
//
|
|
class WordMatch {
|
|
public:
|
|
|
|
//-
|
|
// Return a textual representation of the object.
|
|
//
|
|
String Get() const;
|
|
|
|
//-
|
|
// The document that matched
|
|
//
|
|
WordKey match;
|
|
//-
|
|
// An ascii description of why it matched.
|
|
//
|
|
String info;
|
|
};
|
|
|
|
String WordMatch::Get() const
|
|
{
|
|
String tmp;
|
|
match.Get(tmp);
|
|
if(!info.empty())
|
|
tmp << "(" << info << ")";
|
|
return tmp;
|
|
}
|
|
|
|
// ************************* WordSearch implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// Solve a query from a WordTree syntax tree
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordSearch.h>
|
|
//
|
|
// WordTree* expr = get_query();
|
|
// WordSearch search;
|
|
// search.limit_count = NUMBER_OF_RESULTS;
|
|
// WordMatch* search.Search(expr);
|
|
// ...
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// The WordSearch class is a wrapper to query an inverted index
|
|
// using a WordTree syntax tree.
|
|
//
|
|
// END
|
|
//
|
|
class WordSearch {
|
|
public:
|
|
WordSearch();
|
|
|
|
//-
|
|
// Perform a search from the <b>expr</b> specifications.
|
|
// Restore the context from <i>context_in</i> on <b>expr</b>.
|
|
// Then skip (using WalkNext) <i>limit_bottom</i> entries.
|
|
// Then collect in a WordMatch array of size <i>limit_count</i>
|
|
// each match returned by WalkNext. When finished store
|
|
// the context (ContextSave) in <i>context_out</i>.
|
|
// It is the responsibility of the caller to free the WordMatch
|
|
// array. If no match are found a null pointer is returned.
|
|
//
|
|
WordMatch *Search();
|
|
|
|
//
|
|
// Search backend, only run the WalkNext loop but does not
|
|
// allocate/deallocate data.
|
|
//
|
|
int SearchLoop(WordTree *expr);
|
|
|
|
//
|
|
// Return a context description string to resume the
|
|
// search at a given point.
|
|
//
|
|
const String& Context() const { return context_out; }
|
|
|
|
//
|
|
// Input
|
|
//
|
|
unsigned int limit_bottom;
|
|
unsigned int limit_count;
|
|
String context_in;
|
|
WordTree* expr;
|
|
|
|
//
|
|
// Output
|
|
//
|
|
WordMatch* matches;
|
|
unsigned int matches_size;
|
|
unsigned int matches_length;
|
|
String context_out;
|
|
};
|
|
|
|
WordSearch::WordSearch()
|
|
{
|
|
//
|
|
// Input
|
|
//
|
|
limit_bottom = 0;
|
|
limit_count = 0;
|
|
context_in.trunc();
|
|
expr = 0;
|
|
|
|
//
|
|
// Output
|
|
//
|
|
matches = 0;
|
|
matches_size = 0;
|
|
matches_length = 0;
|
|
context_out.trunc();
|
|
}
|
|
|
|
WordMatch *WordSearch::Search()
|
|
{
|
|
int ret = 0;
|
|
|
|
if(verbose) fprintf(stderr, "WordSearch::Search: non optimized expression %s\n", (char*)expr->Get());
|
|
if(expr->Optimize() != OK)
|
|
return 0;
|
|
if(verbose) fprintf(stderr, "WordSearch::Search: optimized expression %s\n", (char*)expr->Get());
|
|
|
|
|
|
//
|
|
// Build space for results
|
|
//
|
|
matches_size = limit_count + 1;
|
|
matches = new WordMatch[matches_size];
|
|
matches_length = 0;
|
|
|
|
//
|
|
// Move to first possible position.
|
|
//
|
|
if(expr->WalkInit() != OK)
|
|
goto end;
|
|
|
|
if(expr->ContextRestore(context_in) == NOTOK)
|
|
goto end;
|
|
ret = SearchLoop(expr);
|
|
//
|
|
// Don't bother saving the context if at end of
|
|
// search (WORD_WALK_ATEND) or error (NOTOK)
|
|
//
|
|
if(ret == OK && expr->ContextSave(context_out) == NOTOK)
|
|
goto end;
|
|
|
|
end:
|
|
expr->WalkFinish();
|
|
|
|
if(ret == NOTOK || matches_length <= 0) {
|
|
delete [] matches;
|
|
matches = 0;
|
|
}
|
|
|
|
return matches;
|
|
}
|
|
|
|
int WordSearch::SearchLoop(WordTree *expr)
|
|
{
|
|
int ret = OK;
|
|
unsigned int i;
|
|
//
|
|
// Skip the first <limit_bottom> documents
|
|
//
|
|
{
|
|
for(i = 0; i < limit_bottom; i++) {
|
|
if((ret = expr->WalkNext()) != OK)
|
|
return ret;
|
|
}
|
|
}
|
|
//
|
|
// Get documents up to <limit_count> or exhaustion
|
|
//
|
|
for(matches_length = 0; matches_length < limit_count; matches_length++) {
|
|
if((ret = expr->WalkNext()) != OK) {
|
|
break;
|
|
} else {
|
|
matches[matches_length].match = expr->GetDocument();
|
|
if(expr->IsA() != WORD_TREE_LITERAL)
|
|
matches[matches_length].info = ((WordTreeOperand*)expr)->GetInfo();
|
|
if(verbose) fprintf(stderr, "WordSearch::Search: match %s\n", (char*)matches[matches_length].match.Get());
|
|
}
|
|
}
|
|
|
|
if(ret == WORD_WALK_ATEND)
|
|
matches[matches_length].match.Clear();
|
|
|
|
return ret;
|
|
}
|
|
|
|
// ************************* WordParser implementation ********************
|
|
//
|
|
// NAME
|
|
//
|
|
// Textual query parser for test purpose
|
|
//
|
|
// SYNOPSIS
|
|
//
|
|
// #include <WordParser.h>
|
|
//
|
|
// WordParser parser;
|
|
// WordTree* expr = parser.Parse("( or \"scope1\" a query )");
|
|
// ...
|
|
// delete expr;
|
|
//
|
|
// DESCRIPTION
|
|
//
|
|
// The WordParser class implement a lisp-like parser for queries
|
|
// implemented by the WordTree derived classes. The syntax is rigid
|
|
// and should not be used for human input. The generic syntax of an
|
|
// expression is
|
|
// <pre>
|
|
// ( operator "scope" operand [operand ...] )
|
|
// </pre>
|
|
// The parenthesis must <b>always</b> be surrounded by white space otherwise
|
|
// the parser will be lost. The separator is white space and newline.
|
|
// Tabulation may be used in scope to separate key fields.
|
|
//
|
|
// As a special case a single word is strictly equivalent
|
|
// to
|
|
// <pre>
|
|
// ( literal "" word )
|
|
// </pre>
|
|
//
|
|
// Operators can be lower case or upper case. There is almost no syntax
|
|
// checking and it's the responsibility of the caller to associate meaningfull
|
|
// operands. For instance ( near ( not foo ) bar ) is meaningless.
|
|
//
|
|
// OPERATORS
|
|
//
|
|
// <dl>
|
|
//
|
|
// <dt> optional
|
|
// <dd> WordTreeOptional
|
|
//
|
|
// <dt> or
|
|
// <dd> WordTreeOr
|
|
//
|
|
// <dt> and
|
|
// <dd> WordTreeAnd
|
|
//
|
|
// <dt> near
|
|
// <dd> WordTreeNear
|
|
//
|
|
// <dt> not,forbiden
|
|
// <dd> WordTreeNot
|
|
//
|
|
// <dt> mandatory
|
|
// <dd> WordTreeMandatory
|
|
//
|
|
// <dt> literal
|
|
// <dd> WordTreeLiteral
|
|
//
|
|
// </dl>
|
|
//
|
|
//
|
|
// END
|
|
|
|
//
|
|
// Possible values of the info argument of ParseOperands
|
|
//
|
|
#define WORD_TREE_MANY 0x01
|
|
#define WORD_TREE_ONE 0x02
|
|
#define WORD_TREE_TWO 0x04
|
|
|
|
class WordParser {
|
|
public:
|
|
WordTree *Parse(const String& expr);
|
|
WordTree *ParseList(StringList& terms);
|
|
|
|
WordTree *ParseExpr(StringList& terms);
|
|
WordTree *ParseUnary(StringList& terms);
|
|
WordTree *ParseConj(StringList& terms);
|
|
void ParseOperands(StringList& terms, int info, WordTreeOperand* expr);
|
|
WordTree *ParseLiteral(StringList& terms);
|
|
char *ParseScope(StringList& terms);
|
|
|
|
void Shift(StringList& terms);
|
|
char *Term(StringList& terms);
|
|
};
|
|
|
|
WordTree *WordParser::Parse(const String& expr)
|
|
{
|
|
StringList terms(expr, " \n");
|
|
return ParseList(terms);
|
|
}
|
|
|
|
WordTree *WordParser::ParseList(StringList& terms)
|
|
{
|
|
WordTree *expr = ParseExpr(terms);
|
|
return expr;
|
|
}
|
|
|
|
WordTree *WordParser::ParseExpr(StringList& terms)
|
|
{
|
|
WordTree *expr = 0;
|
|
char* term = strdup(Term(terms));
|
|
if(!strcmp(term, "(")) {
|
|
Shift(terms);
|
|
expr = ParseExpr(terms);
|
|
} else if(!strcmp(term, ")")) {
|
|
//
|
|
// At end of expression, return null
|
|
//
|
|
} else if(!mystrcasecmp(term, "optional") ||
|
|
!mystrcasecmp(term, "or") ||
|
|
!mystrcasecmp(term, "and") ||
|
|
!mystrcasecmp(term, "near")) {
|
|
expr = ParseConj(terms);
|
|
} else if(!mystrcasecmp(term, "not") ||
|
|
!mystrcasecmp(term, "mandatory") ||
|
|
!mystrcasecmp(term, "forbiden")) {
|
|
expr = ParseUnary(terms);
|
|
} else {
|
|
expr = ParseLiteral(terms);
|
|
}
|
|
free(term);
|
|
return expr;
|
|
}
|
|
|
|
WordTree *WordParser::ParseUnary(StringList& terms)
|
|
{
|
|
int op = 0;
|
|
if(!mystrcasecmp(Term(terms), "mandatory"))
|
|
op = WORD_TREE_MANDATORY;
|
|
else if(!mystrcasecmp(Term(terms), "forbiden") ||
|
|
!mystrcasecmp(Term(terms), "not"))
|
|
op = WORD_TREE_NOT;
|
|
|
|
Shift(terms);
|
|
char* scope = ParseScope(terms);
|
|
WordTreeOperand *expr = 0;
|
|
switch(op) {
|
|
case WORD_TREE_MANDATORY:
|
|
expr = new WordTreeMandatory(scope);
|
|
break;
|
|
case WORD_TREE_NOT:
|
|
expr = new WordTreeNot(scope);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "WordParser::ParseUnary: unexpected operator %d\n", op);
|
|
exit(1);
|
|
break;
|
|
}
|
|
free(scope);
|
|
ParseOperands(terms, WORD_TREE_ONE, expr);
|
|
return expr;
|
|
}
|
|
|
|
WordTree *WordParser::ParseConj(StringList& terms)
|
|
{
|
|
int op = 0;
|
|
if(!mystrcasecmp(Term(terms), "optional"))
|
|
op = WORD_TREE_OPTIONAL;
|
|
else if(!mystrcasecmp(Term(terms), "or"))
|
|
op = WORD_TREE_OR;
|
|
else if(!mystrcasecmp(Term(terms), "and"))
|
|
op = WORD_TREE_AND;
|
|
else if(!mystrcasecmp(Term(terms), "near"))
|
|
op = WORD_TREE_NEAR;
|
|
|
|
Shift(terms);
|
|
char* scope = ParseScope(terms);
|
|
WordTreeOperand *expr = 0;
|
|
switch(op) {
|
|
case WORD_TREE_OR:
|
|
expr = new WordTreeOr(scope);
|
|
break;
|
|
case WORD_TREE_OPTIONAL:
|
|
expr = new WordTreeOptional(scope);
|
|
break;
|
|
case WORD_TREE_AND:
|
|
expr = new WordTreeAnd(scope);
|
|
break;
|
|
case WORD_TREE_NEAR:
|
|
expr = new WordTreeNear(scope);
|
|
break;
|
|
default:
|
|
fprintf(stderr, "WordParser::ParseOrAnd: unexpected operator %d\n", op);
|
|
exit(1);
|
|
break;
|
|
}
|
|
free(scope);
|
|
ParseOperands(terms, WORD_TREE_MANY, expr);
|
|
return expr;
|
|
}
|
|
|
|
void WordParser::ParseOperands(StringList& terms, int info, WordTreeOperand* expr)
|
|
{
|
|
unsigned int operands_length = 0;
|
|
unsigned int operands_size = 1;
|
|
WordTree **operands = (WordTree**)malloc(operands_size * sizeof(WordTree*));
|
|
WordTree *subexpr = 0;
|
|
while((subexpr = ParseExpr(terms))) {
|
|
operands_length++;
|
|
if((info & WORD_TREE_ONE) && operands_length > 1) {
|
|
fprintf(stderr, "WordParser::ParseOperands: expected only one operands\n");
|
|
exit(1);
|
|
} else if((info & WORD_TREE_TWO) && operands_length > 2) {
|
|
fprintf(stderr, "WordParser::ParseOperands: expected only two operands\n");
|
|
exit(1);
|
|
}
|
|
if(operands_length > operands_size) {
|
|
operands_size = operands_length * 2;
|
|
operands = (WordTree**)realloc(operands, operands_size * sizeof(WordTree*));
|
|
}
|
|
operands[operands_length - 1] = subexpr;
|
|
}
|
|
//
|
|
// Discard close parenthesis
|
|
//
|
|
if(strcmp(Term(terms), ")")) {
|
|
fprintf(stderr, "WordParser::ParseOperands: expected close parenthesis\n");
|
|
exit(1);
|
|
}
|
|
Shift(terms);
|
|
|
|
expr->cursors = operands;
|
|
expr->cursors_length = operands_length;
|
|
}
|
|
|
|
WordTree *WordParser::ParseLiteral(StringList& terms)
|
|
{
|
|
char* term = strdup(Term(terms));
|
|
char* scope = 0;
|
|
if(!mystrcasecmp(term, "literal")) {
|
|
Shift(terms);
|
|
scope = ParseScope(terms);
|
|
free(term);
|
|
term = strdup(Term(terms));
|
|
Shift(terms);
|
|
} else {
|
|
scope = strdup("");
|
|
}
|
|
WordTreeLiteral *expr = new WordTreeLiteral(term, scope);
|
|
Shift(terms);
|
|
free(scope);
|
|
free(term);
|
|
return expr;
|
|
}
|
|
|
|
char *WordParser::ParseScope(StringList& terms)
|
|
{
|
|
char *scope = Term(terms);
|
|
int scope_length = strlen(scope);
|
|
|
|
//
|
|
// Remove surrounding quotes, if any
|
|
//
|
|
if(scope_length > 0) {
|
|
if(scope[scope_length - 1] == '"')
|
|
scope[--scope_length] = '\0';
|
|
if(scope[0] == '"')
|
|
scope++;
|
|
}
|
|
|
|
scope = strdup(scope);
|
|
|
|
Shift(terms);
|
|
|
|
return scope;
|
|
}
|
|
|
|
char *WordParser::Term(StringList& terms)
|
|
{
|
|
char *term = terms[0];
|
|
if(!term) {
|
|
fprintf(stderr, "WordParser::Term: unexpected end of expression\n");
|
|
exit(1);
|
|
}
|
|
return term;
|
|
}
|
|
|
|
void WordParser::Shift(StringList& terms)
|
|
{
|
|
terms.Shift(LIST_REMOVE_DESTROY);
|
|
}
|
|
|
|
// ************************* main loop implementation ********************
|
|
|
|
//
|
|
// Store all options from the command line
|
|
//
|
|
class params_t
|
|
{
|
|
public:
|
|
char* dbfile;
|
|
char* find;
|
|
unsigned int bottom;
|
|
unsigned int count;
|
|
char* context;
|
|
int uniq_server;
|
|
int proximity;
|
|
int nop;
|
|
int exclude;
|
|
};
|
|
|
|
//
|
|
// Explain options
|
|
//
|
|
static void usage();
|
|
//
|
|
// Torture WordExclude* classes
|
|
//
|
|
static void exclude_test();
|
|
|
|
int main(int ac, char **av)
|
|
{
|
|
int c;
|
|
extern char *optarg;
|
|
params_t params;
|
|
|
|
params.dbfile = strdup("test");
|
|
params.find = 0;
|
|
params.bottom = 0;
|
|
params.count = 10;
|
|
params.context = 0;
|
|
params.uniq_server = 0;
|
|
params.proximity = WORD_SEARCH_DEFAULT_PROXIMITY;
|
|
params.nop = 0;
|
|
params.exclude = 0;
|
|
|
|
while ((c = getopt(ac, av, "vB:f:b:c:C:SP:ne")) != -1)
|
|
{
|
|
switch (c)
|
|
{
|
|
case 'v':
|
|
verbose++;
|
|
break;
|
|
case 'B':
|
|
free(params.dbfile);
|
|
params.dbfile = strdup(optarg);
|
|
break;
|
|
case 'f':
|
|
params.find = strdup(optarg);
|
|
break;
|
|
case 'b':
|
|
params.bottom = (unsigned int)atoi(optarg);
|
|
break;
|
|
case 'c':
|
|
params.count = (unsigned int)atoi(optarg);
|
|
break;
|
|
case 'C':
|
|
params.context = strdup(optarg);
|
|
break;
|
|
case 'P':
|
|
params.proximity = atoi(optarg);
|
|
break;
|
|
case 'S':
|
|
params.uniq_server = SERVER;
|
|
break;
|
|
case 'n':
|
|
params.nop = 1;
|
|
break;
|
|
case 'e':
|
|
params.exclude = 1;
|
|
break;
|
|
case '?':
|
|
usage();
|
|
break;
|
|
}
|
|
}
|
|
|
|
if(params.exclude) {
|
|
exclude_test();
|
|
exit(0);
|
|
}
|
|
|
|
if(!params.find)
|
|
usage();
|
|
|
|
Configuration* config = WordContext::Initialize();
|
|
if(!config) {
|
|
fprintf(stderr, "search: no config file found\n");
|
|
exit(1);
|
|
}
|
|
|
|
//
|
|
// Forward command line verbosity to htword library.
|
|
//
|
|
if(verbose > 1) {
|
|
String tmp;
|
|
tmp << (verbose - 1);
|
|
config->Add("wordlist_verbose", tmp);
|
|
}
|
|
|
|
//
|
|
// Prepare the index (-B).
|
|
//
|
|
WordList words(*config);
|
|
words.Open(params.dbfile, O_RDONLY);
|
|
|
|
//
|
|
// Try the query parser alone
|
|
//
|
|
if(params.nop) {
|
|
WordTree* expr = WordParser().Parse(params.find);
|
|
printf("%s\n", (char*)expr->Get());
|
|
exit(0);
|
|
}
|
|
|
|
//
|
|
// Build a syntax tree from the expression provided by user
|
|
//
|
|
WordTree* expr = WordParser().Parse(params.find);
|
|
|
|
//
|
|
// Define the semantic of the key
|
|
//
|
|
{
|
|
#define DOCUMENT_LENGTH 3
|
|
static int document[DOCUMENT_LENGTH] = {
|
|
TAG,
|
|
SERVER,
|
|
URL
|
|
};
|
|
int document_length = DOCUMENT_LENGTH;
|
|
int location = LOCATION;
|
|
if(expr->Prepare(&words, params.uniq_server, params.proximity, document, document_length, location) != OK)
|
|
exit(1);
|
|
}
|
|
|
|
WordSearch* search = new WordSearch();
|
|
|
|
//
|
|
// Forward query options to WordSearch object
|
|
//
|
|
search->limit_bottom = params.bottom; // -b
|
|
search->limit_count = params.count; // -c
|
|
if(params.context) // -C
|
|
search->context_in.set(params.context, strlen(params.context));
|
|
|
|
//
|
|
// Perform the search (-f)
|
|
//
|
|
search->expr = expr;
|
|
WordMatch* matches = search->Search();
|
|
|
|
//
|
|
// Display results, if any.
|
|
//
|
|
if(matches) {
|
|
int i;
|
|
for(i = 0; !matches[i].match.Empty(); i++)
|
|
printf("match: %s\n", (char*)matches[i].Get());
|
|
const String& context = search->Context();
|
|
if(!context.empty())
|
|
printf("context: %s\n", (const char*)context);
|
|
delete [] matches;
|
|
} else {
|
|
printf("match: none\n");
|
|
}
|
|
|
|
//
|
|
// Cleanup
|
|
//
|
|
delete search;
|
|
if(params.context) free(params.context);
|
|
if(params.find) free(params.find);
|
|
if(params.dbfile) free(params.dbfile);
|
|
delete expr;
|
|
|
|
words.Close();
|
|
delete config;
|
|
}
|
|
|
|
static void exclude_test()
|
|
{
|
|
static unsigned int expected[] = {
|
|
0x00000001,
|
|
0x00000002,
|
|
0x00000004,
|
|
0x00000008,
|
|
0x00000010,
|
|
0x00000003,
|
|
0x00000005,
|
|
0x00000006,
|
|
0x00000009,
|
|
0x0000000a,
|
|
0x0000000c,
|
|
0x00000011,
|
|
0x00000012,
|
|
0x00000014,
|
|
0x00000018,
|
|
0x00000007,
|
|
0x0000000b,
|
|
0x0000000d,
|
|
0x0000000e,
|
|
0x00000013,
|
|
0x00000015,
|
|
0x00000016,
|
|
0x00000019,
|
|
0x0000001a,
|
|
0x0000001c,
|
|
0x0000000f,
|
|
0x00000017,
|
|
0x0000001b,
|
|
0x0000001d,
|
|
0x0000001e,
|
|
0x0000001f
|
|
};
|
|
|
|
//
|
|
// WordExclude
|
|
//
|
|
if(verbose) fprintf(stderr, "exclude_test: testing WordExclude\n");
|
|
{
|
|
WordExclude exclude;
|
|
exclude.Initialize(5);
|
|
int count = 0;
|
|
while(exclude.Next() == WORD_EXCLUDE_OK) {
|
|
if(expected[count] != exclude.Mask()) {
|
|
fprintf(stderr, "exclude_test: WordExclude iteration %d expected 0x%08x but got 0x%08x\n", count, expected[count], exclude.Mask());
|
|
exit(1);
|
|
}
|
|
count++;
|
|
}
|
|
if(count != sizeof(expected)/sizeof(unsigned int)) {
|
|
fprintf(stderr, "exclude_test: WordExclude expected %d iterations but got %d\n", (int)(sizeof(expected)/sizeof(unsigned int)), count);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
//
|
|
// WordExcludeMask without ignore bits behaves exactly the same
|
|
// as WordExclude.
|
|
//
|
|
if(verbose) fprintf(stderr, "exclude_test: testing WordExcludeMask behaving like WordExclude\n");
|
|
{
|
|
WordExcludeMask exclude;
|
|
exclude.Initialize(5, 0, 0);
|
|
int count = 0;
|
|
while(exclude.Next() == WORD_EXCLUDE_OK) {
|
|
if(expected[count] != exclude.Mask()) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 1 iteration %d expected 0x%08x but got 0x%08x\n", count, expected[count], exclude.Mask());
|
|
exit(1);
|
|
}
|
|
count++;
|
|
}
|
|
if(count != sizeof(expected)/sizeof(unsigned int)) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 1 expected %d iterations but got %d\n", (int)(sizeof(expected)/sizeof(unsigned int)), count);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
//
|
|
// WordExcludeMask
|
|
//
|
|
if(verbose) fprintf(stderr, "exclude_test: testing WordExcludeMask\n");
|
|
{
|
|
static unsigned int expected[] = {
|
|
0x00000102,
|
|
0x00000108,
|
|
0x00000120,
|
|
0x00000180,
|
|
0x0000010a,
|
|
0x00000122,
|
|
0x00000128,
|
|
0x00000182,
|
|
0x00000188,
|
|
0x000001a0,
|
|
0x0000012a,
|
|
0x0000018a,
|
|
0x000001a2,
|
|
0x000001a8,
|
|
0x000001aa
|
|
};
|
|
static unsigned int excluded[] = {
|
|
1,
|
|
0,
|
|
0,
|
|
0,
|
|
1,
|
|
1,
|
|
0,
|
|
1,
|
|
0,
|
|
0,
|
|
1,
|
|
1,
|
|
1,
|
|
0,
|
|
1
|
|
};
|
|
|
|
WordExcludeMask exclude;
|
|
unsigned int ignore = 0x155;
|
|
unsigned int ignore_mask = 0x100;
|
|
exclude.Initialize(9, ignore, ignore_mask);
|
|
if(verbose) {
|
|
fprintf(stderr, "exclude_test: ignore\n");
|
|
show_bits(ignore);
|
|
fprintf(stderr, "exclude_test: ignore_mask\n");
|
|
show_bits(ignore_mask);
|
|
}
|
|
if(exclude.NotExcludedCount() != 8) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 2 expected NoExcludedCount = 8 but got %d\n", exclude.NotExcludedCount());
|
|
exit(1);
|
|
}
|
|
int count = 0;
|
|
while(exclude.Next() == WORD_EXCLUDE_OK) {
|
|
if(expected[count] != exclude.Mask()) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 2 iteration %d expected 0x%08x but got 0x%08x\n", count, expected[count], exclude.Mask());
|
|
exit(1);
|
|
}
|
|
//
|
|
// Test Excluded() method on ignored bit
|
|
// Is bit 5 set ? (9 - 4) = 5 (counting from 1)
|
|
//
|
|
if(exclude.Excluded(4)) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 2 iteration %d bit 5 was set 0x%08x\n", count, exclude.Mask());
|
|
exit(1);
|
|
}
|
|
//
|
|
// Test Excluded() method on variable bit
|
|
// Is bit 2 set ? (9 - 2) = 7 (counting from 1)
|
|
//
|
|
if((exclude.Excluded(7) && !excluded[count]) ||
|
|
(!exclude.Excluded(7) && excluded[count])) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 2 iteration %d expected bit 2 %s but was %s in 0x%08x\n", count, (excluded[count] ? "set" : "not set"), (excluded[count] ? "not set" : "set"), expected[count]);
|
|
exit(1);
|
|
}
|
|
count++;
|
|
}
|
|
if(count != sizeof(expected)/sizeof(unsigned int)) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask 2 expected %d iterations but got %d\n", (int)(sizeof(expected)/sizeof(unsigned int)), count);
|
|
exit(1);
|
|
}
|
|
}
|
|
|
|
{
|
|
WordExclude exclude;
|
|
String ascii("110101");
|
|
String tmp;
|
|
exclude.Set(ascii);
|
|
exclude.Get(tmp);
|
|
if(tmp != ascii) {
|
|
fprintf(stderr, "exclude_test: WordExclude::Get/Set expected %s but got %s\n", (char*)ascii, (char*)tmp);
|
|
exit(1);
|
|
}
|
|
if(exclude.Mask() != 0x2b) {
|
|
fprintf(stderr, "exclude_test: WordExclude::Mask expected 0x2b but got 0x%02x\n", exclude.Mask());
|
|
exit(1);
|
|
}
|
|
}
|
|
{
|
|
WordExcludeMask exclude;
|
|
String ascii("12031");
|
|
String tmp;
|
|
exclude.Set(ascii);
|
|
exclude.Get(tmp);
|
|
if(tmp != ascii) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask::Get/Set expected %s but got %s\n", (char*)ascii, (char*)tmp);
|
|
exit(1);
|
|
}
|
|
if(exclude.Mask() != 0x19) {
|
|
fprintf(stderr, "exclude_test: WordExcludeMask::Mask expected 0x19 but got 0x%02x\n", exclude.Mask());
|
|
exit(1);
|
|
}
|
|
}
|
|
}
|
|
|
|
// *****************************************************************************
|
|
// void usage()
|
|
// Display program usage information
|
|
//
|
|
static void usage()
|
|
{
|
|
printf("usage:\tsearch -f words [options]\n");
|
|
printf("\tsearch -e\n");
|
|
printf("Options:\n");
|
|
printf("\t-v\t\tIncreases the verbosity.\n");
|
|
printf("\t-B dbfile\tUse <dbfile> as a db file name (default test).\n");
|
|
printf("\t-f expr\t\tLisp like search expression.\n");
|
|
printf("\t\t\tSee WordParser comments in source for more information.\n");
|
|
printf("\t-b number\tSkip number documents before retrieving.\n");
|
|
printf("\t-c number\tRetrieve number documents at most.\n");
|
|
printf("\t-n\t\tOnly parse the search expression and print it.\n");
|
|
printf("\t-P proximity\tUse with near/optional, proximity tolerance is <proximity>\n");
|
|
printf("\t\t\tif negative order of terms is not meaningful\n");
|
|
printf("\t\t\t(default 1).\n");
|
|
printf("\t-C context\tResume search at <context>.\n");
|
|
printf("\t-S\t\tReturn at most one match per server.\n");
|
|
printf("\n");
|
|
printf("\t-e\t\tRun tests on WordExclude and WordExcludeMask.\n");
|
|
exit(1);
|
|
}
|