9#include <boost/tokenizer.hpp>
26 const Xapian::Database& iDatabase)
27 : _resultHolder (NULL), _database (iDatabase),
28 _queryString (iQueryString), _hasFullTextMatched (false),
43 std::ostringstream oStr;
50 std::ostringstream oStr;
52 if (_correctedQueryString.empty() ==
false
53 && _correctedQueryString != _queryString) {
54 oStr <<
"(corrected into '" << _correctedQueryString
55 <<
"' with an edit distance/error of " << _editDistance
56 <<
" over an allowable distance of " << _allowableEditDistance
66 std::ostringstream oStr;
69 if (_documentList.empty() ==
true) {
70 oStr <<
"No match" << std::endl;
73 assert (_hasFullTextMatched ==
true);
75 unsigned short idx = 0;
76 for (DocumentList_T::const_iterator itDoc = _documentList.begin();
77 itDoc != _documentList.end(); ++itDoc, ++idx) {
80 const Xapian::Document& lXapianDoc = lDocumentPair.first;
81 const Xapian::docid& lDocID = lXapianDoc.get_docid();
83 const ScoreBoard& lScoreBoard = lDocumentPair.second;
88 oStr <<
"Doc ID: " << lDocID <<
", matching with ("
89 << lScoreBoard.
describe() <<
"), containing: '"
90 << lXapianDoc.get_data() <<
"'";
107 getDocumentPair (
const Xapian::docid& iDocID)
const {
110 DocumentMap_T::const_iterator itDoc = _documentMap.find (iDocID);
112 if (itDoc == _documentMap.end()) {
114 <<
") can not be found in the Result object "
117 assert (itDoc != _documentMap.end());
123 return oDocumentPair;
127 const Xapian::Document& Result::
128 getDocument (
const Xapian::docid& iDocID)
const {
134 const Xapian::Document& oXapianDocument = lDocumentPair.first;
137 return oXapianDocument;
151 Score_T lCorrectedScore = iScore;
152 if (_editDistance > 0) {
153 lCorrectedScore = iScore / (_editDistance * _editDistance * _editDistance);
161 lXapianScoreType, lCorrectedScore);
164 const Xapian::docid& lDocID = iDocument.get_docid();
180 _documentList.push_back (lDocumentPair);
183 const bool hasInsertBeenSuccessful =
184 _documentMap.insert (DocumentMap_T::value_type (lDocID,
185 lDocumentPair)).second;
187 if (hasInsertBeenSuccessful ==
false) {
188 std::ostringstream errorStr;
189 errorStr <<
"Error while inserting the Xapian Document pair into "
190 <<
"the internal STL map";
193 assert (hasInsertBeenSuccessful ==
true);
202 for (Xapian::MSetIterator itDoc = iMatchingSet.begin();
203 itDoc != iMatchingSet.end(); ++itDoc) {
204 const int& lXapianPercentage = itDoc.get_percent();
205 const Xapian::Document& lDocument = itDoc.get_document();
232 <<
", " << _bestCombinedWeight <<
"% [" << _bestDocData
254 return oEditDistance;
274 const std::string& lDocumentDataStr = iDocument.get_data();
303 const Score_T oEnvelopeID =
static_cast<const Score_T> (lEnvelopeIDInt);
322 Xapian::MSet& ioMatchingSet,
323 const std::string& iNegativeQuery) {
324 std::string oMatchedString;
330 Xapian::QueryParser lQueryParser;
331 lQueryParser.set_database (iDatabase);
339 lQueryParser.set_default_op (Xapian::Query::OP_PHRASE);
351 Xapian::Enquire enquire (iDatabase);
352 enquire.set_weighting_scheme(Xapian::BM25Weight(1.0, 0.0, 1.0, 0.0, 0.5));
360 std::string lQueryStringToParse = iQueryString;
361 if (!iNegativeQuery.empty()) {
362 lQueryStringToParse +=
" " + iNegativeQuery;
365 const Xapian::Query& lXapianQuery =
366 lQueryParser.parse_query (lQueryStringToParse,
367 Xapian::QueryParser::FLAG_BOOLEAN
368 | Xapian::QueryParser::FLAG_PHRASE
369 | Xapian::QueryParser::FLAG_LOVEHATE);
372 enquire.set_query (lXapianQuery);
376 ioMatchingSet = enquire.get_mset (0, K_DEFAULT_XAPIAN_MATCHING_SET_SIZE);
379 int nbMatches = ioMatchingSet.size();
383 <<
"', i.e.: `" << lXapianQuery.get_description()
384 <<
"' => " << nbMatches <<
" result(s) found");
386 if (nbMatches != 0) {
388 const NbOfErrors_T lEditDistance = 0;
389 setEditDistance (lEditDistance);
392 setAllowableEditDistance (lEditDistance);
395 oMatchedString = iQueryString;
398 setHasFullTextMatched (
true);
402 setCorrectedQueryString (oMatchedString);
406 <<
"' provides " << nbMatches <<
" exact matches.");
408 return oMatchedString;
410 assert (ioMatchingSet.empty() ==
true);
421 const std::string& lCorrectedString =
422 iDatabase.get_spelling_suggestion (iQueryString, lAllowableEditDistance);
426 if (lCorrectedString.empty() ==
true || lCorrectedString == iQueryString) {
429 << iQueryString <<
"' provides no match, "
430 <<
"and there is no spelling suggestion, "
431 <<
"even with an edit distance of "
432 << lAllowableEditDistance);
435 setHasFullTextMatched (
false);
438 return oMatchedString;
440 assert (lCorrectedString.empty() ==
false
441 && lCorrectedString != iQueryString);
445 Levenshtein::getDistance (iQueryString, lCorrectedString);
454 std::string lCorrectedQueryStringToParse = lCorrectedString;
455 if (!iNegativeQuery.empty()) {
456 lCorrectedQueryStringToParse +=
" " + iNegativeQuery;
459 const Xapian::Query& lCorrectedXapianQuery =
460 lQueryParser.parse_query (lCorrectedQueryStringToParse,
461 Xapian::QueryParser::FLAG_BOOLEAN
462 | Xapian::QueryParser::FLAG_PHRASE
463 | Xapian::QueryParser::FLAG_LOVEHATE);
467 enquire.set_query (lCorrectedXapianQuery);
468 ioMatchingSet = enquire.get_mset (0, K_DEFAULT_XAPIAN_MATCHING_SET_SIZE);
471 nbMatches = ioMatchingSet.size();
476 << lCorrectedXapianQuery.get_description()
477 <<
"' => " << nbMatches <<
" result(s) found");
479 if (nbMatches != 0) {
481 setEditDistance (lEditDistance);
484 setAllowableEditDistance (lAllowableEditDistance);
487 oMatchedString = lCorrectedString;
490 setHasFullTextMatched (
true);
493 setCorrectedQueryString (oMatchedString);
497 << iQueryString <<
"', spelling suggestion: `"
499 <<
"', with a Levenshtein edit distance of "
501 <<
" over an allowable edit distance of "
502 << lAllowableEditDistance <<
", provides "
503 << nbMatches <<
" matches.");
506 return oMatchedString;
511 << iQueryString <<
"', spelling suggestion: `"
513 <<
"', with a Levenshtein edit distance of "
515 <<
" over an allowable edit distance of "
516 << lAllowableEditDistance <<
", provides no match, "
517 <<
"which is not consistent with the existence of "
518 <<
"the spelling correction.");
521 }
catch (
const Xapian::Error& error) {
523 throw XapianException (error.get_msg());
527 setHasFullTextMatched (
false);
529 return oMatchedString;
535 const std::string& iNegativeQuery) {
536 std::string oMatchedString;
549 Xapian::MSet lMatchingSet;
550 if (isToBeAdded ==
true) {
551 oMatchedString =
fullTextMatch (iDatabase, iQueryString, lMatchingSet, iNegativeQuery);
558 if (isToBeAdded ==
false) {
561 <<
"' is not made of searchable words");
566 }
catch (
const Xapian::Error& error) {
571 return oMatchedString;
577 for (DocumentList_T::const_iterator itDoc = _documentList.begin();
578 itDoc != _documentList.end(); ++itDoc) {
582 const Xapian::Document& lXapianDoc = lDocumentPair.first;
585 const Xapian::docid& lDocID = lXapianDoc.get_docid();
591 const ScoreBoard& lScoreBoard = lDocumentPair.second;
598 <<
"' with (" << lLocationKey <<
", doc ID = "
599 << lDocID <<
") matches at " << lXapianPct
609 DocumentMap_T::iterator itDoc = _documentMap.find (iDocID);
611 if (itDoc == _documentMap.end()) {
613 <<
") can not be found in the Result object "
616 assert (itDoc != _documentMap.end());
620 ScoreBoard& lScoreBoard = lXapianDocPair.second;
623 lScoreBoard.
setScore (iType, iScore);
629 for (DocumentList_T::iterator itDoc = _documentList.begin();
630 itDoc != _documentList.end(); ++itDoc) {
634 const Xapian::Document& lXapianDoc = lDocumentPair.first;
637 const Xapian::docid& lDocID = lXapianDoc.get_docid();
646 if (lEnvelopeIDInt != 0) {
648 <<
"] (" << lLocationKey <<
", doc ID = "
649 << lDocID <<
") has a non-null envelope ID ("
650 << lEnvelopeIDInt <<
") => match of 0.10%");
654 const Score_T lEnvelopeID =
static_cast<const Score_T> (lEnvelopeIDInt);
657 ScoreBoard& lScoreBoard = lDocumentPair.second;
668 for (DocumentList_T::iterator itDoc = _documentList.begin();
669 itDoc != _documentList.end(); ++itDoc) {
673 const Xapian::Document& lXapianDoc = lDocumentPair.first;
676 const Xapian::docid& lDocID = lXapianDoc.get_docid();
683 bool hasCodeFullyMatched =
false;
688 std::string lFilteredString (_queryString);
696 lFilteredQueryWordList);
697 const NbOfWords_T nbOfFilteredQueryWords = lFilteredQueryWordList.size();
700 if (_hasFullTextMatched ==
true) {
707 const size_t lNbOfLetters = lFilteredString.size();
708 if (nbOfFilteredQueryWords == 1
709 && lNbOfLetters >= 3 && lNbOfLetters <= 4
710 && _correctedQueryString == _queryString) {
713 std::string lUpperQueryWord;
714 lUpperQueryWord.resize (lNbOfLetters);
715 std::transform (lFilteredString.begin(), lFilteredString.end(),
716 lUpperQueryWord.begin(), ::toupper);
723 if (lUpperQueryWord == lIataCode) {
728 hasCodeFullyMatched =
true;
732 if (hasCodeFullyMatched ==
true) {
735 <<
"' matches the IATA/ICAO code ("
736 << lLocationKey <<
", doc ID = "
737 << lDocID <<
") => match of "
742 <<
"' does not match with the IATA/ICAO "
743 <<
"code (" << lLocationKey <<
", doc ID = "
744 << lDocID <<
") => match of "
750 ScoreBoard& lScoreBoard = lDocumentPair.second;
761 for (DocumentList_T::iterator itDoc = _documentList.begin();
762 itDoc != _documentList.end(); ++itDoc) {
766 const Xapian::Document& lXapianDoc = lDocumentPair.first;
769 const Xapian::docid& lDocID = lXapianDoc.get_docid();
779 <<
"] (" << lLocationKey <<
", doc ID = "
780 << lDocID <<
") has a PageRank of "
781 << lPageRank <<
"%");
784 ScoreBoard& lScoreBoard = lDocumentPair.second;
804 std::string lBestDocData;
807 Xapian::docid lBestDocID = 0;
808 for (DocumentList_T::iterator itDoc = _documentList.begin();
809 itDoc != _documentList.end(); ++itDoc) {
813 const Xapian::Document& lXapianDoc = lDocumentPair.first;
814 const Xapian::docid& lDocID = lXapianDoc.get_docid();
815 const std::string& lDocData = lXapianDoc.get_data();
821 ScoreBoard& lScoreBoard = lDocumentPair.second;
834 if (lPercentage > lMaxPercentage) {
835 lMaxPercentage = lPercentage;
837 lBestDocData = lDocData;
844 lOriginalQueryWordList);
845 const NbOfWords_T nbOfOriginalQueryWords = lOriginalQueryWordList.size();
848 if (_hasFullTextMatched ==
true) {
852 const Xapian::Document& lXapianDoc = lXapianDocPair.first;
853 const ScoreBoard& lScoreBoard = lXapianDocPair.second;
858 <<
"' matches at " << lMaxPercentage
859 <<
"% for " << lLocationKey <<
" (doc ID = "
860 << lBestDocID <<
"). Score calculation: "
870 if (nbOfOriginalQueryWords == 1 && shouldBeKept ==
true) {
876 lMaxPercentage = 100.0;
880 <<
"' does not match, but it is a non black-listed "
881 <<
"single-word string; hence, the weight is "
882 << lMaxPercentage <<
"%");
893 lMaxPercentage = std::pow (10.0, -3*nbOfOriginalQueryWords);
897 <<
"' does not match, and is either a multiple-word "
898 <<
"string or black-listed; hence, the weight is "
899 << lMaxPercentage <<
"%");
#define OPENTREP_LOG_ERROR(iToBeLogged)
#define OPENTREP_LOG_DEBUG(iToBeLogged)
#define OPENTREP_LOG_NOTIFICATION(iToBeLogged)
const Location & generateLocation()
Class modelling a place/POR (point of reference).
void setEditDistance(const NbOfErrors_T &iEditDistance)
void setDocID(const XapianDocID_T &iDocID)
void setAllowableEditDistance(const NbOfErrors_T &iAllowableEditDistance)
void setPercentage(const MatchingPercentage_T &iPercentage)
void setOriginalKeywords(const std::string &iOriginalKeywords)
const LocationKey & getKey() const
void setCorrectedKeywords(const std::string &iCorrectedKeywords)
void setScoreOnDocMap(const Xapian::docid &, const ScoreType &, const Score_T &)
void addDocument(const Xapian::Document &, const Score_T &)
void setBestCombinedWeight(const Percentage_T &iPercentage)
void displayXapianPercentages() const
void fillResult(const Xapian::MSet &iMatchingSet)
const XapianDocumentPair_T & getDocumentPair(const Xapian::docid &) const
void setBestDocData(const std::string &iDocData)
void fillPlace(Place &) const
void toStream(std::ostream &ioOut) const
void calculatePageRanks()
void setBestDocID(const Xapian::docid &iDocID)
static Score_T getEnvelopeID(const Xapian::Document &)
static PageRank_T getPageRank(const Xapian::Document &)
std::string toString() const
void fromStream(std::istream &ioIn)
std::string describeShortKey() const
static Location retrieveLocation(const Xapian::Document &)
std::string fullTextMatch(const Xapian::Database &, const TravelQuery_T &, const std::string &="")
void calculateHeuristicWeights()
void calculateEnvelopeWeights()
void calculateCodeMatches()
static LocationKey getPrimaryKey(const Xapian::Document &)
void calculateCombinedWeights()
std::string describeKey() const
static void tokeniseStringIntoWordList(const TravelQuery_T &, WordList_T &)
std::list< Word_T > WordList_T
unsigned int NbOfLetters_T
unsigned short NbOfErrors_T
const Percentage_T K_DEFAULT_MODIFIED_MATCHING_PCT
std::string TravelQuery_T
static unsigned int calculateEditDistance(const TravelQuery_T &iPhrase)
Helper function.
unsigned int EnvelopeID_T
const Percentage_T K_DEFAULT_FULL_CODE_MATCH_PCT
std::pair< Xapian::Document, ScoreBoard > XapianDocumentPair_T
const NbOfErrors_T K_DEFAULT_SIZE_FOR_SPELLING_ERROR_UNIT
unsigned short NbOfWords_T
static void trim(std::string &ioPhrase, const NbOfLetters_T &iMinWordLength=4)
static bool shouldKeep(const std::string &iPhrase, const std::string &iWord)
Class modelling the primary key of a location/POR (point of reference).
const IATACode_T & getIataCode() const
Structure modelling a (geographical) location.
const LocationKey & getKey() const
const EnvelopeID_T & getEnvelopeID() const
const PageRank_T & getPageRank() const
Structure holding a board for all the types of score/matching having been performed.
Percentage_T calculateCombinedWeight()
std::string describe() const
void setScore(const ScoreType &, const Score_T &)
Score_T getScore(const ScoreType &) const
Enumeration of score types.