KeyfileIncIndex.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


#ifndef _LEMUR_KEYFILE_INCINDEX_HPP
#define _LEMUR_KEYFILE_INCINDEX_HPP

/*
 * NAME DATE - COMMENTS
 * tnt 01/02 - created
 * dmf 07/03 - converted to incremental berkeley db btree indexer with
 * integrated document manager.
 * tds 09/03 - modified from BTIncIndex to use keyfile
 * dmf 12/03 - update to 2.1 API and remove parser/docmgr components.
 */
#include "common_headers.hpp"
#include "Index.hpp"
#include "InvFPDocList.hpp"
#include "InvFPTermList.hpp"
#include "InvFPTypes.hpp"
#include "BasicCollectionProps.hpp"
#include "Param.hpp"
#include "PushIndex.hpp"
#include "MemCache.hpp"
#include "Keyfile.hpp"
#include "KeyfileDocMgr.hpp"
#include "ReadBuffer.hpp"
#include "WriteBuffer.hpp"
#include "TermCache.hpp"
#include <cstring>
#include <queue>
namespace lemur
{
  namespace index 
  {
    
    // for counts array
#define UNIQUE_TERMS 0
#define TOTAL_TERMS  1
#define DOCS         2
#define DT_FILES     3
#define INV_FILES    4
    // keyref.h -- 512
#define MAX_DOCID_LENGTH 512
#define MAX_TERM_LENGTH 512

#define KEYFILE_MAX_SEGMENTS (16)

    // we love multiple inheritance

    class KeyfileIncIndex : public PushIndex, public lemur::api::Index {
    public:
      class record {
      public:
        lemur::file::File::offset_type offset;
        int len;
        int totalLen;
        int num;     
      };
      struct SegmentOffset {
        unsigned int segment;
        unsigned int length;
        lemur::file::File::offset_type offset;
      };
      struct TermData {
        lemur::api::COUNT_T totalCount;
        lemur::api::COUNT_T documentCount;
        SegmentOffset segments[ KEYFILE_MAX_SEGMENTS ];
      };
      KeyfileIncIndex(const string &prefix, int cachesize=128000000, 
                      lemur::api::DOCID_T startdocid=1);
      KeyfileIncIndex();
      ~KeyfileIncIndex();

      void setName(const string &prefix);

      bool beginDoc(const lemur::parse::DocumentProps* dp);

      bool addTerm(const lemur::api::Term& t);

      void endDoc(const lemur::parse::DocumentProps* dp);

      virtual void endDoc(const lemur::parse::DocumentProps* dp, const string &mgr);

      void endCollection(const lemur::parse::CollectionProps* cp);

      void setDocManager(const string &mgrID);
    
    protected:
      bool tryOpen();
      void writeTOC(const lemur::parse::CollectionProps* cp);
      void writeCache( bool lastRun = false );
      void lastWriteCache();

      void mergeCacheSegments();
      void writeCacheSegment();
      void writeDocMgrIDs();
      int docMgrID(const string &mgr);
      virtual void doendDoc(const lemur::parse::DocumentProps* dp, int mgrid);
      int listlengths;
  
    public:


      bool open(const string &indexName);



      lemur::api::TERMID_T term(const lemur::api::TERM_T &word) const;

      const lemur::api::TERM_T term(lemur::api::TERMID_T termID) const;

      lemur::api::DOCID_T document(const lemur::api::EXDOCID_T &docIDStr) const;

      const lemur::api::EXDOCID_T document(lemur::api::DOCID_T docID) const; 

      const lemur::api::DocumentManager *docManager(lemur::api::DOCID_T docID) const;

      const lemur::parse::CollectionProps *collectionProps() const;



      lemur::api::COUNT_T docCount() const { return counts[DOCS]; };

      lemur::api::COUNT_T termCountUnique() const { return counts[UNIQUE_TERMS]; };

      lemur::api::COUNT_T termCount(lemur::api::TERMID_T termID) const;

      lemur::api::COUNT_T termCount() const { return counts[TOTAL_TERMS]; };

      float docLengthAvg() const;

      lemur::api::COUNT_T docCount(lemur::api::TERMID_T termID) const;

      lemur::api::COUNT_T docLength(lemur::api::DOCID_T docID) const;

      virtual lemur::api::COUNT_T totaldocLength (lemur::api::DOCID_T docID) const;

      lemur::api::COUNT_T docLengthCounted(lemur::api::DOCID_T docID) const;




      lemur::api::DocInfoList* docInfoList(lemur::api::TERMID_T termID) const;

      lemur::api::TermInfoList* termInfoList(lemur::api::DOCID_T docID) const;
      lemur::api::TermInfoList* termInfoListSeq(lemur::api::DOCID_T docID) const;


      void setMesgStream(ostream * lemStream);
      void addKnownTerm( lemur::api::TERMID_T termID, lemur::api::LOC_T position );
      lemur::api::TERMID_T addUnknownTerm( const InvFPTerm* term );
      lemur::api::TERMID_T addUncachedTerm( const InvFPTerm* term );

    protected:
      void openDBs();
      void openSegments();
      void createDBs();

      void fullToc();
      bool docMgrIDs();
      record fetchDocumentRecord( lemur::api::DOCID_T key ) const;
      void addDocumentLookup( lemur::api::DOCID_T documentKey, const char* documentName );
      void addTermLookup( lemur::api::TERMID_T termKey, const char* termSpelling );
      void addGeneralLookup( lemur::file::Keyfile& numberNameIndex, 
                             lemur::file::Keyfile& nameNumberIndex, 
                             lemur::api::TERMID_T number, const char* name );
      InvFPDocList* internalDocInfoList(lemur::api::TERMID_T termID) const;
      void _updateTermlist( InvFPDocList* curlist, lemur::api::LOC_T position );
      int _cacheSize();
      void _computeMemoryBounds( int memorySize );
      void _resetEstimatePoint();
      lemur::api::COUNT_T* counts;    
      std::vector<std::string> names;
      float aveDocLen; 
      vector<std::string> docmgrs;
      ostream* msgstream;

      // All database handles are marked mutable since they sometimes
      // must be used to fetch values during const methods
      mutable lemur::file::Keyfile invlookup;
  
      // int <-> string mappings for documents and terms
      mutable lemur::file::Keyfile dIDs;
      mutable lemur::file::Keyfile dSTRs;
      mutable lemur::file::Keyfile tIDs;
      mutable lemur::file::Keyfile tSTRs;
      mutable lemur::file::File dtlookup; 
      lemur::file::ReadBuffer* dtlookupReadBuffer; 
      mutable lemur::file::File writetlist; 

      mutable char termKey[MAX_TERM_LENGTH];
      mutable char docKey[MAX_DOCID_LENGTH];
      int _listsSize;
      int _memorySize;
      std::string name;
      vector<InvFPDocList*> invertlists; 
      vector<LocatedTerm> termlist; 
      int curdocmgr; 
      vector<lemur::api::DocumentManager*> docMgrs; 
      lemur::utility::TermCache _cache;
      mutable lemur::parse::BasicCollectionProps* cprops;

      std::vector<lemur::file::File*> _segments;
      lemur::api::TERMID_T _largestFlushedTermID;
      int _estimatePoint; 
      bool ignoreDoc;  
      bool _readOnly;
    };
  }
}


#endif //_LEMUR_KEYFILE_INCINDEX_HPP

---