IndexEnvironment.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2004 University of Massachusetts.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


//
// IndexEnvironment
//
// 19 July 2004 -- tds
//

#ifndef INDRI_INDEXENVIRONMENT_HPP
#define INDRI_INDEXENVIRONMENT_HPP

#include <string>
#include "indri/Parameters.hpp"
#include "indri/HTMLParser.hpp"
#include "indri/ConflationPattern.hpp"
#include "indri/Repository.hpp"
#include "indri/IndriParser.hpp"
#include "indri/IndriTokenizer.hpp"
#include "indri/DocumentIterator.hpp"
#include "indri/AnchorTextAnnotator.hpp"
#include "indri/OffsetAnnotationAnnotator.hpp"
#include "indri/OffsetMetadataAnnotator.hpp"
#include "indri/Transformation.hpp"
#include "indri/DocumentIteratorFactory.hpp"
#include "indri/ParserFactory.hpp"
#include "indri/FileClassEnvironmentFactory.hpp"
#include <map>
namespace indri 
{
  namespace api 
  {
    
    struct IndexStatus {
      enum action_code {
        FileOpen,
        FileSkip,
        FileError,
        FileClose,
        DocumentCount
      };

      virtual void operator () ( int code, const std::string& documentPath, const std::string& error, int documentsIndexed, int documentsSeen ) {
        status( code, documentPath, error, documentsIndexed, documentsSeen );
      }

      virtual void status( int code, const std::string& documentPath, const std::string& error, int documentsIndexed, int documentsSeen ) {};
    };

    class IndexEnvironment {
    private:
      IndexStatus* _callback;
      Parameters* _options;

      std::string _repositoryPath;
      indri::collection::Repository _repository;
      int _documents;
      std::string _error;

      std::string _offsetAnnotationsRoot;
      std::string _offsetMetadataRoot;
      std::string _anchorTextRoot;
      std::string _documentRoot;

      Parameters _parameters;
      indri::parse::FileClassEnvironmentFactory _fileClassFactory;

      indri::parse::AnchorTextAnnotator _annotator;
      indri::parse::OffsetAnnotationAnnotator _oa_annotator;
      indri::parse::OffsetMetadataAnnotator _om_annotator;

      std::map<std::string, indri::parse::FileClassEnvironment*> _environments;

      int _documentsIndexed;
      int _documentsSeen;

      void _getParsingContext( indri::parse::Parser** parser,
                               indri::parse::Tokenizer** tokenizer,
                               indri::parse::DocumentIterator** iterator,
                               indri::parse::Conflater** conflater,
                               const std::string& extension );

      std::vector<indri::parse::Transformation*> _createAnnotators( const std::string& fileName, 
                                                                    const std::string& fileClass, 
                                                                    indri::parse::Conflater** conflater);

      ParsedDocument* _applyAnnotators( std::vector<indri::parse::Transformation*>& annotators, 
                                        ParsedDocument* parsed ); 


    public:
      friend class QueryEnvironment;

      IndexEnvironment();
      ~IndexEnvironment();

      void setOffsetAnnotationsPath( const std::string& offsetAnnotationsRoot );

      void setOffsetMetadataPath( const std::string& offsetMetadataRoot );

      void setAnchorTextPath( const std::string& anchorTextRoot );

      void setDocumentRoot( const std::string& documentRoot );

      void addFileClass( const std::string& name, 
                         const std::string& iterator,
                         const std::string& parser,
                         const std::string& tokenizer,
                         const std::string& startDocTag,
                         const std::string& endDocTag,
                         const std::string& endMetadataTag,
                         const std::vector<std::string>& include,
                         const std::vector<std::string>& exclude,
                         const std::vector<std::string>& index,
                         const std::vector<std::string>& metadata, 
                         const std::map<indri::parse::ConflationPattern*,std::string>& conflations );

      indri::parse::FileClassEnvironmentFactory::Specification *getFileClassSpec( const std::string& name) {
        return _fileClassFactory.getFileClassSpec(name);
      }

      void addFileClass( const indri::parse::FileClassEnvironmentFactory::Specification &spec ){
        _fileClassFactory.addFileClass(spec);
      }
  
      void setIndexedFields( const std::vector<std::string>& fieldNames );

      void setNumericField( const std::string& fieldName, bool isNumeric,
                            const std::string &parserName = "");

      void setOrdinalField( const std::string& fieldName, bool isOrdinal);

      void setParentalField( const std::string& fieldName, bool isParental);


      void setMetadataIndexedFields( const std::vector<std::string>& forwardFieldNames, const std::vector<std::string>& backwardFieldNames );

      void setStopwords( const std::vector<std::string>& stopwords );

      void setStemmer( const std::string& stemmer );

      void setMemory( UINT64 memory );

      void setNormalization( bool flag );

      void setStoreDocs( bool flag );

      void setOffsetAnnotationIndexHint(indri::parse::OffsetAnnotationIndexHint hintType);

      void create( const std::string& repositoryPath, IndexStatus* callback = 0 );

      void open( const std::string& repositoryPath, IndexStatus* callback = 0 );

      void close();
  
      void addFile( const std::string& fileName );

      void addFile( const std::string& fileName, const std::string& fileClass );

      lemur::api::DOCID_T addString( const std::string& documentString, 
                     const std::string& fileClass, 
                     const std::vector<indri::parse::MetadataPair>& metadata );

      lemur::api::DOCID_T addString( const std::string& documentString, 
                     const std::string& fileClass, 
                     const std::vector<indri::parse::MetadataPair>& metadata, 
                     const std::vector<indri::parse::TagExtent *> &tags );
      
      lemur::api::DOCID_T addParsedDocument( ParsedDocument* document );

      void deleteDocument( lemur::api::DOCID_T documentID );

      int documentsIndexed();

      int documentsSeen();

      void compact();

      static void merge( const std::string& outputIndex, const std::vector<std::string>& inputIndexes );
    };
  }
}

#endif // INDRI_INDEXENVIRONMENT_HPP

---