BasicDocStream.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */


#ifndef _BASICFILESTREAM_HPP
#define _BASICFILESTREAM_HPP
#include "common_headers.hpp"
#include <cassert>
#include <cstdio>
#include <cstring>
#include "DocStream.hpp"
#include "Exception.hpp"


namespace lemur 
{
  namespace parse 
  {
    

#define MAXLINE 65536



    class BasicTokenDoc : public lemur::api::Document {
    public:
      BasicTokenDoc() {
      }
      BasicTokenDoc(ifstream *stream): docStr(stream) {
      }
      void startTermIteration() const;  
  
      const char *getID() const { return id;}

      bool hasMore() const{ return (strcmp(curWord, "</DOC>") != 0);}
    
      const lemur::api::Term * nextTerm() const;

      void skipToEnd() const;
      friend class BasicDocStream;
    private:
      void readID(); 
      mutable char *curWord;
      mutable char buf1[20000];
      mutable char buf2[20000];
      char id[2000];
      ifstream *docStr;
      streampos startPos; // starting position of the terms in the file
      //replace  static BasicTokenTerm t; with attribute
      mutable lemur::api::Term t;
    };


    class BasicDocStream : public lemur::api::DocStream
    {
    public:
      BasicDocStream() {}
      BasicDocStream (const string &inputFile);

      virtual ~BasicDocStream() {  delete ifs;}

    public:
        
      bool hasMore(); 

      void startDocIteration();

      lemur::api::Document *nextDoc();

    private:
      char file[1024];
      ifstream *ifs;
      char buf[2000];
      bool nextTokenRead;
      // replace static BasicTokenDoc doc;  with attribute
      BasicTokenDoc doc;
    };
  }
}
#endif

---