FreqCounter.hpp

Go to the documentation of this file.

/*==========================================================================
 * Copyright (c) 2001 Carnegie Mellon University.  All Rights Reserved.
 *
 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval
 * is subject to the terms of the software license set forth in the LICENSE
 * file included with this software, and also available at
 * http://www.lemurproject.org/license.html
 *
 *==========================================================================
 */

#ifndef _FREQCOUNTER_HPP
#define _FREQCOUNTER_HPP

#include <map>
#include <set>
#include "InvFPTypes.hpp"
#include "TextHandler.hpp"
#include "Stopper.hpp"
namespace lemur 
{
  namespace distrib 
  {
    
#define R_CTF 0

#define R_DF 1

#define R_AVE_TF 2

#define R_UNIFORM 3

    typedef struct freqinfo_tt {
      char * word;
      int ctf;
      int df;
    } freqinfo_t;

    typedef map<std::string , freqinfo_t, less<std::string> > freqmap;
    typedef set<std::string , less<std::string> > stringset;


    class FreqCounter : public lemur::api::TextHandler {

    public:
      FreqCounter(const lemur::api::Stopper * stopWords = NULL);
      FreqCounter(const string &filename, const lemur::api::Stopper * stopWords = NULL);
  
      ~FreqCounter();

      void clear();

      void output(const string &filename) const;

      char * randomWord();
      void setRandomMode(int mode);
      int getRandomMode() const;

      char * randomCtf() const;
      char * randomDf() const;
      char * randomAveTf() const;
      char * randomUniform() const;


      int numWords() const;
      int totWords() const;

      const freqmap * getFreqInfo() const;

      int getCtf(const char * word) const;
      int getDf(const char * word) const;
      double getAveTf(const char * word) const;

      double ctfRatio(FreqCounter & lm1) const;

      char * handleDoc(char * docno);
      char * handleWord(char * word);

      void endDoc();

      void setName(const string &freqCounterName);
      const string & getName() const;

      void pruneBottomWords(int topWords);
  

    protected:
      /* Loads a language model from file. */
      void input(const string &filename);

      /* Collection term frequencies. */
      mutable freqmap freqInfo;

      /* Words in a doc. */
      stringset doc;
      /* Random words returned so far. */
      stringset randdone;

      /* The frequency counter's name. */
      string name;

      /* Stopword list */
      const lemur::api::Stopper * stopper;


      /* used for calculating probabilities when
       * selecting a random word
       */
      /* Sum over words of ctf. */
      long ctfTot;
      /* Sum over words of df. */
      int dfTot;  
      /* Sum over words of average tf. */
      mutable long double avetfTot;
      /* Indicates whether avetfTot is valid (true)
       * or needs to be recalculated (false). */
      mutable bool atfValid;
      /* Random selection mode. */
      int randomMode;
      /* Number of unique words. */
      int nWords;


    };
 
  }
}


#endif

---