SimpleKLRetMethod.hpp

Go to the documentation of this file.

/*==========================================================================
 *
 *  Original source copyright (c) 2001, Carnegie Mellon University.
 *  See copyright.cmu for details.
 *  Modifications copyright (c) 2002, University of Massachusetts.
 *  See copyright.umass for details.
 *
 *==========================================================================
 */


#ifndef _SIMPLEKLRETMETHOD_HPP
#define _SIMPLEKLRETMETHOD_HPP

#include <cmath>
#include "UnigramLM.hpp"
#include "ScoreFunction.hpp"
#include "SimpleKLDocModel.hpp"
#include "TextQueryRep.hpp"
#include "TextQueryRetMethod.hpp"
#include "Counter.hpp"
#include "DocUnigramCounter.hpp"

namespace lemur 
{
  namespace retrieval 
  {
    

    class SimpleKLQueryModel : public ArrayQueryRep {
    public:
      SimpleKLQueryModel(const lemur::api::TermQuery &qry, 
                         const lemur::api::Index &dbIndex) : 
        ArrayQueryRep(dbIndex.termCountUnique()+1, qry, dbIndex), qm(NULL), 
        ind(dbIndex), colKLComputed(false) {
        colQLikelihood = 0;
        colQueryLikelihood();    
      }

      SimpleKLQueryModel(const lemur::api::Index &dbIndex) : 
        ArrayQueryRep(dbIndex.termCountUnique()+1), qm(NULL), ind(dbIndex), 
        colKLComputed(false) {
        colQLikelihood = 0;
        startIteration();
        while (hasMore()) {
          lemur::api::QueryTerm *qt = nextTerm();
          setCount(qt->id(), 0);
          delete qt;
        }
      }


      virtual ~SimpleKLQueryModel(){ if (qm) delete qm;}



      virtual void interpolateWith(const lemur::langmod::UnigramLM &qModel, 
                                   double origModCoeff, 
                                   int howManyWord, double prSumThresh=1, 
                                   double prThresh=0);
      virtual double scoreConstant() const {
        return totalCount();
      }
  
      virtual void load(istream &is);

      virtual void save(ostream &os);

      virtual void clarity(ostream &os);
      virtual double clarity() const;

      double colDivergence() const {
        if (colKLComputed) {
          return colKL;
        } else {
          colKLComputed = true;
          double d=0;
          startIteration();
          while (hasMore()) {
            lemur::api::QueryTerm *qt=nextTerm();
            double pr = qt->weight()/(double)totalCount();
            double colPr = ((double)ind.termCount(qt->id()) /
                            (double)(ind.termCount())); // ML smoothing
            d += pr*log(pr/colPr);
            delete qt;
          }
          colKL=d;
          return d;
        }
      }

      double KLDivergence(const lemur::langmod::UnigramLM &refMod) {
        double d=0;
        startIteration();
        while (hasMore()) {
          lemur::api::QueryTerm *qt=nextTerm();
          double pr = qt->weight()/(double)totalCount();
          d += pr*log(pr/refMod.prob(qt->id()));
          delete qt;
        }
        return d;
      }

      double colQueryLikelihood() const {
        if (colQLikelihood == 0) {
          //Sum w in Q qtf * log(qtcf/termcount);
          lemur::api::COUNT_T tc = ind.termCount();
          startIteration();
          while (hasMore()) {
            lemur::api::QueryTerm *qt = nextTerm();
            lemur::api::TERMID_T id = qt->id();
            double qtf = qt->weight();
            lemur::api::COUNT_T qtcf = ind.termCount(id);
            double s = qtf * log((double)qtcf/(double)tc);
            colQLikelihood += s;
            delete qt;
          }
        }
        return colQLikelihood;
      }
  

    protected:
      // For Query likelihood adjusted score
      mutable double colQLikelihood;
      mutable double colKL;
      mutable bool colKLComputed;

      lemur::api::IndexedRealVector *qm;
      const lemur::api::Index &ind;
    };




    class SimpleKLScoreFunc : public lemur::api::ScoreFunction {
    public:
      enum SimpleKLParameter::adjustedScoreMethods adjScoreMethod;
      void setScoreMethod(enum SimpleKLParameter::adjustedScoreMethods adj) {
        adjScoreMethod = adj;
      }  
      virtual double matchedTermWeight(const lemur::api::QueryTerm *qTerm, 
                                       const lemur::api::TextQueryRep *qRep, 
                                       const lemur::api::DocInfo *info,
                                       const lemur::api::DocumentRep *dRep) const { 
        double w = qTerm->weight();
        double d = dRep->termWeight(qTerm->id(),info);
        double l = log(d);
        double score = w*l;
        /*
          cerr << "M:" << qTerm->id() <<" d:" << info->docID() << " w:" << w 
          << " d:" << d << " l:" << l << " s:" << score << endl;
        */
        return score;
        //    return (qTerm->weight()*log(dRep->termWeight(qTerm->id(),info)));
      }
      virtual double adjustedScore(double origScore, 
                                   const lemur::api::TextQueryRep *qRep, 
                                   const lemur::api::DocumentRep *dRep) const {
        const SimpleKLQueryModel *qm = dynamic_cast<const SimpleKLQueryModel *>(qRep);
        // this cast is unnecessary
        //    SimpleKLDocModel *dm = (SimpleKLDocModel *)dRep;
        // dynamic_cast<SimpleKLDocModel *>dRep;

        double qsc = qm->scoreConstant();
        double dsc = log(dRep->scoreConstant());
        double cql = qm->colQueryLikelihood();
        // real query likelihood
        double s = dsc * qsc + origScore + cql;
        double qsNorm = origScore/qsc;
        double qmD = qm->colDivergence();
        /*
          cerr << "A:"<< origScore << " dsc:" << dsc  << " qsc:" << qsc  
          << " cql:" << cql << " s:"  << s << endl;
        */
        switch (adjScoreMethod) {
        case SimpleKLParameter::QUERYLIKELIHOOD:
          // this is the original query likelihood scoring formula


          return s;
          //      return (origScore+log(dm->scoreConstant())*qm->scoreConstant());
        case SimpleKLParameter::CROSSENTROPY:
          // This is the normalized query-likelihood, i.e., cross-entropy
          assert(qm->scoreConstant()!=0);
          // return (origScore/qm->scoreConstant() + log(dm->scoreConstant()));
          // add the term colQueryLikelihood/qm->scoreConstant
          s = qsNorm + dsc + cql/qsc;
          return (s);
        case SimpleKLParameter::NEGATIVEKLD:
          // This is the exact (negative) KL-divergence value, i.e., -D(Mq||Md)
          assert(qm->scoreConstant()!=0);
          s = qsNorm + dsc - qmD;
          /*
            cerr << origScore << ":" << qsNorm << ":" << dsc  << ":" << qmD  << ":" << s << endl;
          */
          return s;
          //      return (origScore/qm->scoreConstant() + log(dm->scoreConstant())
          //          - qm->colDivergence());
        default:
          cerr << "unknown adjusted score method" << endl;
          return origScore;
        }
      }
  
    };

    class SimpleKLRetMethod : public lemur::api::TextQueryRetMethod {
    public:

      SimpleKLRetMethod(const lemur::api::Index &dbIndex, 
                        const string &supportFileName, 
                        lemur::api::ScoreAccumulator &accumulator);
      virtual ~SimpleKLRetMethod();
  
      virtual lemur::api::TextQueryRep *computeTextQueryRep(const lemur::api::TermQuery &qry) {
        return (new SimpleKLQueryModel(qry, ind));
      }
  
      virtual lemur::api::DocumentRep *computeDocRep(lemur::api::DOCID_T docID);
  

      virtual lemur::api::ScoreFunction *scoreFunc() {
        return (scFunc);
      }

      virtual void updateTextQuery(lemur::api::TextQueryRep &origRep, 
                                   const lemur::api::DocIDSet &relDocs);

      void setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam);
      void setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam);

    protected:

      double *mcNorm; 
  
      double *docProbMass; 
      lemur::api::COUNT_T *uniqueTermCount; 
      lemur::langmod::UnigramLM *collectLM; 
      lemur::langmod::DocUnigramCounter *collectLMCounter; 
      SimpleKLScoreFunc *scFunc; 



      void computeMixtureFBModel(SimpleKLQueryModel &origRep, 
                                 const lemur::api::DocIDSet & relDocs);
      void computeDivMinFBModel(SimpleKLQueryModel &origRep, 
                                const lemur::api::DocIDSet &relDocs);
      void computeMarkovChainFBModel(SimpleKLQueryModel &origRep, 
                                     const lemur::api::DocIDSet &relDocs) ;
      void computeRM1FBModel(SimpleKLQueryModel &origRep, 
                             const lemur::api::DocIDSet & relDocs);
      void computeRM2FBModel(SimpleKLQueryModel &origRep, 
                             const lemur::api::DocIDSet & relDocs);

      SimpleKLParameter::DocSmoothParam docParam;
      SimpleKLParameter::QueryModelParam qryParam;

      void loadSupportFile();
      const string supportFile;
    };


    inline  void SimpleKLRetMethod::setDocSmoothParam(SimpleKLParameter::DocSmoothParam &docSmthParam)
    {
      docParam = docSmthParam;
      loadSupportFile();
    }

    inline  void SimpleKLRetMethod::setQueryModelParam(SimpleKLParameter::QueryModelParam &queryModParam)
    {
      qryParam = queryModParam;
      // add a parameter to the score function.
      // isn't available in the constructor.
      scFunc->setScoreMethod(qryParam.adjScoreMethod);
      loadSupportFile();
    }
  }
}

#endif /* _SIMPLEKLRETMETHOD_HPP */

---