Main Page | Namespace List | Class Hierarchy | Class List | File List | Namespace Members | Class Members | File Members | Related Pages

indri::parse::Combiner Class Reference

#include <Combiner.hpp>

List of all members.

Public Member Functions

 Combiner (int bins=10)
void combineRedirectDestinationBuckets (const std::string &tmpPath)
void combineBuckets (const std::string &outputPath, const std::string &tmpPath)
void hashRedirectTargets (const std::string &bucketPath, const std::string &redirectsPath)
void hashToBuckets (const std::string &bucketPath, const std::string &inputPath)
void sortCorpusFiles (const std::string &outputPath, const std::string &preSortPath, const std::string &inputPath)

Private Types

typedef indri::utility::HashTable<
char *, url_entry *, strhash,
typedef indri::utility::HashTable<
char *, std::vector< url_entry * >,
strhash, strcompst

Private Member Functions

url_entry_newUrlEntry (const char *url, const char *corpusPath, const char *docNo)
void _deleteUrlEntry (void *buffer)
void _readLinks (UrlEntryTable &urlTable, std::ifstream &linkIn)
void _readRedirects (UrlEntryTable &urlTable, const std::string &redirectPath, int number)
void _writeCorpusTable (UrlEntryVectorTable &corpusTable, const std::string &outputPath)
void _hashToCorpusTable (UrlEntryVectorTable &corpusTable, UrlEntryTable &urlTable)
void _openWriteBuckets (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, const std::string &path, int bins)
void _flushWriteBuffer (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, bool force, int i)
void _flushWriteBuffers (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets, bool force)
void _closeWriteBuckets (std::vector< std::stringstream * > &buffers, std::vector< std::ofstream * > &buckets)
void _openReadBuckets (std::vector< std::ifstream * > &buckets, const std::string &path, int bins)
void _readDocBucket (UrlEntryTable &urlTable, std::ifstream &docIn)
int hashString (const char *str)
void hashToBuckets (std::ifstream &in, const std::string &path)
void createBuckets (const std::string &tmpPath)
void closeBuckets ()
void combineBucket (const std::string &outputPath, const std::string &tmpPath, int bucket)
void hashToBuckets (const std::string &inputPath)
void combineRedirectDestinationBucket (const std::string &tmpPath, int i, std::vector< std::stringstream * > &outBuffers, std::vector< std::ofstream * > &outputFiles)

Private Attributes

std::vector< std::ofstream * > _docBucketFiles
std::vector< std::ofstream * > _linkBucketFiles
std::vector< std::stringstream * > _docBuckets
std::vector< std::stringstream * > _linkBuckets
int _bins

Member Typedef Documentation

typedef indri::utility::HashTable<char*, url_entry*, strhash, strcompst> indri::parse::Combiner::UrlEntryTable [private]

typedef indri::utility::HashTable<char*, std::vector<url_entry*>, strhash, strcompst> indri::parse::Combiner::UrlEntryVectorTable [private]

Constructor & Destructor Documentation

indri::parse::Combiner::Combiner int  bins = 10  )  [inline]

Member Function Documentation

void indri::parse::Combiner::_closeWriteBuckets std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets

void indri::parse::Combiner::_deleteUrlEntry void *  buffer  )  [private]

void indri::parse::Combiner::_flushWriteBuffer std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
bool  force,
int  i

void indri::parse::Combiner::_flushWriteBuffers std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
bool  force

void indri::parse::Combiner::_hashToCorpusTable UrlEntryVectorTable corpusTable,
UrlEntryTable urlTable

indri::parse::Combiner::url_entry * indri::parse::Combiner::_newUrlEntry const char *  url,
const char *  corpusPath,
const char *  docNo

void indri::parse::Combiner::_openReadBuckets std::vector< std::ifstream * > &  buckets,
const std::string &  path,
int  bins

void indri::parse::Combiner::_openWriteBuckets std::vector< std::stringstream * > &  buffers,
std::vector< std::ofstream * > &  buckets,
const std::string &  path,
int  bins

void indri::parse::Combiner::_readDocBucket UrlEntryTable urlTable,
std::ifstream &  docIn

void indri::parse::Combiner::_readLinks UrlEntryTable urlTable,
std::ifstream &  linkIn

void indri::parse::Combiner::_readRedirects UrlEntryTable urlTable,
const std::string &  redirectPath,
int  number

void indri::parse::Combiner::_writeCorpusTable UrlEntryVectorTable corpusTable,
const std::string &  outputPath

void indri::parse::Combiner::closeBuckets  )  [private]

void indri::parse::Combiner::combineBucket const std::string &  outputPath,
const std::string &  tmpPath,
int  bucket

void indri::parse::Combiner::combineBuckets const std::string &  outputPath,
const std::string &  tmpPath

void indri::parse::Combiner::combineRedirectDestinationBucket const std::string &  tmpPath,
int  i,
std::vector< std::stringstream * > &  outBuffers,
std::vector< std::ofstream * > &  outputFiles

void indri::parse::Combiner::combineRedirectDestinationBuckets const std::string &  tmpPath  ) 

void indri::parse::Combiner::createBuckets const std::string &  tmpPath  )  [private]

void indri::parse::Combiner::hashRedirectTargets const std::string &  bucketPath,
const std::string &  redirectsPath

int indri::parse::Combiner::hashString const char *  str  )  [private]

void indri::parse::Combiner::hashToBuckets const std::string &  bucketPath,
const std::string &  inputPath

void indri::parse::Combiner::hashToBuckets const std::string &  inputPath  )  [private]

void indri::parse::Combiner::hashToBuckets std::ifstream &  in,
const std::string &  path

void indri::parse::Combiner::sortCorpusFiles const std::string &  outputPath,
const std::string &  preSortPath,
const std::string &  inputPath

Member Data Documentation

int indri::parse::Combiner::_bins [private]

std::vector< std::ofstream* > indri::parse::Combiner::_docBucketFiles [private]

std::vector< std::stringstream* > indri::parse::Combiner::_docBuckets [private]

std::vector< std::ofstream* > indri::parse::Combiner::_linkBucketFiles [private]

std::vector< std::stringstream* > indri::parse::Combiner::_linkBuckets [private]

The documentation for this class was generated from the following files:
Generated on Tue Jun 15 11:03:03 2010 for Lemur by doxygen 1.3.4