00001 /*========================================================================== 00002 * Copyright (c) 2003 Carnegie Mellon University. All Rights Reserved. 00003 * 00004 * Use of the Lemur Toolkit for Language Modeling and Information Retrieval 00005 * is subject to the terms of the software license set forth in the LICENSE 00006 * file included with this software, and also available at 00007 * http://www.lemurproject.org/license.html 00008 * 00009 *========================================================================== 00010 */ 00011 00012 00013 #ifndef _BRILLPOSTOK_HPP 00014 #define _BRILLPOSTOK_HPP 00015 #include "TextHandler.hpp" 00016 namespace lemur 00017 { 00018 namespace parse 00019 { 00020 00031 00032 class BrillPOSTokenizer : public lemur::api::TextHandler { 00033 00034 public: 00036 BrillPOSTokenizer(); 00037 00039 BrillPOSTokenizer(char s); 00040 00042 void setDelimiter(char s) {splitter = s;}; 00043 00046 char * handleWord(char* word, const char* original, PropertyList* list); 00047 00048 protected: 00049 char splitter; 00050 // keep setting one property. PropertyList copies. 00051 Property pos; 00052 }; 00053 } 00054 } 00055 #endif
1.3.4