public class WordTokenizer
extends java.lang.Object
implements java.util.Enumeration<java.lang.StringBuilder>
Modifier and Type | Class and Description |
---|---|
private class |
WordTokenizer.unsievedWordsEnum |
Modifier and Type | Field and Description |
---|---|
private java.lang.StringBuilder |
buffer |
private WordTokenizer.unsievedWordsEnum |
e |
private WordCache |
meaningLib |
Constructor and Description |
---|
WordTokenizer(SentenceReader sr,
WordCache meaningLib) |
Modifier and Type | Method and Description |
---|---|
void |
close() |
static java.util.SortedMap<byte[],java.lang.Integer> |
hashSentence(java.lang.String sentence,
int maxlength)
tokenize the given sentence and generate a word-wordPos mapping
|
boolean |
hasMoreElements() |
java.lang.StringBuilder |
nextElement() |
private java.lang.StringBuilder |
nextElement0() |
void |
pre(boolean x) |
static java.lang.StringBuilder |
trim(java.lang.StringBuilder sb) |
private java.lang.StringBuilder buffer
private WordTokenizer.unsievedWordsEnum e
private final WordCache meaningLib
public WordTokenizer(SentenceReader sr, WordCache meaningLib)
public void pre(boolean x)
private java.lang.StringBuilder nextElement0()
public boolean hasMoreElements()
hasMoreElements
in interface java.util.Enumeration<java.lang.StringBuilder>
public java.lang.StringBuilder nextElement()
nextElement
in interface java.util.Enumeration<java.lang.StringBuilder>
public void close()
public static java.lang.StringBuilder trim(java.lang.StringBuilder sb)
public static java.util.SortedMap<byte[],java.lang.Integer> hashSentence(java.lang.String sentence, int maxlength)
sentence
- the sentence to be tokenized