net.yacy.document

Class Condenser

java.lang.Object

net.yacy.document.Tokenizer

net.yacy.document.Condenser

public final class Condenser
extends Tokenizer

Field Summary

Fields

Modifier and Type	Field and Description
java.util.LinkedHashSet<java.util.Date>	dates_in_content
private long	exact_signature
private long	fuzzy_signature
private java.lang.String	fuzzy_signature_text
private Identificator	languageIdentificator

Fields inherited from class net.yacy.document.Tokenizer

flag_cat_hasapp, flag_cat_hasaudio, flag_cat_hasimage, flag_cat_haslocation, flag_cat_hasvideo, flag_cat_indexof, pseudostemming, RESULT_FLAGS, RESULT_NUMB_SENTENCES, RESULT_NUMB_WORDS, tags, wordcut, wordminsize, words

Constructor Summary

Constructors

Constructor and Description
Condenser(Document document, VocabularyScraper scraper, boolean indexText, boolean indexMedia, WordCache meaningLib, boolean doAutotagging, boolean findDatesInContent, int timezoneOffset)

Method Summary

Methods

Modifier and Type	Method and Description
long	exactSignature()
int	excludeWords(java.util.SortedSet<java.lang.String> stopwords)
long	fuzzySignature()
java.lang.String	fuzzySignatureText()
private void	insertTextToWords(SentenceReader text, int phrase, int flagpos, Bitfield flagstemplate, boolean useForLanguageIdentification, WordCache meaningLib)
java.lang.String	language()
double	languageProbability() get the probability of the detected language received by language()
static void	main(java.lang.String[] args)

Methods inherited from class net.yacy.document.Tokenizer

getWords, synonyms, tags, words

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

fuzzy_signature

private long fuzzy_signature

exact_signature

private long exact_signature

fuzzy_signature_text

private java.lang.String fuzzy_signature_text

languageIdentificator

private final Identificator languageIdentificator

dates_in_content

public java.util.LinkedHashSet<java.util.Date> dates_in_content

Constructor Detail

Condenser

public Condenser(Document document,
         VocabularyScraper scraper,
         boolean indexText,
         boolean indexMedia,
         WordCache meaningLib,
         boolean doAutotagging,
         boolean findDatesInContent,
         int timezoneOffset)

Method Detail

insertTextToWords

private void insertTextToWords(SentenceReader text,
                     int phrase,
                     int flagpos,
                     Bitfield flagstemplate,
                     boolean useForLanguageIdentification,
                     WordCache meaningLib)

excludeWords

public int excludeWords(java.util.SortedSet<java.lang.String> stopwords)

fuzzySignature

public long fuzzySignature()

fuzzySignatureText

public java.lang.String fuzzySignatureText()

exactSignature

public long exactSignature()

language

public java.lang.String language()

languageProbability

public double languageProbability()

get the probability of the detected language received by language()

Returns:

0.0 to 1.0

main

public static void main(java.lang.String[] args)