nlp/com.londogard.nlp.tokenizer/TokenizerSpecialTokens

TokenizerSpecialTokens

public class TokenizerSpecialTokens

Special tokens that is usable to Machine Learning.

Functions

public final Character getAllCaps()

public final Character getBOS()

public final Character getBOW()

getCharRepetition

public final Character getCharRepetition()

public final Character getEOS()

public final Character getNumber()

getNumberPattern

public final Regex getNumberPattern()

public final String getNumberStr()

public final Character getPad()

public final Character getSpace()

public final Character getStartOfWord()

public final Character getUpper()

getWordRepetition

public final Character getWordRepetition()

Properties

private final Character AllCaps

private final Character BOS

private final Character BOW

private final Character CharRepetition

private final Character EOS

public final static TokenizerSpecialTokens INSTANCE

private final Character Number

private final Regex NumberPattern

private final String NumberStr

private final Character Pad

private final Character Space

private final Character StartOfWord

private final Character Upper

private final Character WordRepetition