nlp/com.londogard.nlp.embeddings/BpeEmbeddings

BpeEmbeddings

public final class BpeEmbeddings implements Embeddings

BPEEmbeddings are subword embeddings that embeds SentencePieceTokenizer tokenized data. Studies show that performance are on par with GloVe (+-5%) while only using few MB's of data rather than GB's. Supports 275 languages through bpemb.

Constructors

BpeEmbeddings

public BpeEmbeddings BpeEmbeddings( Path filePath, Integer dimensions, Character delimiter, Tokenizer tokenizer)

Types

Companion

public class Companion

Functions

contains

public Boolean contains(String word)

Check if the word has an embedding.

cosineDistance

public Float cosineDistance(String w1, String w2)

euclideanDistance

public Float euclideanDistance(String w1, String w2)

getDelimiter

public Character getDelimiter()

getDimensions

public Integer getDimensions()

getEmbeddings

public Map<String, NDArray<Float, D1>> getEmbeddings()

getFilePath

public Path getFilePath()

getVocabulary

public Set<String> getVocabulary()

subwordVector

public final NDArray<Float, D1> subwordVector(String subword)

traverseVectors

public List<NDArray<Float, D1>> traverseVectors(List<String> words)

traverseVectorsOrNull

public List<NDArray<Float, D1>> traverseVectorsOrNull(List<String> words)

vector

public NDArray<Float, D1> vector(String word)

Fetches Embedding if it exists for word

Properties

delimiter

private final Character delimiter

dimensions

private final Integer dimensions

embeddings

private final Map<String, NDArray<Float, D1>> embeddings

filePath

private final Path filePath

vocabulary

private final Set<String> vocabulary