de.datexis.parvec.encoder

Class ParVecEncoder

java.lang.Object

de.datexis.annotator.AnnotatorComponent

de.datexis.encoder.Encoder

de.datexis.encoder.LookupCacheEncoder

de.datexis.parvec.encoder.ParVecEncoder

All Implemented Interfaces:

IComponent, IEncoder

Direct Known Subclasses:

ParVecSentenceEncoder, ParVecWordsEncoder

public class ParVecEncoder
extends LookupCacheEncoder

Field Summary

Fields

Modifier and Type	Field and Description
protected int	batchSize
protected int	iterations
protected List<org.deeplearning4j.models.word2vec.VocabWord>	labelsList
protected int	layerSize
protected double	learningRate
protected static org.slf4j.Logger	log
protected double	minLearningRate
protected org.deeplearning4j.models.paragraphvectors.ParagraphVectors	model
protected int	numEpochs
protected static org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess	preprocessor
protected List<String>	stopwords
protected int	targetSize
protected org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory	tokenizerFactory
protected int	windowSize
protected org.deeplearning4j.models.embeddings.wordvectors.WordVectors	word2Vec

Fields inherited from class de.datexis.encoder.LookupCacheEncoder

totalWords, vocab

Fields inherited from class de.datexis.annotator.AnnotatorComponent

id, modelAvailable, name, timer

Constructor Summary

Constructors

Constructor and Description
ParVecEncoder()

Method Summary

Modifier and Type	Method and Description
org.nd4j.linalg.api.ndarray.INDArray	encode(Annotation ann, Document doc)
org.nd4j.linalg.api.ndarray.INDArray	encode(Span span)
org.nd4j.linalg.api.ndarray.INDArray	encode(String text)
long	getEmbeddingVectorSize()
int	getIndex(String word)
int	getInputVectorSize()
String	getNearestNeighbour(org.nd4j.linalg.api.ndarray.INDArray v)
Collection<String>	getNearestNeighbours(org.nd4j.linalg.api.ndarray.INDArray v, int k)
long	getOutputVectorSize()
org.nd4j.linalg.api.ndarray.INDArray	getPredictions(org.nd4j.linalg.api.ndarray.INDArray v)
int	getTotalWords()
String	getWord(int index)
List<String>	getWords()
static ParVecEncoder	load(Resource path)
void	loadModel(Resource modelFile)
org.nd4j.linalg.api.ndarray.INDArray	oneHot(String word)
void	saveModel(Resource modelPath, String name)
void	setModelParams(int layerSize, int windowSize)
void	setStopWords(List<String> words)
void	setTrainingParams(double learningRate, double minLearningRate, int batchSize, int iterations, int numEpochs)
void	trainModel(Collection<Document> documents)
void	trainModel(Dataset train)
ParVecEncoder	withWordEmbedding(org.deeplearning4j.models.embeddings.wordvectors.WordVectors word2Vec)

Methods inherited from class de.datexis.encoder.LookupCacheEncoder

getConfidence, getFrequency, getMaxConfidence, getNearestNeighbours, getProbability, isUnknown, setTotalWords

Methods inherited from class de.datexis.encoder.Encoder

encode, encodeEach, encodeEach, encodeEach, encodeMatrix, trainModel

Methods inherited from class de.datexis.annotator.AnnotatorComponent

appendTestLog, appendTestLog, appendTrainLog, appendTrainLog, clearTestLog, clearTrainLog, getConf, getId, getModel, getName, getTestLog, getTrainLog, isModelAvailable, isModelAvailableInChildren, setConf, setId, setModel, setModelAvailable, setModelFilename, setName

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface de.datexis.annotator.IComponent

getEncoders, getId, getName, isModelAvailable, setEncoders

Field Detail

log

protected static final org.slf4j.Logger log

word2Vec

protected org.deeplearning4j.models.embeddings.wordvectors.WordVectors word2Vec

model

protected org.deeplearning4j.models.paragraphvectors.ParagraphVectors model

learningRate

protected double learningRate

minLearningRate

protected double minLearningRate

batchSize

protected int batchSize

numEpochs

protected int numEpochs

iterations

protected int iterations

layerSize

protected int layerSize

targetSize

protected int targetSize

windowSize

protected int windowSize

preprocessor

protected static final org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess preprocessor

tokenizerFactory

protected final org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory tokenizerFactory

labelsList

protected List<org.deeplearning4j.models.word2vec.VocabWord> labelsList

stopwords

protected List<String> stopwords

Constructor Detail

ParVecEncoder

public ParVecEncoder()

Method Detail

withWordEmbedding

public ParVecEncoder withWordEmbedding(org.deeplearning4j.models.embeddings.wordvectors.WordVectors word2Vec)

setModelParams

public void setModelParams(int layerSize,
                           int windowSize)

setTrainingParams

public void setTrainingParams(double learningRate,
                              double minLearningRate,
                              int batchSize,
                              int iterations,
                              int numEpochs)

setStopWords

public void setStopWords(List<String> words)

trainModel

public void trainModel(Collection<Document> documents)

Specified by:

trainModel in class Encoder

trainModel

public void trainModel(Dataset train)

encode

public org.nd4j.linalg.api.ndarray.INDArray encode(Span span)

encode

public org.nd4j.linalg.api.ndarray.INDArray encode(Annotation ann,
                                                   Document doc)

encode

public org.nd4j.linalg.api.ndarray.INDArray encode(String text)

saveModel

public void saveModel(Resource modelPath,
                      String name)

Specified by:

saveModel in interface IComponent

Overrides:

saveModel in class LookupCacheEncoder

load

public static ParVecEncoder load(Resource path)
                          throws IOException

Throws:

IOException

loadModel

public void loadModel(Resource modelFile)
               throws IOException

Specified by:

loadModel in interface IComponent

Overrides:

loadModel in class LookupCacheEncoder

Throws:

IOException

getWords

public List<String> getWords()

Overrides:

getWords in class LookupCacheEncoder

getTotalWords

public int getTotalWords()

Overrides:

getTotalWords in class LookupCacheEncoder

getEmbeddingVectorSize

public long getEmbeddingVectorSize()

Specified by:

getEmbeddingVectorSize in interface IEncoder

Overrides:

getEmbeddingVectorSize in class LookupCacheEncoder

getOutputVectorSize

public long getOutputVectorSize()

getInputVectorSize

public int getInputVectorSize()

getWord

public String getWord(int index)

Overrides:

getWord in class LookupCacheEncoder

getIndex

public int getIndex(String word)

Overrides:

getIndex in class LookupCacheEncoder

oneHot

public org.nd4j.linalg.api.ndarray.INDArray oneHot(String word)

Overrides:

oneHot in class LookupCacheEncoder

getNearestNeighbour

public String getNearestNeighbour(org.nd4j.linalg.api.ndarray.INDArray v)

Overrides:

getNearestNeighbour in class LookupCacheEncoder

getNearestNeighbours

public Collection<String> getNearestNeighbours(org.nd4j.linalg.api.ndarray.INDArray v,
                                               int k)

Overrides:

getNearestNeighbours in class LookupCacheEncoder

getPredictions

public org.nd4j.linalg.api.ndarray.INDArray getPredictions(org.nd4j.linalg.api.ndarray.INDArray v)