de.datexis.index.impl

Class LuceneArticleIndex

java.lang.Object

de.datexis.index.ArticleIndex

de.datexis.index.impl.LuceneArticleIndex

Direct Known Subclasses:

KNNArticleIndex, VectorArticleIndex

public class LuceneArticleIndex
extends ArticleIndex

Author:

Sebastian Arnold

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.lucene.analysis.Analyzer	analyzer
protected static String	FIELD_DESCRIPTION
protected static String	FIELD_NAMES
protected static String	FIELD_REFIDS
protected static String	FIELD_REFIDS_FREEBASE
protected static String	FIELD_REFIDS_ICD10
protected static String	FIELD_REFIDS_UMLS
protected static String	FIELD_REFIDS_WIKIDATA
protected static String	FIELD_REFIDS_WIKIPEDIA
protected static String	FIELD_REFURLS_WIKIPEDIA
protected static String	FIELD_TERMS
protected static String	FIELD_TEXT
protected static String	FIELD_TITLE
protected static String	FIELD_TYPE
protected static String	FIELD_VECTOR
protected static org.slf4j.Logger	log
protected static int	NUM_BM25_CANDIDATES
protected static String	PARAM_FUZZY
protected static String	PARAM_PROXIMITY
protected org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess	preprocessor
protected org.apache.lucene.index.IndexReader	reader
protected org.apache.lucene.search.IndexSearcher	searcher
protected info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity	sim

Constructor Summary

Constructors

Constructor and Description
LuceneArticleIndex()

Method Summary

Modifier and Type	Method and Description
protected void	addNameField(org.apache.lucene.document.Document doc, String name, String value) A NameField is a tokenized lowercase field in Lucene.
protected void	addStringField(org.apache.lucene.document.Document doc, String name, String value) A StringField is a non-tokenized field in Lucene.
protected void	addTextField(org.apache.lucene.document.Document doc, String name, String value, org.apache.lucene.document.Field.Store store) A TextField is a tokenized field in Lucene.
protected void	addVectorField(org.apache.lucene.document.Document doc, String name, org.nd4j.linalg.api.ndarray.INDArray arr) A VectorField is a compressed INDArray as Base64 encoded string.
protected org.apache.lucene.analysis.Analyzer	buildAnalyzer()
void	createIndex(Iterator<? extends Article> articles, org.apache.lucene.store.Directory index)
void	createIndexDirectory(Iterator<? extends Article> articles, Resource cacheDir)
void	createIndexRAM(Iterator<? extends Article> articles) Creates and opens a new index in local RAM
protected org.apache.lucene.document.Document	createLuceneDocument(WikiDataArticle article)
protected Article	createWikidataArticle(org.apache.lucene.document.Document doc)
protected ArticleRef	createWikidataArticleRef(org.apache.lucene.document.Document doc)
protected String	decodeWikiUrl(String url)
Collection<String>	getAllArticleIDs()
Collection<String>	getAllArticleNames()
Collection<String>	getAllArticleTerms()
Collection<String>	getAllArticleTitles()
Collection<String>	getAllArticleURLs()
protected Collection<String>	getAllFields(String field)
boolean	openIndex(Resource path) tries to open an existing index from given path
Optional<ArticleRef>	queryID(String id) Retrieve the article with a given ID
protected Optional<org.apache.lucene.document.Document>	queryIndexID(String field, String id)
protected List<org.apache.lucene.document.Document>	queryIndexNames(String name, int hits)
protected List<ArticleRef>	queryIndexPrefix(String prefix, int hits)
List<ArticleRef>	queryNames(String name, int hits) Retrieve candidates for a query on the "name" field.
List<ArticleRef>	queryPrefixNames(String prefix, int hits) Retrieve candidates for auto completion on the "name" field.
Optional<ArticleRef>	queryWikidataID(String id) Retrieve the article with a given ID
Optional<ArticleRef>	queryWikipediaPage(String name) Retrieve the article with a given name
Optional<ArticleRef>	queryWikipediaURL(String url) Retrieve the article with a given URL
protected String	splitString(String name, String suffix)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

log

protected static final org.slf4j.Logger log

FIELD_TITLE

protected static final String FIELD_TITLE

See Also:

Constant Field Values

FIELD_TYPE

protected static final String FIELD_TYPE

See Also:

Constant Field Values

FIELD_REFIDS

protected static final String FIELD_REFIDS

See Also:

Constant Field Values

FIELD_REFIDS_WIKIDATA

protected static final String FIELD_REFIDS_WIKIDATA

See Also:

Constant Field Values

FIELD_REFIDS_WIKIPEDIA

protected static final String FIELD_REFIDS_WIKIPEDIA

See Also:

Constant Field Values

FIELD_REFIDS_FREEBASE

protected static final String FIELD_REFIDS_FREEBASE

See Also:

Constant Field Values

FIELD_REFIDS_UMLS

protected static final String FIELD_REFIDS_UMLS

See Also:

Constant Field Values

FIELD_REFIDS_ICD10

protected static final String FIELD_REFIDS_ICD10

See Also:

Constant Field Values

FIELD_REFURLS_WIKIPEDIA

protected static final String FIELD_REFURLS_WIKIPEDIA

See Also:

Constant Field Values

FIELD_TEXT

protected static final String FIELD_TEXT

See Also:

Constant Field Values

FIELD_DESCRIPTION

protected static final String FIELD_DESCRIPTION

See Also:

Constant Field Values

FIELD_NAMES

protected static final String FIELD_NAMES

See Also:

Constant Field Values

FIELD_TERMS

protected static final String FIELD_TERMS

See Also:

Constant Field Values

FIELD_VECTOR

protected static final String FIELD_VECTOR

See Also:

Constant Field Values

PARAM_PROXIMITY

protected static final String PARAM_PROXIMITY

See Also:

Constant Field Values

PARAM_FUZZY

protected static final String PARAM_FUZZY

See Also:

Constant Field Values

NUM_BM25_CANDIDATES

protected static final int NUM_BM25_CANDIDATES

See Also:

Constant Field Values

reader

protected org.apache.lucene.index.IndexReader reader

searcher

protected org.apache.lucene.search.IndexSearcher searcher

analyzer

protected org.apache.lucene.analysis.Analyzer analyzer

sim

protected final info.debatty.java.stringsimilarity.interfaces.NormalizedStringSimilarity sim

preprocessor

protected final org.deeplearning4j.text.tokenization.tokenizer.TokenPreProcess preprocessor

Constructor Detail

LuceneArticleIndex

public LuceneArticleIndex()

Method Detail

openIndex

public boolean openIndex(Resource path)

tries to open an existing index from given path

createIndexRAM

public void createIndexRAM(Iterator<? extends Article> articles)

Creates and opens a new index in local RAM

createIndexDirectory

public void createIndexDirectory(Iterator<? extends Article> articles,
                                 Resource cacheDir)
                          throws IOException

Throws:

IOException

createIndex

public void createIndex(Iterator<? extends Article> articles,
                        org.apache.lucene.store.Directory index)

queryNames

public List<ArticleRef> queryNames(String name,
                                   int hits)

Retrieve candidates for a query on the "name" field. All given words have to match (lowercase, any order).

Specified by:

queryNames in class ArticleIndex

Parameters:

name - The name to search, e.g. "obama barack"

hits - Max number of hits to generate

Returns:

An List of matching ArticleRefs, e.g. Wiki URL of "Barack Obama"

queryPrefixNames

public List<ArticleRef> queryPrefixNames(String prefix,
                                         int hits)

Retrieve candidates for auto completion on the "name" field.

Specified by:

queryPrefixNames in class ArticleIndex

Parameters:

prefix -

hits -

Returns:

queryID

public Optional<ArticleRef> queryID(String id)

Retrieve the article with a given ID

Specified by:

queryID in class ArticleIndex

Parameters:

id - The indexed ID, e.g. "Q64"

Returns:

The ArticleRef, if exists

queryWikidataID

public Optional<ArticleRef> queryWikidataID(String id)

Retrieve the article with a given ID

Parameters:

id - The Wikidata ID, e.g. "Q64"

Returns:

The Wiki article

queryWikipediaURL

public Optional<ArticleRef> queryWikipediaURL(String url)

Retrieve the article with a given URL

Returns:

The Wiki article

queryWikipediaPage

public Optional<ArticleRef> queryWikipediaPage(String name)

Retrieve the article with a given name

Returns:

The Wiki article

decodeWikiUrl

protected String decodeWikiUrl(String url)

getAllArticleTitles

public Collection<String> getAllArticleTitles()

getAllArticleNames

public Collection<String> getAllArticleNames()

getAllArticleTerms

public Collection<String> getAllArticleTerms()

getAllArticleURLs

public Collection<String> getAllArticleURLs()

getAllArticleIDs

public Collection<String> getAllArticleIDs()

getAllFields

protected Collection<String> getAllFields(String field)

createLuceneDocument

protected org.apache.lucene.document.Document createLuceneDocument(WikiDataArticle article)

createWikidataArticleRef

protected ArticleRef createWikidataArticleRef(org.apache.lucene.document.Document doc)

createWikidataArticle

protected Article createWikidataArticle(org.apache.lucene.document.Document doc)

splitString

protected String splitString(String name,
                             String suffix)

addTextField

protected void addTextField(org.apache.lucene.document.Document doc,
                            String name,
                            String value,
                            org.apache.lucene.document.Field.Store store)

A TextField is a tokenized field in Lucene.

addStringField

protected void addStringField(org.apache.lucene.document.Document doc,
                              String name,
                              String value)

A StringField is a non-tokenized field in Lucene. We always store.

addNameField

protected void addNameField(org.apache.lucene.document.Document doc,
                            String name,
                            String value)

A NameField is a tokenized lowercase field in Lucene. We always store.

addVectorField

protected void addVectorField(org.apache.lucene.document.Document doc,
                              String name,
                              org.nd4j.linalg.api.ndarray.INDArray arr)

A VectorField is a compressed INDArray as Base64 encoded string.

buildAnalyzer

protected org.apache.lucene.analysis.Analyzer buildAnalyzer()

queryIndexNames

protected List<org.apache.lucene.document.Document> queryIndexNames(String name,
                                                                    int hits)

queryIndexPrefix

protected List<ArticleRef> queryIndexPrefix(String prefix,
                                            int hits)

queryIndexID

protected Optional<org.apache.lucene.document.Document> queryIndexID(String field,
                                                                     String id)