org.apache.nifi.processors.elasticsearch

Class ScrollElasticsearchHttp

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.elasticsearch.AbstractElasticsearchProcessor

org.apache.nifi.processors.elasticsearch.AbstractElasticsearchHttpProcessor

org.apache.nifi.processors.elasticsearch.ScrollElasticsearchHttp

All Implemented Interfaces:

ConfigurableComponent, Processor

@InputRequirement(value=INPUT_FORBIDDEN)
 @EventDriven
 @SupportsBatching
 @Tags(value={"elasticsearch","query","scroll","read","get","http"})
 @CapabilityDescription(value="Scrolls through an Elasticsearch query using the specified connection properties. This processor is intended to be run on the primary node, and is designed for scrolling through huge result sets, as in the case of a reindex.  The state must be cleared before another query can be run.  Each page of results is returned, wrapped in a JSON object like so: { \"hits\" : [ <doc1>, <doc2>, <docn> ] }.  Note that the full body of each page of documents will be read into memory before being written to a Flow File for transfer.")
 @WritesAttribute(attribute="es.index",description="The Elasticsearch index containing the document") @WritesAttribute(attribute="es.type",description="The Elasticsearch document type")
 @DynamicProperty(name="A URL query parameter",
                 value="The value to set it to",
                 expressionLanguageScope=VARIABLE_REGISTRY,
                 description="Adds the specified property name/value as a query parameter in the Elasticsearch URL used for processing")
 @Stateful(description="After each successful scroll page, the latest scroll_id is persisted in scrollId as input for the next scroll call.  Once the entire query is complete, finishedQuery state will be set to true, and the processor will not execute unless this is cleared.",
          scopes=LOCAL)
public class ScrollElasticsearchHttp
extends AbstractElasticsearchHttpProcessor

Field Summary

Fields

Modifier and Type	Field and Description
static PropertyDescriptor	FIELDS
private static String	FINISHED_QUERY_STATE
static PropertyDescriptor	INDEX
static PropertyDescriptor	PAGE_SIZE
private static List<PropertyDescriptor>	propertyDescriptors
static PropertyDescriptor	QUERY
static Relationship	REL_FAILURE
static Relationship	REL_SUCCESS
private static Set<Relationship>	relationships
static PropertyDescriptor	SCROLL_DURATION
private static String	SCROLL_ID_STATE
private static String	SCROLL_QUERY_PARAM
static PropertyDescriptor	SORT
static PropertyDescriptor	TYPE

Fields inherited from class org.apache.nifi.processors.elasticsearch.AbstractElasticsearchHttpProcessor

COMMON_PROPERTY_DESCRIPTORS, CONNECT_TIMEOUT, ES_URL, FIELD_INCLUDE_QUERY_PARAM, PROXY_CONFIGURATION_SERVICE, PROXY_HOST, PROXY_PASSWORD, PROXY_PORT, PROXY_USERNAME, QUERY_QUERY_PARAM, RESPONSE_TIMEOUT, SIZE_QUERY_PARAM, SORT_QUERY_PARAM

Fields inherited from class org.apache.nifi.processors.elasticsearch.AbstractElasticsearchProcessor

CHARSET, PASSWORD, PROP_SSL_CONTEXT_SERVICE, USERNAME

Constructor Summary

Constructors

Constructor and Description
ScrollElasticsearchHttp()

Method Summary

Modifier and Type	Method and Description
private URL	buildRequestURL(String baseUrl, String query, String index, String type, String fields, String sort, String scrollId, int pageSize, String scroll, ProcessContext context)
private void	finishQuery(StateManager stateManager)
private void	getPage(okhttp3.Response getResponse, URL url, ProcessContext context, ProcessSession session, FlowFile flowFile, ComponentLog logger, long startNanos, Charset charset)
Set<Relationship>	getRelationships()
List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private boolean	isQueryFinished(StateManager stateManager)
private String	loadScrollId(StateManager stateManager)
void	onTrigger(ProcessContext context, ProcessSession session)
private void	saveScrollId(StateManager stateManager, String scrollId)
void	setup(ProcessContext context)

Methods inherited from class org.apache.nifi.processors.elasticsearch.AbstractElasticsearchHttpProcessor

buildBulkCommand, createElasticsearchClient, customValidate, getClient, getSupportedDynamicPropertyDescriptor, isSuccess, parseJsonResponse, sendRequestToElasticsearch

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

FINISHED_QUERY_STATE

private static final String FINISHED_QUERY_STATE

See Also:

Constant Field Values

SCROLL_ID_STATE

private static final String SCROLL_ID_STATE

See Also:

Constant Field Values

SCROLL_QUERY_PARAM

private static final String SCROLL_QUERY_PARAM

See Also:

Constant Field Values

REL_SUCCESS

public static final Relationship REL_SUCCESS

REL_FAILURE

public static final Relationship REL_FAILURE

QUERY

public static final PropertyDescriptor QUERY

SCROLL_DURATION

public static final PropertyDescriptor SCROLL_DURATION

INDEX

public static final PropertyDescriptor INDEX

TYPE

public static final PropertyDescriptor TYPE

FIELDS

public static final PropertyDescriptor FIELDS

SORT

public static final PropertyDescriptor SORT

PAGE_SIZE

public static final PropertyDescriptor PAGE_SIZE

relationships

private static final Set<Relationship> relationships

propertyDescriptors

private static final List<PropertyDescriptor> propertyDescriptors

Constructor Detail

ScrollElasticsearchHttp

public ScrollElasticsearchHttp()

Method Detail

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

public final List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

setup

@OnScheduled
public void setup(ProcessContext context)

Overrides:

setup in class AbstractElasticsearchProcessor

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSession session)
               throws ProcessException

Specified by:

onTrigger in class AbstractProcessor

Throws:

ProcessException

getPage

private void getPage(okhttp3.Response getResponse,
                     URL url,
                     ProcessContext context,
                     ProcessSession session,
                     FlowFile flowFile,
                     ComponentLog logger,
                     long startNanos,
                     Charset charset)
              throws IOException

Throws:

IOException

isQueryFinished

private boolean isQueryFinished(StateManager stateManager)
                         throws IOException

Throws:

IOException

loadScrollId

private String loadScrollId(StateManager stateManager)
                     throws IOException

Throws:

IOException

finishQuery

private void finishQuery(StateManager stateManager)
                  throws IOException

Throws:

IOException

saveScrollId

private void saveScrollId(StateManager stateManager,
                          String scrollId)
                   throws IOException

Throws:

IOException

buildRequestURL

private URL buildRequestURL(String baseUrl,
                            String query,
                            String index,
                            String type,
                            String fields,
                            String sort,
                            String scrollId,
                            int pageSize,
                            String scroll,
                            ProcessContext context)
                     throws MalformedURLException

Throws:

MalformedURLException