Package org.apache.nifi.processors.hadoop

Class MoveHDFS

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.hadoop.AbstractHadoopProcessor

org.apache.nifi.processors.hadoop.MoveHDFS

All Implemented Interfaces:

org.apache.nifi.components.ClassloaderIsolationKeyProvider, org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor

@InputRequirement(INPUT_ALLOWED)
@Tags({"hadoop","HCFS","HDFS","put","move","filesystem","moveHDFS"})
@CapabilityDescription("Rename existing files or a directory of files (non-recursive) on Hadoop Distributed File System (HDFS).")
@ReadsAttribute(attribute="filename",
                description="The name of the file written to HDFS comes from the value of this attribute.")
@WritesAttribute(attribute="filename",description="The name of the file written to HDFS is stored in this attribute.") @WritesAttribute(attribute="absolute.hdfs.path",description="The absolute path to the file on HDFS is stored in this attribute.") @WritesAttribute(attribute="hadoop.file.url",description="The hadoop url for the file is stored in this attribute.")
@SeeAlso({PutHDFS.class,GetHDFS.class})
@Restriction(requiredPermission=READ_DISTRIBUTED_FILESYSTEM,explanation="Provides operator the ability to retrieve any file that NiFi has access to in HDFS or the local filesystem.") @Restriction(requiredPermission=WRITE_DISTRIBUTED_FILESYSTEM,explanation="Provides operator the ability to delete any file that NiFi has access to in HDFS or the local filesystem.")
public class MoveHDFS
extends AbstractHadoopProcessor

This processor renames files on HDFS.

Nested Class Summary

Nested Classes

Modifier and Type	Class	Description
protected static class	MoveHDFS.ProcessorConfiguration

Nested classes/interfaces inherited from class org.apache.nifi.processors.hadoop.AbstractHadoopProcessor

AbstractHadoopProcessor.ValidationResources

Field Summary

Fields

Modifier and Type	Field	Description
static final String	ABSOLUTE_HDFS_PATH_ATTRIBUTE
static final org.apache.nifi.components.PropertyDescriptor	CONFLICT_RESOLUTION
static final String	FAIL_RESOLUTION
static final org.apache.nifi.components.AllowableValue	FAIL_RESOLUTION_AV
static final org.apache.nifi.components.PropertyDescriptor	FILE_FILTER_REGEX
private final BlockingQueue<org.apache.hadoop.fs.Path>	filePathQueue
static final org.apache.nifi.components.PropertyDescriptor	IGNORE_DOTTED_FILES
static final String	IGNORE_RESOLUTION
static final org.apache.nifi.components.AllowableValue	IGNORE_RESOLUTION_AV
static final org.apache.nifi.components.PropertyDescriptor	INPUT_DIRECTORY_OR_FILE
private final Lock	listingLock
private final AtomicLong	logEmptyListing
static final org.apache.nifi.components.PropertyDescriptor	OPERATION
static final org.apache.nifi.components.PropertyDescriptor	OUTPUT_DIRECTORY
private final BlockingQueue<org.apache.hadoop.fs.Path>	processing
protected MoveHDFS.ProcessorConfiguration	processorConfig
private static final List<org.apache.nifi.components.PropertyDescriptor>	PROPERTY_DESCRIPTORS
private final Lock	queueLock
static final org.apache.nifi.processor.Relationship	REL_FAILURE
static final org.apache.nifi.processor.Relationship	REL_SUCCESS
private static final Set<org.apache.nifi.processor.Relationship>	RELATIONSHIPS
static final org.apache.nifi.components.PropertyDescriptor	REMOTE_GROUP
static final org.apache.nifi.components.PropertyDescriptor	REMOTE_OWNER
static final String	REPLACE_RESOLUTION
static final org.apache.nifi.components.AllowableValue	REPLACE_RESOLUTION_AV

Fields inherited from class org.apache.nifi.processors.hadoop.AbstractHadoopProcessor

ADDITIONAL_CLASSPATH_RESOURCES, COMPRESSION_CODEC, DIRECTORY, HADOOP_CONFIGURATION_RESOURCES, HADOOP_FILE_URL_ATTRIBUTE, hdfsResources, KERBEROS_USER_SERVICE, TARGET_HDFS_DIR_CREATED_ATTRIBUTE

Constructor Summary

Constructors

Constructor	Description
MoveHDFS()

Method Summary

Modifier and Type	Method	Description
protected void	changeOwner(org.apache.nifi.processor.ProcessContext context, org.apache.hadoop.fs.FileSystem hdfs, org.apache.hadoop.fs.Path name)
Set<org.apache.nifi.processor.Relationship>	getRelationships()
protected List<org.apache.nifi.components.PropertyDescriptor>	getSupportedPropertyDescriptors()
void	onScheduled(org.apache.nifi.processor.ProcessContext context)
void	onTrigger(org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session)
protected Set<org.apache.hadoop.fs.Path>	performListing(org.apache.nifi.processor.ProcessContext context, org.apache.hadoop.fs.Path path)
protected void	processBatchOfFiles(List<org.apache.hadoop.fs.Path> files, org.apache.nifi.processor.ProcessContext context, org.apache.nifi.processor.ProcessSession session, org.apache.nifi.flowfile.FlowFile parentFlowFile)
protected Set<org.apache.hadoop.fs.Path>	selectFiles(org.apache.hadoop.fs.FileSystem hdfs, org.apache.hadoop.fs.Path inputPath, Set<org.apache.hadoop.fs.Path> filesVisited)

Methods inherited from class org.apache.nifi.processors.hadoop.AbstractHadoopProcessor

abstractOnScheduled, abstractOnStopped, checkHdfsUriForTimeout, customValidate, findCause, getClassloaderIsolationKey, getCommonPropertyDescriptors, getCompressionCodec, getConfigLocations, getConfiguration, getFileSystem, getFileSystem, getFileSystemAsUser, getHadoopConfigurationForValidation, getNormalizedPath, getNormalizedPath, getNormalizedPath, getPathDifference, getUserGroupInformation, handleAuthErrors, init, isFileSystemAccessDenied, isLocalFileSystemAccessDenied, migrateProperties, preProcessConfiguration, resetHDFSResources, validateFileSystem

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful, migrateRelationships

Field Details

REPLACE_RESOLUTION

public static final String REPLACE_RESOLUTION

See Also:

• Constant Field Values

IGNORE_RESOLUTION

public static final String IGNORE_RESOLUTION

See Also:

• Constant Field Values

FAIL_RESOLUTION

public static final String FAIL_RESOLUTION

See Also:

• Constant Field Values

REPLACE_RESOLUTION_AV

public static final org.apache.nifi.components.AllowableValue REPLACE_RESOLUTION_AV

IGNORE_RESOLUTION_AV

public static final org.apache.nifi.components.AllowableValue IGNORE_RESOLUTION_AV

FAIL_RESOLUTION_AV

public static final org.apache.nifi.components.AllowableValue FAIL_RESOLUTION_AV

ABSOLUTE_HDFS_PATH_ATTRIBUTE

public static final String ABSOLUTE_HDFS_PATH_ATTRIBUTE

See Also:

• Constant Field Values

REL_SUCCESS

public static final org.apache.nifi.processor.Relationship REL_SUCCESS

REL_FAILURE

public static final org.apache.nifi.processor.Relationship REL_FAILURE

CONFLICT_RESOLUTION

public static final org.apache.nifi.components.PropertyDescriptor CONFLICT_RESOLUTION

FILE_FILTER_REGEX

public static final org.apache.nifi.components.PropertyDescriptor FILE_FILTER_REGEX

IGNORE_DOTTED_FILES

public static final org.apache.nifi.components.PropertyDescriptor IGNORE_DOTTED_FILES

INPUT_DIRECTORY_OR_FILE

public static final org.apache.nifi.components.PropertyDescriptor INPUT_DIRECTORY_OR_FILE

OUTPUT_DIRECTORY

public static final org.apache.nifi.components.PropertyDescriptor OUTPUT_DIRECTORY

OPERATION

public static final org.apache.nifi.components.PropertyDescriptor OPERATION

REMOTE_OWNER

public static final org.apache.nifi.components.PropertyDescriptor REMOTE_OWNER

REMOTE_GROUP

public static final org.apache.nifi.components.PropertyDescriptor REMOTE_GROUP

RELATIONSHIPS

private static final Set<org.apache.nifi.processor.Relationship> RELATIONSHIPS

PROPERTY_DESCRIPTORS

private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTY_DESCRIPTORS

processorConfig

protected MoveHDFS.ProcessorConfiguration processorConfig

logEmptyListing

private final AtomicLong logEmptyListing

listingLock

private final Lock listingLock

queueLock

private final Lock queueLock

filePathQueue

private final BlockingQueue<org.apache.hadoop.fs.Path> filePathQueue

processing

private final BlockingQueue<org.apache.hadoop.fs.Path> processing

Constructor Details

MoveHDFS

public MoveHDFS()

Method Details

getRelationships

public Set<org.apache.nifi.processor.Relationship> getRelationships()

Specified by:

getRelationships in interface org.apache.nifi.processor.Processor

Overrides:

getRelationships in class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<org.apache.nifi.components.PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractHadoopProcessor

onScheduled

@OnScheduled
public void onScheduled(org.apache.nifi.processor.ProcessContext context)
                 throws Exception

Throws:

Exception

onTrigger

public void onTrigger(org.apache.nifi.processor.ProcessContext context,
 org.apache.nifi.processor.ProcessSession session)
               throws org.apache.nifi.processor.exception.ProcessException

Specified by:

onTrigger in class org.apache.nifi.processor.AbstractProcessor

Throws:

org.apache.nifi.processor.exception.ProcessException

processBatchOfFiles

protected void processBatchOfFiles(List<org.apache.hadoop.fs.Path> files,
 org.apache.nifi.processor.ProcessContext context,
 org.apache.nifi.processor.ProcessSession session,
 org.apache.nifi.flowfile.FlowFile parentFlowFile)

performListing

protected Set<org.apache.hadoop.fs.Path> performListing(org.apache.nifi.processor.ProcessContext context,
 org.apache.hadoop.fs.Path path)
                                                 throws IOException,
InterruptedException

Throws:

IOException

InterruptedException

changeOwner

protected void changeOwner(org.apache.nifi.processor.ProcessContext context,
 org.apache.hadoop.fs.FileSystem hdfs,
 org.apache.hadoop.fs.Path name)

selectFiles

protected Set<org.apache.hadoop.fs.Path> selectFiles(org.apache.hadoop.fs.FileSystem hdfs,
 org.apache.hadoop.fs.Path inputPath,
 Set<org.apache.hadoop.fs.Path> filesVisited)
                                              throws IOException,
InterruptedException

Throws:

IOException

InterruptedException