Package org.apache.nifi.processors.hadoop

Class GetHDFSFileInfo

java.lang.Object
org.apache.nifi.components.AbstractConfigurableComponent
org.apache.nifi.processor.AbstractSessionFactoryProcessor
org.apache.nifi.processor.AbstractProcessor
org.apache.nifi.processors.hadoop.AbstractHadoopProcessor
org.apache.nifi.processors.hadoop.GetHDFSFileInfo
All Implemented Interfaces:
org.apache.nifi.components.ClassloaderIsolationKeyProvider, org.apache.nifi.components.ConfigurableComponent, org.apache.nifi.processor.Processor
@InputRequirement(INPUT_ALLOWED) @Tags({"hadoop","HCFS","HDFS","get","list","ingest","source","filesystem"}) @CapabilityDescription("Retrieves a listing of files and directories from HDFS. This processor creates a FlowFile(s) that represents the HDFS file/dir with relevant information. Main purpose of this processor to provide functionality similar to HDFS Client, i.e. count, du, ls, test, etc. Unlike ListHDFS, this processor is stateless, supports incoming connections and provides information on a dir level. ") @WritesAttribute(attribute="hdfs.objectName",description="The name of the file/dir found on HDFS.") @WritesAttribute(attribute="hdfs.path",description="The path is set to the absolute path of the object\'s parent directory on HDFS. For example, if an object is a directory \'foo\', under directory \'/bar\' then \'hdfs.objectName\' will have value \'foo\', and \'hdfs.path\' will be \'/bar\'") @WritesAttribute(attribute="hdfs.type",description="The type of an object. Possible values: directory, file, link") @WritesAttribute(attribute="hdfs.owner",description="The user that owns the object in HDFS") @WritesAttribute(attribute="hdfs.group",description="The group that owns the object in HDFS") @WritesAttribute(attribute="hdfs.lastModified",description="The timestamp of when the object in HDFS was last modified, as milliseconds since midnight Jan 1, 1970 UTC") @WritesAttribute(attribute="hdfs.length",description="In case of files: The number of bytes in the file in HDFS. In case of dirs: Retuns storage space consumed by directory. ") @WritesAttribute(attribute="hdfs.count.files",description="In case of type=\'directory\' will represent total count of files under this dir. Won\'t be populated to other types of HDFS objects. ") @WritesAttribute(attribute="hdfs.count.dirs",description="In case of type=\'directory\' will represent total count of directories under this dir (including itself). Won\'t be populated to other types of HDFS objects. ") @WritesAttribute(attribute="hdfs.replication",description="The number of HDFS replicas for the file") @WritesAttribute(attribute="hdfs.permissions",description="The permissions for the object in HDFS. This is formatted as 3 characters for the owner, 3 for the group, and 3 for other users. For example rw-rw-r--") @WritesAttribute(attribute="hdfs.status",description="The status contains comma separated list of file/dir paths, which couldn\'t be listed/accessed. Status won\'t be set if no errors occured.") @WritesAttribute(attribute="hdfs.full.tree",description="When destination is \'attribute\', will be populated with full tree of HDFS directory in JSON format.WARNING: In case when scan finds thousands or millions of objects, having huge values in attribute could impact flow file repo and GC/heap usage. Use content destination for such cases") @SeeAlso({ListHDFS.class,GetHDFS.class,FetchHDFS.class,PutHDFS.class}) public class GetHDFSFileInfo extends AbstractHadoopProcessor

  • Field Details

    • APPLICATION_JSON

      public static final String APPLICATION_JSON
      See Also:

    • FULL_PATH

      public static final org.apache.nifi.components.PropertyDescriptor FULL_PATH

    • RECURSE_SUBDIRS

      public static final org.apache.nifi.components.PropertyDescriptor RECURSE_SUBDIRS

    • DIR_FILTER

      public static final org.apache.nifi.components.PropertyDescriptor DIR_FILTER

    • FILE_FILTER

      public static final org.apache.nifi.components.PropertyDescriptor FILE_FILTER

    • FILE_EXCLUDE_FILTER

      public static final org.apache.nifi.components.PropertyDescriptor FILE_EXCLUDE_FILTER

    • IGNORE_DOTTED_DIRS

      public static final org.apache.nifi.components.PropertyDescriptor IGNORE_DOTTED_DIRS

    • IGNORE_DOTTED_FILES

      public static final org.apache.nifi.components.PropertyDescriptor IGNORE_DOTTED_FILES

    • GROUP_ALL

      static final org.apache.nifi.components.AllowableValue GROUP_ALL

    • GROUP_PARENT_DIR

      static final org.apache.nifi.components.AllowableValue GROUP_PARENT_DIR

    • GROUP_NONE

      static final org.apache.nifi.components.AllowableValue GROUP_NONE

    • GROUPING

      public static final org.apache.nifi.components.PropertyDescriptor GROUPING

    • BATCH_SIZE

      public static final org.apache.nifi.components.PropertyDescriptor BATCH_SIZE

    • DESTINATION_ATTRIBUTES

      static final org.apache.nifi.components.AllowableValue DESTINATION_ATTRIBUTES

    • DESTINATION_CONTENT

      static final org.apache.nifi.components.AllowableValue DESTINATION_CONTENT

    • DESTINATION

      public static final org.apache.nifi.components.PropertyDescriptor DESTINATION

    • REL_SUCCESS

      public static final org.apache.nifi.processor.Relationship REL_SUCCESS

    • REL_NOT_FOUND

      public static final org.apache.nifi.processor.Relationship REL_NOT_FOUND

    • REL_ORIGINAL

      public static final org.apache.nifi.processor.Relationship REL_ORIGINAL

    • REL_FAILURE

      public static final org.apache.nifi.processor.Relationship REL_FAILURE

    • PROPERTY_DESCRIPTORS

      private static final List<org.apache.nifi.components.PropertyDescriptor> PROPERTY_DESCRIPTORS

    • RELATIONSHIPS

      private static final Set<org.apache.nifi.processor.Relationship> RELATIONSHIPS

  • Constructor Details

    • GetHDFSFileInfo

      public GetHDFSFileInfo()

  • Method Details