org.apache.nifi.atlas.reporting

Class ReportLineageToAtlas

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.reporting.AbstractReportingTask

org.apache.nifi.atlas.reporting.ReportLineageToAtlas

All Implemented Interfaces:

ConfigurableComponent, ReportingTask

@DeprecationNotice(reason="Planned for removal in NiFi 2.0")
 @Tags(value={"atlas","lineage"})
 @CapabilityDescription(value="Report NiFi flow data set level lineage to Apache Atlas. End-to-end lineages across NiFi environments and other systems can be reported if those are connected by different protocols and data set, such as NiFi Site-to-Site, Kafka topic or Hive tables ... etc. Atlas lineage reported by this reporting task can be useful to grasp the high level relationships between processes and data sets, in addition to NiFi provenance events providing detailed event level lineage. See \'Additional Details\' for further description and limitations.")
 @Stateful(scopes=LOCAL,
          description="Stores the Reporting Task\'s last event Id so that on restart the task knows where it left off.")
 @DynamicProperty(name="hostnamePattern.<namespace>",
                 value="hostname Regex patterns",
                 description="White space delimited (including new line) Regular Expressions to resolve a namespace from a hostname or IP address of a transit URI of NiFi provenance record.",
                 expressionLanguageScope=VARIABLE_REGISTRY)
 @RequiresInstanceClassLoading
public class ReportLineageToAtlas
extends AbstractReportingTask

Field Summary

Fields

Modifier and Type	Field and Description
(package private) static AllowableValue	ATLAS_AUTHN_BASIC
(package private) static AllowableValue	ATLAS_AUTHN_KERBEROS
(package private) static PropertyDescriptor	ATLAS_AUTHN_METHOD
(package private) static PropertyDescriptor	ATLAS_CONF_CREATE
(package private) static PropertyDescriptor	ATLAS_CONF_DIR
static PropertyDescriptor	ATLAS_CONNECT_TIMEOUT
static PropertyDescriptor	ATLAS_DEFAULT_CLUSTER_NAME
private static String	ATLAS_KAFKA_PREFIX
static PropertyDescriptor	ATLAS_NIFI_URL
static PropertyDescriptor	ATLAS_PASSWORD
private static String	ATLAS_PROPERTIES_FILENAME
private static String	ATLAS_PROPERTY_CLIENT_CONNECT_TIMEOUT_MS
private static String	ATLAS_PROPERTY_CLIENT_READ_TIMEOUT_MS
private static String	ATLAS_PROPERTY_CLUSTER_NAME
private static String	ATLAS_PROPERTY_ENABLE_TLS
private static String	ATLAS_PROPERTY_KAFKA_BOOTSTRAP_SERVERS
private static String	ATLAS_PROPERTY_KAFKA_CLIENT_ID
private static String	ATLAS_PROPERTY_METADATA_NAMESPACE
private static String	ATLAS_PROPERTY_REST_ADDRESS
static PropertyDescriptor	ATLAS_READ_TIMEOUT
private static String	ATLAS_URL_DELIMITER
(package private) static PropertyDescriptor	ATLAS_URLS
static PropertyDescriptor	ATLAS_USER
private AtlasAuthN	atlasAuthN
private Properties	atlasProperties
(package private) static PropertyDescriptor	AWS_S3_MODEL_VERSION
(package private) static AllowableValue	AWS_S3_MODEL_VERSION_V1
(package private) static AllowableValue	AWS_S3_MODEL_VERSION_V2
private ProvenanceEventConsumer	consumer
private String	defaultMetadataNamespace
(package private) static PropertyDescriptor	FILESYSTEM_PATHS_LEVEL
(package private) static AllowableValue	FILESYSTEM_PATHS_LEVEL_DIRECTORY
(package private) static AllowableValue	FILESYSTEM_PATHS_LEVEL_FILE
private boolean	isTypeDefCreated
(package private) static PropertyDescriptor	KAFKA_BOOTSTRAP_SERVERS
(package private) static PropertyDescriptor	KAFKA_KERBEROS_SERVICE_NAME
(package private) static PropertyDescriptor	KAFKA_SECURITY_PROTOCOL
static PropertyDescriptor	KERBEROS_CREDENTIALS_SERVICE
static PropertyDescriptor	KERBEROS_KEYTAB
static PropertyDescriptor	KERBEROS_PRINCIPAL
(package private) static PropertyDescriptor	LINEAGE_STRATEGY
(package private) static AllowableValue	LINEAGE_STRATEGY_COMPLETE_PATH
(package private) static AllowableValue	LINEAGE_STRATEGY_SIMPLE_PATH
private LineageStrategy	lineageStrategy
private ServiceLoader<NamespaceResolver>	namespaceResolverLoader
private NamespaceResolvers	namespaceResolvers
private NiFiAtlasHook	nifiAtlasHook
(package private) static AllowableValue	SEC_PLAINTEXT
(package private) static AllowableValue	SEC_SASL_PLAINTEXT
(package private) static AllowableValue	SEC_SASL_SSL
(package private) static AllowableValue	SEC_SSL
private static String	SSL_CLIENT_XML_FILENAME
private static String	SSL_CLIENT_XML_TRUSTSTORE_LOCATION
private static String	SSL_CLIENT_XML_TRUSTSTORE_PASSWORD
private static String	SSL_CLIENT_XML_TRUSTSTORE_TYPE
(package private) static PropertyDescriptor	SSL_CONTEXT_SERVICE

Constructor Summary

Constructors

Constructor and Description
ReportLineageToAtlas()

Method Summary

Modifier and Type	Method and Description
private void	consumeNiFiProvenanceEvents(ReportingContext context, NiFiFlow nifiFlow)
protected NiFiAtlasClient	createNiFiAtlasClient(ReportingContext context) In order to avoid authentication expiration issues (i.e.
private NiFiFlow	createNiFiFlow(ReportingContext context, NiFiAtlasClient atlasClient)
private void	createSslClientXml(File confDir, SSLContextService sslContextService)
protected Collection<ValidationResult>	customValidate(ValidationContext context)
private void	deleteSslClientXml(File confDir)
private AtlasAuthN	getAtlasAuthN(String atlasAuthNMethod)
protected PropertyDescriptor	getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private void	initAtlasProperties(ConfigurationContext context)
private void	initLineageStrategy(ConfigurationContext context)
private void	initNamespaceResolvers(ConfigurationContext context)
private void	initProvenanceConsumer(ConfigurationContext context)
void	onStopped()
void	onTrigger(ReportingContext context)
void	onUnscheduled()
private List<String>	parseAtlasUrls(PropertyValue atlasUrlsProp)
private void	setAtlasSSLConfig(Properties atlasProperties, ConfigurationContext context, List<String> urls, File confDir)
private void	setKafkaConfig(Map<Object,Object> mapToPopulate, PropertyContext context)
private void	setKafkaJaasConfig(Map<Object,Object> mapToPopulate, PropertyContext context) Populate Kafka JAAS properties for Atlas notification.
void	setup(ConfigurationContext context)
private void	setValue(Consumer<String> setter, Runnable emptyHandler, PropertyValue elEnabledPropertyValue, String... properties)
private void	validateKafkaProperties(ValidationContext context, Collection<ValidationResult> results)

Methods inherited from class org.apache.nifi.reporting.AbstractReportingTask

getControllerServiceLookup, getIdentifier, getLogger, getName, getNodeTypeProvider, getSchedulingPeriod, init, initialize, setComponentInfo

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, hashCode, onPropertyModified, toString, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.reporting.ReportingTask

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

ATLAS_URL_DELIMITER

private static final String ATLAS_URL_DELIMITER

See Also:

Constant Field Values

ATLAS_URLS

static final PropertyDescriptor ATLAS_URLS

ATLAS_CONNECT_TIMEOUT

public static final PropertyDescriptor ATLAS_CONNECT_TIMEOUT

ATLAS_READ_TIMEOUT

public static final PropertyDescriptor ATLAS_READ_TIMEOUT

ATLAS_AUTHN_BASIC

static final AllowableValue ATLAS_AUTHN_BASIC

ATLAS_AUTHN_KERBEROS

static final AllowableValue ATLAS_AUTHN_KERBEROS

ATLAS_AUTHN_METHOD

static final PropertyDescriptor ATLAS_AUTHN_METHOD

ATLAS_USER

public static final PropertyDescriptor ATLAS_USER

ATLAS_PASSWORD

public static final PropertyDescriptor ATLAS_PASSWORD

ATLAS_CONF_DIR

static final PropertyDescriptor ATLAS_CONF_DIR

ATLAS_NIFI_URL

public static final PropertyDescriptor ATLAS_NIFI_URL

ATLAS_DEFAULT_CLUSTER_NAME

public static final PropertyDescriptor ATLAS_DEFAULT_CLUSTER_NAME

ATLAS_CONF_CREATE

static final PropertyDescriptor ATLAS_CONF_CREATE

SSL_CONTEXT_SERVICE

static final PropertyDescriptor SSL_CONTEXT_SERVICE

KAFKA_BOOTSTRAP_SERVERS

static final PropertyDescriptor KAFKA_BOOTSTRAP_SERVERS

SEC_PLAINTEXT

static final AllowableValue SEC_PLAINTEXT

SEC_SSL

static final AllowableValue SEC_SSL

SEC_SASL_PLAINTEXT

static final AllowableValue SEC_SASL_PLAINTEXT

SEC_SASL_SSL

static final AllowableValue SEC_SASL_SSL

KAFKA_SECURITY_PROTOCOL

static final PropertyDescriptor KAFKA_SECURITY_PROTOCOL

KERBEROS_PRINCIPAL

public static final PropertyDescriptor KERBEROS_PRINCIPAL

KERBEROS_KEYTAB

public static final PropertyDescriptor KERBEROS_KEYTAB

KERBEROS_CREDENTIALS_SERVICE

public static final PropertyDescriptor KERBEROS_CREDENTIALS_SERVICE

KAFKA_KERBEROS_SERVICE_NAME

static final PropertyDescriptor KAFKA_KERBEROS_SERVICE_NAME

LINEAGE_STRATEGY_SIMPLE_PATH

static final AllowableValue LINEAGE_STRATEGY_SIMPLE_PATH

LINEAGE_STRATEGY_COMPLETE_PATH

static final AllowableValue LINEAGE_STRATEGY_COMPLETE_PATH

LINEAGE_STRATEGY

static final PropertyDescriptor LINEAGE_STRATEGY

AWS_S3_MODEL_VERSION_V1

static final AllowableValue AWS_S3_MODEL_VERSION_V1

AWS_S3_MODEL_VERSION_V2

static final AllowableValue AWS_S3_MODEL_VERSION_V2

AWS_S3_MODEL_VERSION

static final PropertyDescriptor AWS_S3_MODEL_VERSION

FILESYSTEM_PATHS_LEVEL_FILE

static final AllowableValue FILESYSTEM_PATHS_LEVEL_FILE

FILESYSTEM_PATHS_LEVEL_DIRECTORY

static final AllowableValue FILESYSTEM_PATHS_LEVEL_DIRECTORY

FILESYSTEM_PATHS_LEVEL

static final PropertyDescriptor FILESYSTEM_PATHS_LEVEL

ATLAS_PROPERTIES_FILENAME

private static final String ATLAS_PROPERTIES_FILENAME

See Also:

Constant Field Values

ATLAS_PROPERTY_CLIENT_CONNECT_TIMEOUT_MS

private static final String ATLAS_PROPERTY_CLIENT_CONNECT_TIMEOUT_MS

See Also:

Constant Field Values

ATLAS_PROPERTY_CLIENT_READ_TIMEOUT_MS

private static final String ATLAS_PROPERTY_CLIENT_READ_TIMEOUT_MS

See Also:

Constant Field Values

ATLAS_PROPERTY_METADATA_NAMESPACE

private static final String ATLAS_PROPERTY_METADATA_NAMESPACE

See Also:

Constant Field Values

ATLAS_PROPERTY_CLUSTER_NAME

private static final String ATLAS_PROPERTY_CLUSTER_NAME

See Also:

Constant Field Values

ATLAS_PROPERTY_REST_ADDRESS

private static final String ATLAS_PROPERTY_REST_ADDRESS

See Also:

Constant Field Values

ATLAS_PROPERTY_ENABLE_TLS

private static final String ATLAS_PROPERTY_ENABLE_TLS

See Also:

Constant Field Values

ATLAS_KAFKA_PREFIX

private static final String ATLAS_KAFKA_PREFIX

See Also:

Constant Field Values

ATLAS_PROPERTY_KAFKA_BOOTSTRAP_SERVERS

private static final String ATLAS_PROPERTY_KAFKA_BOOTSTRAP_SERVERS

See Also:

Constant Field Values

ATLAS_PROPERTY_KAFKA_CLIENT_ID

private static final String ATLAS_PROPERTY_KAFKA_CLIENT_ID

See Also:

Constant Field Values

SSL_CLIENT_XML_FILENAME

private static final String SSL_CLIENT_XML_FILENAME

See Also:

Constant Field Values

SSL_CLIENT_XML_TRUSTSTORE_LOCATION

private static final String SSL_CLIENT_XML_TRUSTSTORE_LOCATION

See Also:

Constant Field Values

SSL_CLIENT_XML_TRUSTSTORE_PASSWORD

private static final String SSL_CLIENT_XML_TRUSTSTORE_PASSWORD

See Also:

Constant Field Values

SSL_CLIENT_XML_TRUSTSTORE_TYPE

private static final String SSL_CLIENT_XML_TRUSTSTORE_TYPE

See Also:

Constant Field Values

namespaceResolverLoader

private final ServiceLoader<NamespaceResolver> namespaceResolverLoader

atlasAuthN

private volatile AtlasAuthN atlasAuthN

atlasProperties

private volatile Properties atlasProperties

isTypeDefCreated

private volatile boolean isTypeDefCreated

defaultMetadataNamespace

private volatile String defaultMetadataNamespace

consumer

private volatile ProvenanceEventConsumer consumer

namespaceResolvers

private volatile NamespaceResolvers namespaceResolvers

nifiAtlasHook

private volatile NiFiAtlasHook nifiAtlasHook

lineageStrategy

private volatile LineageStrategy lineageStrategy

Constructor Detail

ReportLineageToAtlas

public ReportLineageToAtlas()

Method Detail

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getSupportedDynamicPropertyDescriptor

protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName)

Overrides:

getSupportedDynamicPropertyDescriptor in class AbstractConfigurableComponent

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext context)

Overrides:

customValidate in class AbstractConfigurableComponent

validateKafkaProperties

private void validateKafkaProperties(ValidationContext context,
                                     Collection<ValidationResult> results)

setup

@OnScheduled
public void setup(ConfigurationContext context)
                        throws Exception

Throws:

Exception

initLineageStrategy

private void initLineageStrategy(ConfigurationContext context)
                          throws IOException

Throws:

IOException

initNamespaceResolvers

private void initNamespaceResolvers(ConfigurationContext context)

initAtlasProperties

private void initAtlasProperties(ConfigurationContext context)
                          throws Exception

Throws:

Exception

parseAtlasUrls

private List<String> parseAtlasUrls(PropertyValue atlasUrlsProp)

setValue

private void setValue(Consumer<String> setter,
                      Runnable emptyHandler,
                      PropertyValue elEnabledPropertyValue,
                      String... properties)

setAtlasSSLConfig

private void setAtlasSSLConfig(Properties atlasProperties,
                               ConfigurationContext context,
                               List<String> urls,
                               File confDir)
                        throws Exception

Throws:

Exception

deleteSslClientXml

private void deleteSslClientXml(File confDir)
                         throws Exception

Throws:

Exception

createSslClientXml

private void createSslClientXml(File confDir,
                                SSLContextService sslContextService)
                         throws Exception

Throws:

Exception

createNiFiAtlasClient

protected NiFiAtlasClient createNiFiAtlasClient(ReportingContext context)

In order to avoid authentication expiration issues (i.e. Kerberos ticket and DelegationToken expiration), create Atlas client instance at every onTrigger execution.

getAtlasAuthN

private AtlasAuthN getAtlasAuthN(String atlasAuthNMethod)

initProvenanceConsumer

private void initProvenanceConsumer(ConfigurationContext context)
                             throws IOException

Throws:

IOException

onUnscheduled

@OnUnscheduled
public void onUnscheduled()

onStopped

@OnStopped
public void onStopped()

onTrigger

public void onTrigger(ReportingContext context)

createNiFiFlow

private NiFiFlow createNiFiFlow(ReportingContext context,
                                NiFiAtlasClient atlasClient)

consumeNiFiProvenanceEvents

private void consumeNiFiProvenanceEvents(ReportingContext context,
                                         NiFiFlow nifiFlow)

setKafkaConfig

private void setKafkaConfig(Map<Object,Object> mapToPopulate,
                            PropertyContext context)

setKafkaJaasConfig

private void setKafkaJaasConfig(Map<Object,Object> mapToPopulate,
                                PropertyContext context)

Populate Kafka JAAS properties for Atlas notification. Since Atlas 0.8.1 uses Kafka client 0.10.0.0, we can not use 'sasl.jaas.config' property as it is available since 0.10.2, implemented by KAFKA-4259. Instead, this method uses old property names.

Parameters:

mapToPopulate - Map of configuration properties

context - Context