@PrimaryNodeOnly @InputRequirement(value=INPUT_FORBIDDEN) @TriggerSerially @TriggerWhenEmpty @Tags(value={"airtable","query","database"}) @CapabilityDescription(value="Query records from an Airtable table. Records are incrementally retrieved based on the last modified time of the records. Records can also be further filtered by setting the \'Custom Filter\' property which supports the formulas provided by the Airtable API. This processor is intended to be run on the Primary Node only.") @Stateful(scopes=CLUSTER, description="The last successful query\'s time is stored in order to enable incremental loading. The initial query returns all the records in the table and each subsequent query filters the records by their last modified time. In other words, if a record is updated after the last successful query only the updated records will be returned in the next query. State is stored across the cluster, so this Processor can run only on the Primary Node and if a new Primary Node is selected, the new node can pick up where the previous one left off without duplicating the data.") @WritesAttribute(attribute="record.count",description="Sets the number of records in the FlowFile.") @WritesAttribute(attribute="fragment.identifier",description="If \'Max Records Per FlowFile\' is set then all FlowFiles from the same query result set will have the same value for the fragment.identifier attribute. This can then be used to correlate the results.") @WritesAttribute(attribute="fragment.count",description="If \'Max Records Per FlowFile\' is set then this is the total number of FlowFiles produced by a single ResultSet. This can be used in conjunction with the fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet.") @WritesAttribute(attribute="fragment.index",description="If \'Max Records Per FlowFile\' is set then the position of this FlowFile in the list of outgoing FlowFiles that were all derived from the same result set FlowFile. This can be used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order FlowFiles were produced") @DefaultSettings(yieldDuration="15 sec") @DefaultSchedule(strategy=TIMER_DRIVEN, period="1 min") public class QueryAirtableTable extends AbstractProcessor
| Modifier and Type | Field and Description |
|---|---|
private AirtableRestService |
airtableRestService |
(package private) static PropertyDescriptor |
API_KEY |
(package private) static PropertyDescriptor |
API_URL |
(package private) static PropertyDescriptor |
BASE_ID |
(package private) static PropertyDescriptor |
CUSTOM_FILTER |
(package private) static PropertyDescriptor |
FIELDS |
private static String |
LAST_QUERY_TIME_WINDOW_END |
(package private) static PropertyDescriptor |
MAX_RECORDS_PER_FLOWFILE |
private static List<PropertyDescriptor> |
PROPERTIES |
(package private) static PropertyDescriptor |
QUERY_PAGE_SIZE |
(package private) static PropertyDescriptor |
QUERY_TIME_WINDOW_LAG |
(package private) static Relationship |
REL_SUCCESS |
private static Set<Relationship> |
RELATIONSHIPS |
(package private) static PropertyDescriptor |
TABLE_ID |
(package private) static PropertyDescriptor |
WEB_CLIENT_SERVICE_PROVIDER |
| Constructor and Description |
|---|
QueryAirtableTable() |
| Modifier and Type | Method and Description |
|---|---|
private void |
addFragmentAttributesToFlowFiles(ProcessSession session,
List<FlowFile> flowFiles) |
private AirtableGetRecordsParameters |
buildGetRecordsParameters(ProcessContext context,
String lastRecordFetchTime,
String nowDateTimeString) |
Set<Relationship> |
getRelationships() |
protected List<PropertyDescriptor> |
getSupportedPropertyDescriptors() |
void |
onScheduled(ProcessContext context) |
void |
onTrigger(ProcessContext context,
ProcessSession session) |
private void |
transferFlowFiles(ProcessSession session,
List<FlowFile> flowFiles,
int totalRecordCount) |
onTriggergetControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTruecustomValidate, equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validateclone, finalize, getClass, notify, notifyAll, wait, wait, waitisStatefulgetPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validatestatic final PropertyDescriptor API_URL
static final PropertyDescriptor API_KEY
static final PropertyDescriptor BASE_ID
static final PropertyDescriptor TABLE_ID
static final PropertyDescriptor FIELDS
static final PropertyDescriptor CUSTOM_FILTER
static final PropertyDescriptor QUERY_TIME_WINDOW_LAG
static final PropertyDescriptor WEB_CLIENT_SERVICE_PROVIDER
static final PropertyDescriptor QUERY_PAGE_SIZE
static final PropertyDescriptor MAX_RECORDS_PER_FLOWFILE
static final Relationship REL_SUCCESS
private static final List<PropertyDescriptor> PROPERTIES
private static final Set<Relationship> RELATIONSHIPS
private static final String LAST_QUERY_TIME_WINDOW_END
private volatile AirtableRestService airtableRestService
protected List<PropertyDescriptor> getSupportedPropertyDescriptors()
getSupportedPropertyDescriptors in class AbstractConfigurableComponentpublic Set<Relationship> getRelationships()
getRelationships in interface ProcessorgetRelationships in class AbstractSessionFactoryProcessor@OnScheduled public void onScheduled(ProcessContext context)
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException
onTrigger in class AbstractProcessorProcessExceptionprivate AirtableGetRecordsParameters buildGetRecordsParameters(ProcessContext context, String lastRecordFetchTime, String nowDateTimeString)
private void addFragmentAttributesToFlowFiles(ProcessSession session, List<FlowFile> flowFiles)
private void transferFlowFiles(ProcessSession session, List<FlowFile> flowFiles, int totalRecordCount)
Copyright © 2023 Apache NiFi Project. All rights reserved.