001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file;
018
019 import java.util.ArrayList;
020 import java.util.Collections;
021 import java.util.List;
022
023 import org.apache.camel.AsyncCallback;
024 import org.apache.camel.Exchange;
025 import org.apache.camel.Processor;
026 import org.apache.camel.impl.DefaultExchange;
027 import org.apache.camel.impl.ScheduledPollConsumer;
028 import org.apache.camel.processor.DeadLetterChannel;
029 import org.apache.camel.util.ObjectHelper;
030 import org.apache.commons.logging.Log;
031 import org.apache.commons.logging.LogFactory;
032
033 /**
034 * Base class for remote file consumers.
035 */
036 public abstract class GenericFileConsumer<T> extends ScheduledPollConsumer {
037 protected final transient Log log = LogFactory.getLog(getClass());
038 protected GenericFileEndpoint<T> endpoint;
039 protected GenericFileOperations<T> operations;
040 protected boolean loggedIn;
041 protected String fileExpressionResult;
042
043 public GenericFileConsumer(GenericFileEndpoint<T> endpoint, Processor processor, GenericFileOperations<T> operations) {
044 super(endpoint, processor);
045 this.endpoint = endpoint;
046 this.operations = operations;
047 }
048
049 /**
050 * Poll for files
051 */
052 protected void poll() throws Exception {
053 // must reset for each poll
054 fileExpressionResult = null;
055
056 // before we poll is there anything we need to check ? Such as are we
057 // connected to the FTP Server Still ?
058 if (!prePollCheck()) {
059 log.debug("Skipping pool as pre poll check returned false");
060 }
061
062 // gather list of files to process
063 List<GenericFile<T>> files = new ArrayList<GenericFile<T>>();
064
065 String name = endpoint.getConfiguration().getDirectory();
066 pollDirectory(name, files);
067
068 // sort files using file comparator if provided
069 if (endpoint.getSorter() != null) {
070 Collections.sort(files, endpoint.getSorter());
071 }
072
073 // sort using build in sorters that is expression based
074 // first we need to convert to RemoteFileExchange objects so we can sort
075 // using expressions
076 List<GenericFileExchange<T>> exchanges = new ArrayList<GenericFileExchange<T>>(files.size());
077 for (GenericFile<T> file : files) {
078 GenericFileExchange<T> exchange = endpoint.createExchange(file);
079 endpoint.configureMessage(file, exchange.getIn());
080 exchanges.add(exchange);
081 }
082 // sort files using exchange comparator if provided
083 if (endpoint.getSortBy() != null) {
084 Collections.sort(exchanges, endpoint.getSortBy());
085 }
086
087 // consume files one by one
088 int total = exchanges.size();
089 if (total > 0 && log.isDebugEnabled()) {
090 log.debug("Total " + total + " files to consume");
091 }
092 for (int index = 0; index < total && isRunAllowed(); index++) {
093 // only loop if we are started (allowed to run)
094 GenericFileExchange<T> exchange = exchanges.get(index);
095 // add current index and total as headers
096 exchange.getIn().setHeader(Exchange.FILE_BATCH_INDEX, index);
097 exchange.getIn().setHeader(Exchange.FILE_BATCH_SIZE, total);
098 processExchange(exchange);
099 }
100 }
101
102 /**
103 * Override if required. Perform some checks (and perhaps actions) before we
104 * poll.
105 *
106 * @return true to poll, false to skip this poll.
107 */
108 protected boolean prePollCheck() throws Exception {
109 return true;
110 }
111
112 /**
113 * Polls the given directory for files to process
114 *
115 * @param fileName current directory or file
116 * @param fileList current list of files gathered
117 */
118 protected abstract void pollDirectory(String fileName, List<GenericFile<T>> fileList);
119
120 /**
121 * Processes the exchange
122 *
123 * @param exchange the exchange
124 */
125 protected void processExchange(final GenericFileExchange<T> exchange) {
126 if (log.isTraceEnabled()) {
127 log.trace("Processing remote file: " + exchange.getGenericFile());
128 }
129
130 try {
131 final GenericFileProcessStrategy<T> processStrategy = endpoint.getGenericFileProcessStrategy();
132
133 if (processStrategy.begin(operations, endpoint, exchange, exchange.getGenericFile())) {
134
135 // must use file from exchange as it can be updated due the
136 // preMoveNamePrefix/preMoveNamePostfix options
137 final GenericFile<T> target = exchange.getGenericFile();
138 // must use full name when downloading so we have the correct path
139 final String name = target.getAbsoluteFilePath();
140
141 // retrieve the file using the stream
142 if (log.isTraceEnabled()) {
143 log.trace("Retreiving file: " + name + " from: " + endpoint);
144 }
145
146 operations.retrieveFile(name, exchange);
147
148 if (log.isTraceEnabled()) {
149 log.trace("Retrieved file: " + name + " from: " + endpoint);
150 }
151
152 if (log.isDebugEnabled()) {
153 log.debug("About to process file: " + target + " using exchange: " + exchange);
154 }
155 // Use the async processor interface so that processing of
156 // the exchange can happen asynchronously
157 getAsyncProcessor().process(exchange, new AsyncCallback() {
158 public void done(boolean sync) {
159 final GenericFile<T> file = exchange.getGenericFile();
160 boolean failed = exchange.isFailed();
161
162 if (log.isDebugEnabled()) {
163 log.debug("Done processing file: " + file + " using exchange: " + exchange);
164 }
165
166 boolean committed = false;
167 try {
168 if (!failed) {
169 // commit the file strategy if there was no failure or already handled by the DeadLetterChannel
170 processStrategyCommit(processStrategy, exchange, file);
171 committed = true;
172 } else {
173 // there was an exception but it was not handled by the DeadLetterChannel
174 handleException(exchange.getException());
175 }
176 } finally {
177 if (!committed) {
178 processStrategyRollback(processStrategy, exchange, file);
179 }
180 }
181 }
182 });
183 } else {
184 log.warn(endpoint + " cannot process remote file: " + exchange.getGenericFile());
185 }
186 } catch (Exception e) {
187 handleException(e);
188 }
189
190 }
191
192 /**
193 * Strategy when the file was processed and a commit should be executed.
194 *
195 * @param processStrategy the strategy to perform the commit
196 * @param exchange the exchange
197 * @param file the file processed
198 */
199 @SuppressWarnings("unchecked")
200 protected void processStrategyCommit(GenericFileProcessStrategy<T> processStrategy,
201 GenericFileExchange<T> exchange, GenericFile<T> file) {
202 if (endpoint.isIdempotent()) {
203 // only add to idempotent repository if we could process the file
204 // only use the filename as the key as the file could be moved into a done folder
205 endpoint.getIdempotentRepository().add(file.getFileName());
206 }
207
208 try {
209 if (log.isTraceEnabled()) {
210 log.trace("Committing remote file strategy: " + processStrategy + " for file: " + file);
211 }
212 processStrategy.commit(operations, endpoint, exchange, file);
213 } catch (Exception e) {
214 handleException(e);
215 }
216 }
217
218 /**
219 * Strategy when the file was not processed and a rollback should be
220 * executed.
221 *
222 * @param processStrategy the strategy to perform the commit
223 * @param exchange the exchange
224 * @param file the file processed
225 */
226 protected void processStrategyRollback(GenericFileProcessStrategy<T> processStrategy,
227 GenericFileExchange<T> exchange, GenericFile<T> file) {
228 if (log.isWarnEnabled()) {
229 log.warn("Rolling back remote file strategy: " + processStrategy + " for file: " + file);
230 }
231 try {
232 processStrategy.rollback(operations, endpoint, exchange, file);
233 } catch (Exception e) {
234 handleException(e);
235 }
236 }
237
238 /**
239 * Strategy for validating if the given remote file should be included or
240 * not
241 *
242 * @param file the remote file
243 * @param isDirectory wether the file is a directory or a file
244 * @return <tt>true</tt> to include the file, <tt>false</tt> to skip it
245 */
246 @SuppressWarnings("unchecked")
247 protected boolean isValidFile(GenericFile<T> file, boolean isDirectory) {
248 if (!isMatched(file, isDirectory)) {
249 if (log.isTraceEnabled()) {
250 log.trace("Remote file did not match. Will skip this remote file: " + file);
251 }
252 return false;
253 } else if (endpoint.isIdempotent() && endpoint.getIdempotentRepository().contains(file.getFileName())) {
254 // only use the filename as the key as the file could be moved into a done folder
255 if (log.isTraceEnabled()) {
256 log.trace("RemoteFileConsumer is idempotent and the file has been consumed before. Will skip this remote file: " + file);
257 }
258 return false;
259 }
260
261 // file matched
262 return true;
263 }
264
265 /**
266 * Strategy to perform file matching based on endpoint configuration.
267 * <p/>
268 * Will always return <tt>false</tt> for certain files/folders:
269 * <ul>
270 * <li>Starting with a dot</li>
271 * <li>lock files</li>
272 * </ul>
273 * And then <tt>true</tt> for directories.
274 *
275 * @param file the remote file
276 * @param isDirectory wether the file is a directory or a file
277 * @return <tt>true</tt> if the remote file is matched, <tt>false</tt> if not
278 */
279 protected boolean isMatched(GenericFile<T> file, boolean isDirectory) {
280 String name = file.getFileNameOnly();
281
282 // folders/names starting with dot is always skipped (eg. ".", ".camel", ".camelLock")
283 if (name.startsWith(".")) {
284 return false;
285 }
286
287 // lock files should be skipped
288 if (name.endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX)) {
289 return false;
290 }
291
292 // directories so far is always regarded as matched (matching on the name is only for files)
293 if (isDirectory) {
294 return true;
295 }
296
297 if (endpoint.getFilter() != null) {
298 if (!endpoint.getFilter().accept(file)) {
299 return false;
300 }
301 }
302
303 if (ObjectHelper.isNotEmpty(endpoint.getExclude())) {
304 if (name.matches(endpoint.getExclude())) {
305 return false;
306 }
307 }
308
309 if (ObjectHelper.isNotEmpty(endpoint.getInclude())) {
310 if (!name.matches(endpoint.getInclude())) {
311 return false;
312 }
313 }
314
315 // use file expression for a simple dynamic file filter
316 if (endpoint.getFileName() != null) {
317 evaluteFileExpression();
318 if (fileExpressionResult != null) {
319 if (!name.equals(fileExpressionResult)) {
320 return false;
321 }
322 }
323 }
324
325 return true;
326 }
327
328 private void evaluteFileExpression() {
329 if (fileExpressionResult == null) {
330 // create a dummy exchange as Exchange is needed for expression evaluation
331 Exchange dummy = new DefaultExchange(endpoint.getCamelContext());
332 fileExpressionResult = endpoint.getFileName().evaluate(dummy, String.class);
333 }
334 }
335
336
337 }