001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file;
018
019 import java.util.ArrayList;
020 import java.util.Collections;
021 import java.util.LinkedList;
022 import java.util.List;
023 import java.util.Queue;
024
025 import org.apache.camel.BatchConsumer;
026 import org.apache.camel.Exchange;
027 import org.apache.camel.Processor;
028 import org.apache.camel.ShutdownRunningTask;
029 import org.apache.camel.impl.DefaultExchange;
030 import org.apache.camel.impl.ScheduledPollConsumer;
031 import org.apache.camel.spi.ShutdownAware;
032 import org.apache.camel.util.CastUtils;
033 import org.apache.camel.util.ObjectHelper;
034 import org.apache.commons.logging.Log;
035 import org.apache.commons.logging.LogFactory;
036
037 /**
038 * Base class for remote file consumers.
039 */
040 public abstract class GenericFileConsumer<T> extends ScheduledPollConsumer implements BatchConsumer, ShutdownAware {
041 protected final transient Log log = LogFactory.getLog(getClass());
042 protected GenericFileEndpoint<T> endpoint;
043 protected GenericFileOperations<T> operations;
044 protected boolean loggedIn;
045 protected String fileExpressionResult;
046 protected int maxMessagesPerPoll;
047 protected volatile ShutdownRunningTask shutdownRunningTask;
048 protected volatile int pendingExchanges;
049
050 public GenericFileConsumer(GenericFileEndpoint<T> endpoint, Processor processor, GenericFileOperations<T> operations) {
051 super(endpoint, processor);
052 this.endpoint = endpoint;
053 this.operations = operations;
054 }
055
056 /**
057 * Poll for files
058 */
059 protected void poll() throws Exception {
060 // must reset for each poll
061 fileExpressionResult = null;
062 shutdownRunningTask = null;
063 pendingExchanges = 0;
064
065 // before we poll is there anything we need to check ? Such as are we
066 // connected to the FTP Server Still ?
067 if (!prePollCheck()) {
068 if (log.isDebugEnabled()) {
069 log.debug("Skipping pool as pre poll check returned false");
070 }
071 }
072
073 // gather list of files to process
074 List<GenericFile<T>> files = new ArrayList<GenericFile<T>>();
075
076 String name = endpoint.getConfiguration().getDirectory();
077 pollDirectory(name, files);
078
079 // sort files using file comparator if provided
080 if (endpoint.getSorter() != null) {
081 Collections.sort(files, endpoint.getSorter());
082 }
083
084 // sort using build in sorters so we can use expressions
085 LinkedList<Exchange> exchanges = new LinkedList<Exchange>();
086 for (GenericFile<T> file : files) {
087 Exchange exchange = endpoint.createExchange(file);
088 endpoint.configureMessage(file, exchange.getIn());
089 exchanges.add(exchange);
090 }
091 // sort files using exchange comparator if provided
092 if (endpoint.getSortBy() != null) {
093 Collections.sort(exchanges, endpoint.getSortBy());
094 }
095
096 // consume files one by one
097 int total = exchanges.size();
098 if (total > 0 && log.isDebugEnabled()) {
099 log.debug("Total " + total + " files to consume");
100 }
101
102 Queue<Exchange> q = exchanges;
103 processBatch(CastUtils.cast(q));
104
105 postPollCheck();
106 }
107
108 public void setMaxMessagesPerPoll(int maxMessagesPerPoll) {
109 this.maxMessagesPerPoll = maxMessagesPerPoll;
110 }
111
112 @SuppressWarnings("unchecked")
113 public void processBatch(Queue<Object> exchanges) {
114 int total = exchanges.size();
115
116 // limit if needed
117 if (maxMessagesPerPoll > 0 && total > maxMessagesPerPoll) {
118 if (log.isDebugEnabled()) {
119 log.debug("Limiting to maximum messages to poll " + maxMessagesPerPoll + " as there was " + total + " messages in this poll.");
120 }
121 total = maxMessagesPerPoll;
122 }
123
124 for (int index = 0; index < total && isBatchAllowed(); index++) {
125 // only loop if we are started (allowed to run)
126 // use poll to remove the head so it does not consume memory even after we have processed it
127 Exchange exchange = (Exchange) exchanges.poll();
128 // add current index and total as properties
129 exchange.setProperty(Exchange.BATCH_INDEX, index);
130 exchange.setProperty(Exchange.BATCH_SIZE, total);
131 exchange.setProperty(Exchange.BATCH_COMPLETE, index == total - 1);
132
133 // update pending number of exchanges
134 pendingExchanges = total - index - 1;
135
136 // process the current exchange
137 processExchange(exchange);
138 }
139
140 // remove the file from the in progress list in case the batch was limited by max messages per poll
141 while (exchanges.size() > 0) {
142 Exchange exchange = (Exchange) exchanges.poll();
143 GenericFile<T> file = (GenericFile<T>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
144 String key = file.getFileName();
145 endpoint.getInProgressRepository().remove(key);
146 }
147 }
148
149 public boolean deferShutdown(ShutdownRunningTask shutdownRunningTask) {
150 // store a reference what to do in case when shutting down and we have pending messages
151 this.shutdownRunningTask = shutdownRunningTask;
152 // do not defer shutdown
153 return false;
154 }
155
156 public int getPendingExchangesSize() {
157 // only return the real pending size in case we are configured to complete all tasks
158 if (ShutdownRunningTask.CompleteAllTasks == shutdownRunningTask) {
159 return pendingExchanges;
160 } else {
161 return 0;
162 }
163 }
164
165 public boolean isBatchAllowed() {
166 // stop if we are not running
167 boolean answer = isRunAllowed();
168 if (!answer) {
169 return false;
170 }
171
172 if (shutdownRunningTask == null) {
173 // we are not shutting down so continue to run
174 return true;
175 }
176
177 // we are shutting down so only continue if we are configured to complete all tasks
178 return ShutdownRunningTask.CompleteAllTasks == shutdownRunningTask;
179 }
180
181 /**
182 * Override if required. Perform some checks (and perhaps actions) before we
183 * poll.
184 *
185 * @return true to poll, false to skip this poll.
186 */
187 protected boolean prePollCheck() throws Exception {
188 return true;
189 }
190
191 /**
192 * Override if required. Perform some checks (and perhaps actions) after we
193 * have polled.
194 */
195 protected void postPollCheck() {
196 // noop
197 }
198
199 /**
200 * Polls the given directory for files to process
201 *
202 * @param fileName current directory or file
203 * @param fileList current list of files gathered
204 */
205 protected abstract void pollDirectory(String fileName, List<GenericFile<T>> fileList);
206
207 /**
208 * Processes the exchange
209 *
210 * @param exchange the exchange
211 */
212 protected void processExchange(final Exchange exchange) {
213 GenericFile<T> file = getExchangeFileProperty(exchange);
214 if (log.isTraceEnabled()) {
215 log.trace("Processing file: " + file);
216 }
217
218 try {
219 final GenericFileProcessStrategy<T> processStrategy = endpoint.getGenericFileProcessStrategy();
220
221 boolean begin = processStrategy.begin(operations, endpoint, exchange, file);
222 if (!begin) {
223 if (log.isDebugEnabled()) {
224 log.debug(endpoint + " cannot begin processing file: " + file);
225 }
226 // remove file from the in progress list as its no longer in progress
227 endpoint.getInProgressRepository().remove(file.getFileName());
228 return;
229 }
230
231 // must use file from exchange as it can be updated due the
232 // preMoveNamePrefix/preMoveNamePostfix options
233 final GenericFile<T> target = getExchangeFileProperty(exchange);
234 // must use full name when downloading so we have the correct path
235 final String name = target.getAbsoluteFilePath();
236
237 // retrieve the file using the stream
238 if (log.isTraceEnabled()) {
239 log.trace("Retrieving file: " + name + " from: " + endpoint);
240 }
241
242 operations.retrieveFile(name, exchange);
243
244 if (log.isTraceEnabled()) {
245 log.trace("Retrieved file: " + name + " from: " + endpoint);
246 }
247
248 if (log.isDebugEnabled()) {
249 log.debug("About to process file: " + target + " using exchange: " + exchange);
250 }
251
252 // register on completion callback that does the completion strategies
253 // (for instance to move the file after we have processed it)
254 String originalFileName = file.getFileName();
255 exchange.addOnCompletion(new GenericFileOnCompletion<T>(endpoint, operations, target, originalFileName));
256
257 // process the exchange
258 getProcessor().process(exchange);
259
260 } catch (Exception e) {
261 handleException(e);
262 }
263 }
264
265 /**
266 * Strategy for validating if the given remote file should be included or
267 * not
268 *
269 * @param file the remote file
270 * @param isDirectory whether the file is a directory or a file
271 * @return <tt>true</tt> to include the file, <tt>false</tt> to skip it
272 */
273 @SuppressWarnings("unchecked")
274 protected boolean isValidFile(GenericFile<T> file, boolean isDirectory) {
275 if (!isMatched(file, isDirectory)) {
276 if (log.isTraceEnabled()) {
277 log.trace("File did not match. Will skip this file: " + file);
278 }
279 return false;
280 } else if (endpoint.isIdempotent() && endpoint.getIdempotentRepository().contains(file.getFileName())) {
281 // only use the filename as the key as the file could be moved into a done folder
282 if (log.isTraceEnabled()) {
283 log.trace("This consumer is idempotent and the file has been consumed before. Will skip this file: " + file);
284 }
285 return false;
286 }
287
288 // file matched
289 return true;
290 }
291
292 /**
293 * Strategy to perform file matching based on endpoint configuration.
294 * <p/>
295 * Will always return <tt>false</tt> for certain files/folders:
296 * <ul>
297 * <li>Starting with a dot</li>
298 * <li>lock files</li>
299 * </ul>
300 * And then <tt>true</tt> for directories.
301 *
302 * @param file the file
303 * @param isDirectory whether the file is a directory or a file
304 * @return <tt>true</tt> if the remote file is matched, <tt>false</tt> if not
305 */
306 protected boolean isMatched(GenericFile<T> file, boolean isDirectory) {
307 String name = file.getFileNameOnly();
308
309 // folders/names starting with dot is always skipped (eg. ".", ".camel", ".camelLock")
310 if (name.startsWith(".")) {
311 return false;
312 }
313
314 // lock files should be skipped
315 if (name.endsWith(FileComponent.DEFAULT_LOCK_FILE_POSTFIX)) {
316 return false;
317 }
318
319 // directories so far is always regarded as matched (matching on the name is only for files)
320 if (isDirectory) {
321 return true;
322 }
323
324 if (endpoint.getFilter() != null) {
325 if (!endpoint.getFilter().accept(file)) {
326 return false;
327 }
328 }
329
330 if (ObjectHelper.isNotEmpty(endpoint.getExclude())) {
331 if (name.matches(endpoint.getExclude())) {
332 return false;
333 }
334 }
335
336 if (ObjectHelper.isNotEmpty(endpoint.getInclude())) {
337 if (!name.matches(endpoint.getInclude())) {
338 return false;
339 }
340 }
341
342 // use file expression for a simple dynamic file filter
343 if (endpoint.getFileName() != null) {
344 evaluateFileExpression();
345 if (fileExpressionResult != null) {
346 if (!name.equals(fileExpressionResult)) {
347 return false;
348 }
349 }
350 }
351
352 return true;
353 }
354
355 /**
356 * Is the given file already in progress.
357 *
358 * @param file the file
359 * @return <tt>true</tt> if the file is already in progress
360 */
361 protected boolean isInProgress(GenericFile<T> file) {
362 String key = file.getFileName();
363 return !endpoint.getInProgressRepository().add(key);
364 }
365
366 private void evaluateFileExpression() {
367 if (fileExpressionResult == null) {
368 // create a dummy exchange as Exchange is needed for expression evaluation
369 Exchange dummy = new DefaultExchange(endpoint.getCamelContext());
370 fileExpressionResult = endpoint.getFileName().evaluate(dummy, String.class);
371 }
372 }
373
374 @SuppressWarnings("unchecked")
375 private GenericFile<T> getExchangeFileProperty(Exchange exchange) {
376 return (GenericFile<T>) exchange.getProperty(FileComponent.FILE_EXCHANGE_FILE);
377 }
378
379 @Override
380 protected void doStart() throws Exception {
381 super.doStart();
382
383 // prepare on startup
384 endpoint.getGenericFileProcessStrategy().prepareOnStartup(operations, endpoint);
385 }
386 }