001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file.remote;
018
019 import java.io.ByteArrayOutputStream;
020 import java.io.File;
021 import java.io.IOException;
022 import java.util.concurrent.ScheduledExecutorService;
023
024 import org.apache.camel.Processor;
025 import org.apache.camel.component.file.FileComponent;
026 import org.apache.commons.net.ftp.FTPClient;
027 import org.apache.commons.net.ftp.FTPFile;
028
029 public class FtpConsumer extends RemoteFileConsumer<RemoteFileExchange> {
030
031 private FtpEndpoint endpoint;
032 private FTPClient client;
033 private boolean loggedIn;
034
035 public FtpConsumer(FtpEndpoint endpoint, Processor processor, FTPClient client) {
036 super(endpoint, processor);
037 this.endpoint = endpoint;
038 this.client = client;
039 }
040
041 public FtpConsumer(FtpEndpoint endpoint, Processor processor, FTPClient client,
042 ScheduledExecutorService executor) {
043 super(endpoint, processor, executor);
044 this.endpoint = endpoint;
045 this.client = client;
046 }
047
048 protected void doStart() throws Exception {
049 log.info("Starting");
050 super.doStart();
051 }
052
053 protected void doStop() throws Exception {
054 log.info("Stopping");
055 // disconnect when stopping
056 try {
057 disconnect();
058 } catch (Exception e) {
059 // ignore just log a warning
060 String message = "Could not disconnect from " + remoteServer()
061 + ". Reason: " + client.getReplyString() + ". Code: " + client.getReplyCode();
062 log.warn(message);
063 }
064 super.doStop();
065 }
066
067 protected void connectIfNecessary() throws IOException {
068 if (!client.isConnected() || !loggedIn) {
069 if (log.isDebugEnabled()) {
070 log.debug("Not connected/logged in, connecting to " + remoteServer());
071 }
072 loggedIn = FtpUtils.connect(client, endpoint.getConfiguration());
073 if (!loggedIn) {
074 return;
075 }
076 }
077
078 log.info("Connected and logged in to " + remoteServer());
079 }
080
081 protected void disconnect() throws IOException {
082 loggedIn = false;
083 log.debug("Disconnecting from " + remoteServer());
084 FtpUtils.disconnect(client);
085 }
086
087 protected void poll() throws Exception {
088 if (log.isTraceEnabled()) {
089 log.trace("Polling " + endpoint.getConfiguration());
090 }
091
092 try {
093 connectIfNecessary();
094
095 if (!loggedIn) {
096 String message = "Could not connect/login to " + endpoint.getConfiguration();
097 log.warn(message);
098 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
099 }
100
101 final String fileName = endpoint.getConfiguration().getFile();
102 if (endpoint.getConfiguration().isDirectory()) {
103 pollDirectory(fileName);
104 } else {
105 int index = fileName.lastIndexOf('/');
106 if (index > -1) {
107 // cd to the folder of the filename
108 client.changeWorkingDirectory(fileName.substring(0, index));
109 }
110 // list the files in the fold and poll the first file
111 final FTPFile[] files = client.listFiles(fileName.substring(index + 1));
112 if (files != null && files.length > 0) {
113 pollFile(files[0]);
114 }
115 }
116
117 lastPollTime = System.currentTimeMillis();
118
119 } catch (Exception e) {
120 loggedIn = false;
121 if (isStopping() || isStopped()) {
122 // if we are stopping then ignore any exception during a poll
123 log.warn("Consumer is stopping. Ignoring caught exception: "
124 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
125 } else {
126 log.warn("Exception occured during polling: "
127 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
128 disconnect();
129 // Rethrow to signify that we didn't poll
130 throw e;
131 }
132 }
133 }
134
135 protected void pollDirectory(String dir) throws Exception {
136 if (log.isTraceEnabled()) {
137 log.trace("Polling directory: " + dir);
138 }
139 String currentDir = client.printWorkingDirectory();
140
141 client.changeWorkingDirectory(dir);
142 for (FTPFile ftpFile : client.listFiles()) {
143 if (ftpFile.isFile()) {
144 pollFile(ftpFile);
145 } else if (ftpFile.isDirectory()) {
146 if (isRecursive()) {
147 pollDirectory(getFullFileName(ftpFile));
148 }
149 } else {
150 log.debug("Unsupported type of FTPFile: " + ftpFile + " (not a file or directory). It is skipped.");
151 }
152 }
153
154 // change back to original current dir
155 client.changeWorkingDirectory(currentDir);
156 }
157
158 protected String getFullFileName(FTPFile ftpFile) throws IOException {
159 return client.printWorkingDirectory() + "/" + ftpFile.getName();
160 }
161
162 private void pollFile(FTPFile ftpFile) throws Exception {
163 if (ftpFile == null) {
164 return;
165 }
166
167 if (log.isTraceEnabled()) {
168 log.trace("Polling file: " + ftpFile);
169 }
170
171 // if using last polltime for timestamp matcing (to be removed in Camel 2.0)
172 boolean timestampMatched = true;
173 if (isTimestamp()) {
174 // TODO do we need to adjust the TZ? can we?
175 long ts = ftpFile.getTimestamp().getTimeInMillis();
176 timestampMatched = ts > lastPollTime;
177 if (log.isTraceEnabled()) {
178 log.trace("The file is to old + " + ftpFile + ". lastPollTime=" + lastPollTime + " > fileTimestamp=" + ts);
179 }
180 }
181
182 if (timestampMatched && isMatched(ftpFile)) {
183 String fullFileName = getFullFileName(ftpFile);
184
185 // is we use excluse read then acquire the exclusive read (waiting until we got it)
186 if (exclusiveReadLock) {
187 acquireExclusiveReadLock(client, ftpFile);
188 }
189
190 // retrieve the file
191 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
192 client.retrieveFile(ftpFile.getName(), byteArrayOutputStream);
193 if (log.isDebugEnabled()) {
194 log.debug("Retrieved file: " + ftpFile.getName() + " from: " + remoteServer());
195 }
196
197 RemoteFileExchange exchange = endpoint.createExchange(fullFileName, ftpFile.getName(),
198 ftpFile.getSize(), byteArrayOutputStream);
199
200 if (isSetNames()) {
201 // set the filename in the special header filename marker to the ftp filename
202 String ftpBasePath = endpoint.getConfiguration().getFile();
203 String relativePath = fullFileName.substring(ftpBasePath.length() + 1);
204 relativePath = relativePath.replaceFirst("/", "");
205
206 if (log.isDebugEnabled()) {
207 log.debug("Setting exchange filename to " + relativePath);
208 }
209 exchange.getIn().setHeader(FileComponent.HEADER_FILE_NAME, relativePath);
210 }
211
212
213 // all success so lets process it
214 getProcessor().process(exchange);
215
216 if (exchange.isFailed()) {
217 if (log.isDebugEnabled()) {
218 log.debug("Processing of exchange failed, so cannot do FTP post command such as move or delete: " + exchange);
219 }
220 } else {
221 // after processing then do post command such as delete or move
222 if (deleteFile) {
223 // delete file after consuming
224 if (log.isDebugEnabled()) {
225 log.debug("Deleteing file: " + ftpFile.getName() + " from: " + remoteServer());
226 }
227 boolean deleted = client.deleteFile(ftpFile.getName());
228 if (!deleted) {
229 String message = "Can not delete file: " + ftpFile.getName() + " from: " + remoteServer();
230 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
231 }
232 } else if (isMoveFile()) {
233 String fromName = ftpFile.getName();
234 String toName = getMoveFileName(fromName, exchange);
235 if (log.isDebugEnabled()) {
236 log.debug("Moving file: " + fromName + " to: " + toName);
237 }
238
239 // delete any existing file
240 boolean deleted = client.deleteFile(toName);
241 if (!deleted) {
242 // if we could not delete any existing file then maybe the folder is missing
243 // build folder if needed
244 int lastPathIndex = toName.lastIndexOf('/');
245 if (lastPathIndex != -1) {
246 String directory = toName.substring(0, lastPathIndex);
247 if (!FtpUtils.buildDirectory(client, directory)) {
248 log.warn("Can not build directory: " + directory + " (maybe because of denied permissions)");
249 }
250 }
251 }
252
253 // try to rename
254 boolean success = client.rename(fromName, toName);
255 if (!success) {
256 String message = "Can not move file: " + fromName + " to: " + toName;
257 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
258 }
259 }
260 }
261
262 }
263 }
264
265 protected void acquireExclusiveReadLock(FTPClient client, FTPFile ftpFile) throws IOException {
266 if (log.isTraceEnabled()) {
267 log.trace("Waiting for exclusive read lock to file: " + ftpFile);
268 }
269
270 // the trick is to try to rename the file, if we can rename then we have exclusive read
271 // since its a remote file we can not use java.nio to get a RW lock
272 String originalName = ftpFile.getName();
273 String newName = originalName + ".camelExclusiveReadLock";
274 boolean exclusive = false;
275 while (!exclusive) {
276 exclusive = client.rename(originalName, newName);
277 if (exclusive) {
278 if (log.isDebugEnabled()) {
279 log.debug("Acquired exclusive read lock to file: " + originalName);
280 }
281 // rename it back so we can read it
282 client.rename(newName, originalName);
283 } else {
284 log.trace("Exclusive read lock not granted. Sleeping for 1000 millis.");
285 try {
286 Thread.sleep(1000);
287 } catch (InterruptedException e) {
288 // ignore
289 }
290 }
291 }
292 }
293
294 protected String getFileName(Object file) {
295 FTPFile ftpFile = (FTPFile) file;
296 return ftpFile.getName();
297 }
298
299 }