001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file.remote;
018
019 import java.io.ByteArrayOutputStream;
020 import java.io.IOException;
021 import java.util.Vector;
022 import java.util.concurrent.ScheduledExecutorService;
023
024 import com.jcraft.jsch.ChannelSftp;
025 import com.jcraft.jsch.JSchException;
026 import com.jcraft.jsch.Session;
027 import com.jcraft.jsch.SftpException;
028
029 import org.apache.camel.Processor;
030 import org.apache.camel.component.file.FileComponent;
031
032 public class SftpConsumer extends RemoteFileConsumer<RemoteFileExchange> {
033 private final SftpEndpoint endpoint;
034
035 private ChannelSftp channel;
036 private Session session;
037
038 public SftpConsumer(SftpEndpoint endpoint, Processor processor, Session session) {
039 super(endpoint, processor);
040 this.endpoint = endpoint;
041 this.session = session;
042 }
043
044 public SftpConsumer(SftpEndpoint endpoint, Processor processor, Session session, ScheduledExecutorService executor) {
045 super(endpoint, processor, executor);
046 this.endpoint = endpoint;
047 this.session = session;
048 }
049
050 protected void doStart() throws Exception {
051 log.info("Starting");
052 super.doStart();
053 }
054
055 protected void doStop() throws Exception {
056 log.info("Stopping");
057 // disconnect when stopping
058 try {
059 disconnect();
060 } catch (Exception e) {
061 // ignore just log a warning
062 log.warn("Exception occured during disconecting from " + remoteServer() + ". "
063 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
064 }
065 super.doStop();
066 }
067
068 protected void connectIfNecessary() throws JSchException {
069 if (channel == null || !channel.isConnected()) {
070 if (session == null || !session.isConnected()) {
071 log.trace("Session isn't connected, trying to recreate and connect.");
072 session = endpoint.createSession();
073 session.connect();
074 }
075 log.trace("Channel isn't connected, trying to recreate and connect.");
076 channel = endpoint.createChannelSftp(session);
077 channel.connect();
078 log.info("Connected to " + remoteServer());
079 }
080 }
081
082 protected void disconnect() throws JSchException {
083 log.debug("Disconnecting from " + remoteServer());
084 if (session != null) {
085 session.disconnect();
086 }
087 if (channel != null) {
088 channel.disconnect();
089 }
090 }
091
092 protected void poll() throws Exception {
093 if (log.isTraceEnabled()) {
094 log.trace("Polling " + endpoint.getConfiguration());
095 }
096 connectIfNecessary();
097 // If the attempt to connect isn't successful, then the thrown
098 // exception will signify that we couldn't poll
099 try {
100 final String fileName = endpoint.getConfiguration().getFile();
101 if (endpoint.getConfiguration().isDirectory()) {
102 pollDirectory(fileName);
103 } else {
104 int index = fileName.lastIndexOf('/');
105 if (index > -1) {
106 // cd to the folder of the filename
107 channel.cd(fileName.substring(0, index));
108 }
109
110 // list the files in the fold and poll the first file
111 final Vector files = channel.ls(fileName.substring(index + 1));
112 final ChannelSftp.LsEntry file = (ChannelSftp.LsEntry) files.get(0);
113 pollFile(file);
114 }
115 lastPollTime = System.currentTimeMillis();
116 } catch (Exception e) {
117 if (isStopping() || isStopped()) {
118 // if we are stopping then ignore any exception during a poll
119 log.warn("Consumer is stopping. Ignoring caught exception: "
120 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
121 } else {
122 log.warn("Exception occured during polling: "
123 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
124 disconnect();
125 // Rethrow to signify that we didn't poll
126 throw e;
127 }
128 }
129 }
130
131 protected void pollDirectory(String dir) throws Exception {
132 if (log.isTraceEnabled()) {
133 log.trace("Polling directory: " + dir);
134 }
135 String currentDir = channel.pwd();
136
137 channel.cd(dir);
138 Vector files = channel.ls(".");
139 for (int i = 0; i < files.size(); i++) {
140 ChannelSftp.LsEntry sftpFile = (ChannelSftp.LsEntry)files.get(i);
141 if (sftpFile.getFilename().startsWith(".")) {
142 // skip
143 } else if (sftpFile.getAttrs().isDir()) {
144 if (isRecursive()) {
145 pollDirectory(getFullFileName(sftpFile));
146 }
147 } else {
148 pollFile(sftpFile);
149 }
150 }
151
152 // change back to original current dir
153 channel.cd(currentDir);
154 }
155
156 protected String getFullFileName(ChannelSftp.LsEntry sftpFile) throws IOException, SftpException {
157 return channel.pwd() + "/" + sftpFile.getFilename();
158 }
159
160 private void pollFile(ChannelSftp.LsEntry sftpFile) throws Exception {
161 if (log.isTraceEnabled()) {
162 log.trace("Polling file: " + sftpFile);
163 }
164
165 // if using last polltime for timestamp matcing (to be removed in Camel 2.0)
166 boolean timestampMatched = true;
167 if (isTimestamp()) {
168 // TODO do we need to adjust the TZ? can we?
169 long ts = sftpFile.getAttrs().getMTime() * 1000L;
170 timestampMatched = ts > lastPollTime;
171 if (log.isTraceEnabled()) {
172 log.trace("The file is to old + " + sftpFile + ". lastPollTime=" + lastPollTime + " > fileTimestamp=" + ts);
173 }
174 }
175
176 if (timestampMatched && isMatched(sftpFile)) {
177 String fullFileName = getFullFileName(sftpFile);
178
179 // is we use excluse read then acquire the exclusive read (waiting until we got it)
180 if (exclusiveReadLock) {
181 acquireExclusiveReadLock(sftpFile);
182 }
183
184 // retrieve the file
185 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
186 channel.get(sftpFile.getFilename(), byteArrayOutputStream);
187 if (log.isDebugEnabled()) {
188 log.debug("Retrieved file: " + sftpFile.getFilename() + " from: " + remoteServer());
189 }
190
191 RemoteFileExchange exchange = endpoint.createExchange(getFullFileName(sftpFile),
192 sftpFile.getFilename(), sftpFile.getAttrs().getSize(), byteArrayOutputStream);
193
194 if (isSetNames()) {
195 String ftpBasePath = endpoint.getConfiguration().getFile();
196 String relativePath = fullFileName.substring(ftpBasePath.length() + 1);
197 relativePath = relativePath.replaceFirst("/", "");
198
199 if (log.isDebugEnabled()) {
200 log.debug("Setting exchange filename to " + relativePath);
201 }
202 exchange.getIn().setHeader(FileComponent.HEADER_FILE_NAME, relativePath);
203 }
204
205 // all success so lets process it
206 getProcessor().process(exchange);
207
208 if (exchange.isFailed()) {
209 if (log.isDebugEnabled()) {
210 log.debug("Processing of exchange failed, so cannot do FTP post command such as move or delete: " + exchange);
211 }
212 } else {
213 // after processing then do post command such as delete or move
214 if (deleteFile) {
215 // delete file after consuming
216 if (log.isDebugEnabled()) {
217 log.debug("Deleteing file: " + sftpFile.getFilename() + " from: " + remoteServer());
218 }
219 deleteFile(sftpFile.getFilename());
220 } else if (isMoveFile()) {
221 String fromName = sftpFile.getFilename();
222 String toName = getMoveFileName(fromName, exchange);
223 if (log.isDebugEnabled()) {
224 log.debug("Moving file: " + fromName + " to: " + toName);
225 }
226
227 // delete any existing file
228 boolean deleted = deleteFile(toName);
229 if (!deleted) {
230 // if we could not delete any existing file then maybe the folder is missing
231 // build folder if needed
232 int lastPathIndex = toName.lastIndexOf('/');
233 if (lastPathIndex != -1) {
234 String directory = toName.substring(0, lastPathIndex);
235 if (!SftpUtils.buildDirectory(channel, directory)) {
236 log.warn("Can not build directory: " + directory + " (maybe because of denied permissions)");
237 }
238 }
239 }
240
241 // try to rename
242 try {
243 channel.rename(fromName, toName);
244 } catch (SftpException e) {
245 // ignore just log a warning
246 log.warn("Can not move file: " + fromName + " to: " + toName);
247 }
248 }
249 }
250 }
251 }
252
253 private boolean deleteFile(String filename) {
254 try {
255 channel.rm(filename);
256 return true;
257 } catch (SftpException e) {
258 // ignore just log a warning
259 log.warn("Could not delete file: " + filename + " from: " + remoteServer());
260 return false;
261 }
262 }
263
264 protected void acquireExclusiveReadLock(ChannelSftp.LsEntry sftpFile) throws SftpException {
265 if (log.isTraceEnabled()) {
266 log.trace("Waiting for exclusive read lock to file: " + sftpFile);
267 }
268
269 // the trick is to try to rename the file, if we can rename then we have exclusive read
270 // since its a remote file we can not use java.nio to get a RW access
271 String originalName = sftpFile.getFilename();
272 String newName = originalName + ".camelExclusiveReadLock";
273 boolean exclusive = false;
274 while (!exclusive) {
275 try {
276 channel.rename(originalName, newName);
277 exclusive = true;
278 } catch (SftpException e) {
279 // ignore we can not rename it
280 }
281
282 if (exclusive) {
283 if (log.isDebugEnabled()) {
284 log.debug("Acquired exclusive read lock to file: " + originalName);
285 }
286 // rename it back so we can read it
287 channel.rename(newName, originalName);
288 } else {
289 log.trace("Exclusive read lock not granted. Sleeping for 1000 millis");
290 try {
291 Thread.sleep(1000);
292 } catch (InterruptedException e) {
293 // ignore
294 }
295 }
296 }
297 }
298
299 protected String getFileName(Object file) {
300 ChannelSftp.LsEntry sftpFile = (ChannelSftp.LsEntry) file;
301 return sftpFile.getFilename();
302 }
303
304 }