001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.processor;
018
019 import java.util.ArrayList;
020 import java.util.Collection;
021 import java.util.Iterator;
022 import java.util.LinkedList;
023 import java.util.List;
024 import java.util.Queue;
025 import java.util.concurrent.TimeUnit;
026 import java.util.concurrent.locks.Condition;
027 import java.util.concurrent.locks.Lock;
028 import java.util.concurrent.locks.ReentrantLock;
029
030 import org.apache.camel.CamelException;
031 import org.apache.camel.Exchange;
032 import org.apache.camel.Navigate;
033 import org.apache.camel.Processor;
034 import org.apache.camel.impl.LoggingExceptionHandler;
035 import org.apache.camel.impl.ServiceSupport;
036 import org.apache.camel.spi.ExceptionHandler;
037 import org.apache.camel.util.ObjectHelper;
038 import org.apache.camel.util.ServiceHelper;
039 import org.apache.camel.util.concurrent.ExecutorServiceHelper;
040 import org.apache.commons.logging.Log;
041 import org.apache.commons.logging.LogFactory;
042
043 /**
044 * A base class for any kind of {@link Processor} which implements some kind of batch processing.
045 *
046 * @version $Revision: 906381 $
047 */
048 public class BatchProcessor extends ServiceSupport implements Processor, Navigate<Processor> {
049
050 public static final long DEFAULT_BATCH_TIMEOUT = 1000L;
051 public static final int DEFAULT_BATCH_SIZE = 100;
052
053 private static final Log LOG = LogFactory.getLog(BatchProcessor.class);
054
055 private long batchTimeout = DEFAULT_BATCH_TIMEOUT;
056 private int batchSize = DEFAULT_BATCH_SIZE;
057 private int outBatchSize;
058 private boolean groupExchanges;
059 private boolean batchConsumer;
060
061 private final Processor processor;
062 private final Collection<Exchange> collection;
063 private ExceptionHandler exceptionHandler;
064
065 private final BatchSender sender;
066
067 public BatchProcessor(Processor processor, Collection<Exchange> collection) {
068 ObjectHelper.notNull(processor, "processor");
069 ObjectHelper.notNull(collection, "collection");
070
071 // wrap processor in UnitOfWork so what we send out of the batch runs in a UoW
072 this.processor = new UnitOfWorkProcessor(processor);
073 this.collection = collection;
074 this.sender = new BatchSender();
075 }
076
077 @Override
078 public String toString() {
079 return "BatchProcessor[to: " + processor + "]";
080 }
081
082 // Properties
083 // -------------------------------------------------------------------------
084 public ExceptionHandler getExceptionHandler() {
085 if (exceptionHandler == null) {
086 exceptionHandler = new LoggingExceptionHandler(getClass());
087 }
088 return exceptionHandler;
089 }
090
091 public void setExceptionHandler(ExceptionHandler exceptionHandler) {
092 this.exceptionHandler = exceptionHandler;
093 }
094
095 public int getBatchSize() {
096 return batchSize;
097 }
098
099 /**
100 * Sets the <b>in</b> batch size. This is the number of incoming exchanges that this batch processor will
101 * process before its completed. The default value is {@link #DEFAULT_BATCH_SIZE}.
102 *
103 * @param batchSize the size
104 */
105 public void setBatchSize(int batchSize) {
106 // setting batch size to 0 or negative is like disabling it, so we set it as the max value
107 // as the code logic is dependent on a batch size having 1..n value
108 if (batchSize <= 0) {
109 if (LOG.isDebugEnabled()) {
110 LOG.debug("Disabling batch size, will only be triggered by timeout");
111 }
112 this.batchSize = Integer.MAX_VALUE;
113 } else {
114 this.batchSize = batchSize;
115 }
116 }
117
118 public int getOutBatchSize() {
119 return outBatchSize;
120 }
121
122 /**
123 * Sets the <b>out</b> batch size. If the batch processor holds more exchanges than this out size then the
124 * completion is triggered. Can for instance be used to ensure that this batch is completed when a certain
125 * number of exchanges has been collected. By default this feature is <b>not</b> enabled.
126 *
127 * @param outBatchSize the size
128 */
129 public void setOutBatchSize(int outBatchSize) {
130 this.outBatchSize = outBatchSize;
131 }
132
133 public long getBatchTimeout() {
134 return batchTimeout;
135 }
136
137 public void setBatchTimeout(long batchTimeout) {
138 this.batchTimeout = batchTimeout;
139 }
140
141 public boolean isGroupExchanges() {
142 return groupExchanges;
143 }
144
145 public void setGroupExchanges(boolean groupExchanges) {
146 this.groupExchanges = groupExchanges;
147 }
148
149 public boolean isBatchConsumer() {
150 return batchConsumer;
151 }
152
153 public void setBatchConsumer(boolean batchConsumer) {
154 this.batchConsumer = batchConsumer;
155 }
156
157 public Processor getProcessor() {
158 return processor;
159 }
160
161 public List<Processor> next() {
162 if (!hasNext()) {
163 return null;
164 }
165 List<Processor> answer = new ArrayList<Processor>(1);
166 answer.add(processor);
167 return answer;
168 }
169
170 public boolean hasNext() {
171 return processor != null;
172 }
173
174 /**
175 * A strategy method to decide if the "in" batch is completed. That is, whether the resulting exchanges in
176 * the in queue should be drained to the "out" collection.
177 */
178 private boolean isInBatchCompleted(int num) {
179 return num >= batchSize;
180 }
181
182 /**
183 * A strategy method to decide if the "out" batch is completed. That is, whether the resulting exchange in
184 * the out collection should be sent.
185 */
186 private boolean isOutBatchCompleted() {
187 if (outBatchSize == 0) {
188 // out batch is disabled, so go ahead and send.
189 return true;
190 }
191 return collection.size() > 0 && collection.size() >= outBatchSize;
192 }
193
194 /**
195 * Strategy Method to process an exchange in the batch. This method allows derived classes to perform
196 * custom processing before or after an individual exchange is processed
197 */
198 protected void processExchange(Exchange exchange) throws Exception {
199 processor.process(exchange);
200 if (exchange.getException() != null) {
201 getExceptionHandler().handleException("Error processing Exchange: " + exchange, exchange.getException());
202 }
203 }
204
205 protected void doStart() throws Exception {
206 ServiceHelper.startServices(processor);
207 sender.start();
208 }
209
210 protected void doStop() throws Exception {
211 sender.cancel();
212 ServiceHelper.stopServices(sender);
213 ServiceHelper.stopServices(processor);
214 collection.clear();
215 }
216
217 /**
218 * Enqueues an exchange for later batch processing.
219 */
220 public void process(Exchange exchange) throws Exception {
221
222 // if batch consumer is enabled then we need to adjust the batch size
223 // with the size from the batch consumer
224 if (isBatchConsumer()) {
225 int size = exchange.getProperty(Exchange.BATCH_SIZE, Integer.class);
226 if (batchSize != size) {
227 batchSize = size;
228 if (LOG.isTraceEnabled()) {
229 LOG.trace("Using batch consumer completion, so setting batch size to: " + batchSize);
230 }
231 }
232 }
233
234 sender.enqueueExchange(exchange);
235 }
236
237 /**
238 * Sender thread for queued-up exchanges.
239 */
240 private class BatchSender extends Thread {
241
242 private Queue<Exchange> queue;
243 private Lock queueLock = new ReentrantLock();
244 private boolean exchangeEnqueued;
245 private Condition exchangeEnqueuedCondition = queueLock.newCondition();
246
247 public BatchSender() {
248 super(ExecutorServiceHelper.getThreadName("Batch Sender"));
249 this.queue = new LinkedList<Exchange>();
250 }
251
252 @Override
253 public void run() {
254 // Wait until one of either:
255 // * an exchange being queued;
256 // * the batch timeout expiring; or
257 // * the thread being cancelled.
258 //
259 // If an exchange is queued then we need to determine whether the
260 // batch is complete. If it is complete then we send out the batched
261 // exchanges. Otherwise we move back into our wait state.
262 //
263 // If the batch times out then we send out the batched exchanges
264 // collected so far.
265 //
266 // If we receive an interrupt then all blocking operations are
267 // interrupted and our thread terminates.
268 //
269 // The goal of the following algorithm in terms of synchronisation
270 // is to provide fine grained locking i.e. retaining the lock only
271 // when required. Special consideration is given to releasing the
272 // lock when calling an overloaded method i.e. sendExchanges.
273 // Unlocking is important as the process of sending out the exchanges
274 // would otherwise block new exchanges from being queued.
275
276 queueLock.lock();
277 try {
278 do {
279 try {
280 if (!exchangeEnqueued) {
281 exchangeEnqueuedCondition.await(batchTimeout, TimeUnit.MILLISECONDS);
282 }
283
284 if (!exchangeEnqueued) {
285 drainQueueTo(collection, batchSize);
286 } else {
287 exchangeEnqueued = false;
288 while (isInBatchCompleted(queue.size())) {
289 drainQueueTo(collection, batchSize);
290 }
291
292 if (!isOutBatchCompleted()) {
293 continue;
294 }
295 }
296
297 queueLock.unlock();
298 try {
299 try {
300 sendExchanges();
301 } catch (Throwable t) {
302 // a fail safe to handle all exceptions being thrown
303 getExceptionHandler().handleException(new CamelException(t));
304 }
305 } finally {
306 queueLock.lock();
307 }
308
309 } catch (InterruptedException e) {
310 break;
311 }
312
313 } while (isRunAllowed());
314
315 } finally {
316 queueLock.unlock();
317 }
318 }
319
320 /**
321 * This method should be called with queueLock held
322 */
323 private void drainQueueTo(Collection<Exchange> collection, int batchSize) {
324 for (int i = 0; i < batchSize; ++i) {
325 Exchange e = queue.poll();
326 if (e != null) {
327 try {
328 collection.add(e);
329 } catch (Exception t) {
330 e.setException(t);
331 } catch (Throwable t) {
332 getExceptionHandler().handleException(t);
333 }
334 } else {
335 break;
336 }
337 }
338 }
339
340 public void cancel() {
341 interrupt();
342 }
343
344 public void enqueueExchange(Exchange exchange) {
345 queueLock.lock();
346 try {
347 queue.add(exchange);
348 exchangeEnqueued = true;
349 exchangeEnqueuedCondition.signal();
350 } finally {
351 queueLock.unlock();
352 }
353 }
354
355 @SuppressWarnings("unchecked")
356 private void sendExchanges() throws Exception {
357 Iterator<Exchange> iter = collection.iterator();
358 while (iter.hasNext()) {
359 Exchange exchange = iter.next();
360 iter.remove();
361 try {
362 processExchange(exchange);
363 } catch (Throwable t) {
364 // must catch throwable to avoid growing memory
365 getExceptionHandler().handleException("Error processing Exchange: " + exchange, t);
366 }
367 }
368 }
369 }
370
371 }