001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.processor;
018
019 import java.util.ArrayList;
020 import java.util.Collection;
021 import java.util.List;
022 import java.util.concurrent.Callable;
023 import java.util.concurrent.CompletionService;
024 import java.util.concurrent.ExecutionException;
025 import java.util.concurrent.ExecutorCompletionService;
026 import java.util.concurrent.ExecutorService;
027 import java.util.concurrent.Future;
028 import java.util.concurrent.TimeUnit;
029 import java.util.concurrent.atomic.AtomicBoolean;
030 import java.util.concurrent.atomic.AtomicInteger;
031
032 import org.apache.camel.CamelExchangeException;
033 import org.apache.camel.Exchange;
034 import org.apache.camel.Navigate;
035 import org.apache.camel.Processor;
036 import org.apache.camel.Producer;
037 import org.apache.camel.builder.ErrorHandlerBuilder;
038 import org.apache.camel.impl.ServiceSupport;
039 import org.apache.camel.processor.aggregate.AggregationStrategy;
040 import org.apache.camel.spi.RouteContext;
041 import org.apache.camel.spi.TracedRouteNodes;
042 import org.apache.camel.util.ExchangeHelper;
043 import org.apache.camel.util.ServiceHelper;
044 import org.apache.camel.util.concurrent.AtomicExchange;
045 import org.apache.camel.util.concurrent.ExecutorServiceHelper;
046 import org.apache.camel.util.concurrent.SubmitOrderedCompletionService;
047 import org.apache.commons.logging.Log;
048 import org.apache.commons.logging.LogFactory;
049
050 import static org.apache.camel.util.ObjectHelper.notNull;
051
052 /**
053 * Implements the Multicast pattern to send a message exchange to a number of
054 * endpoints, each endpoint receiving a copy of the message exchange.
055 *
056 * @see Pipeline
057 * @version $Revision: 894829 $
058 */
059 public class MulticastProcessor extends ServiceSupport implements Processor, Navigate<Processor>, Traceable {
060
061 private static final int DEFAULT_THREADPOOL_SIZE = 10;
062 private static final transient Log LOG = LogFactory.getLog(MulticastProcessor.class);
063
064 /**
065 * Class that represent each step in the multicast route to do
066 */
067 static class ProcessorExchangePair {
068 private final Processor processor;
069 private final Exchange exchange;
070
071 public ProcessorExchangePair(Processor processor, Exchange exchange) {
072 this.processor = processor;
073 this.exchange = exchange;
074 }
075
076 public Processor getProcessor() {
077 return processor;
078 }
079
080 public Exchange getExchange() {
081 return exchange;
082 }
083 }
084
085 private Collection<Processor> processors;
086 private final AggregationStrategy aggregationStrategy;
087 private final boolean isParallelProcessing;
088 private final boolean streaming;
089 private final boolean stopOnException;
090 private ExecutorService executorService;
091
092 public MulticastProcessor(Collection<Processor> processors) {
093 this(processors, null);
094 }
095
096 public MulticastProcessor(Collection<Processor> processors, AggregationStrategy aggregationStrategy) {
097 this(processors, aggregationStrategy, false, null, false, false);
098 }
099
100 public MulticastProcessor(Collection<Processor> processors, AggregationStrategy aggregationStrategy,
101 boolean parallelProcessing, ExecutorService executorService, boolean streaming, boolean stopOnException) {
102 notNull(processors, "processors");
103 this.processors = processors;
104 this.aggregationStrategy = aggregationStrategy;
105 this.isParallelProcessing = parallelProcessing;
106 this.executorService = executorService;
107 this.streaming = streaming;
108 this.stopOnException = stopOnException;
109
110 if (isParallelProcessing()) {
111 if (this.executorService == null) {
112 // setup default executor as parallel processing requires an executor
113 this.executorService = ExecutorServiceHelper.newScheduledThreadPool(DEFAULT_THREADPOOL_SIZE, "Multicast", true);
114 }
115 }
116 }
117
118 @Override
119 public String toString() {
120 return "Multicast[" + getProcessors() + "]";
121 }
122
123 public String getTraceLabel() {
124 return "multicast";
125 }
126
127 public void process(Exchange exchange) throws Exception {
128 final AtomicExchange result = new AtomicExchange();
129 final Iterable<ProcessorExchangePair> pairs = createProcessorExchangePairs(exchange);
130
131 // multicast uses fine grained error handling on the output processors
132 // so use try .. catch to cater for this
133 try {
134 if (isParallelProcessing()) {
135 doProcessParallel(result, pairs, isStreaming());
136 } else {
137 doProcessSequential(result, pairs);
138 }
139
140 if (result.get() != null) {
141 ExchangeHelper.copyResults(exchange, result.get());
142 }
143 } catch (Exception e) {
144 // multicast uses error handling on its output processors and they have tried to redeliver
145 // so we shall signal back to the other error handlers that we are exhausted and they should not
146 // also try to redeliver as we will then do that twice
147 exchange.setProperty(Exchange.REDELIVERY_EXHAUSTED, Boolean.TRUE);
148 exchange.setException(e);
149 }
150 }
151
152 protected void doProcessParallel(final AtomicExchange result, Iterable<ProcessorExchangePair> pairs, boolean streaming) throws InterruptedException, ExecutionException {
153 final CompletionService<Exchange> completion;
154 final AtomicBoolean running = new AtomicBoolean(true);
155
156 if (streaming) {
157 // execute tasks in parallel+streaming and aggregate in the order they are finished (out of order sequence)
158 completion = new ExecutorCompletionService<Exchange>(executorService);
159 } else {
160 // execute tasks in parallel and aggregate in the order the tasks are submitted (in order sequence)
161 completion = new SubmitOrderedCompletionService<Exchange>(executorService);
162 }
163
164 final AtomicInteger total = new AtomicInteger(0);
165
166 for (ProcessorExchangePair pair : pairs) {
167 final Processor producer = pair.getProcessor();
168 final Exchange subExchange = pair.getExchange();
169 updateNewExchange(subExchange, total.intValue(), pairs);
170
171 completion.submit(new Callable<Exchange>() {
172 public Exchange call() throws Exception {
173 if (!running.get()) {
174 // do not start processing the task if we are not running
175 return subExchange;
176 }
177
178 doProcess(producer, subExchange);
179
180 // should we stop in case of an exception occurred during processing?
181 if (stopOnException && subExchange.getException() != null) {
182 // signal to stop running
183 running.set(false);
184 throw new CamelExchangeException("Parallel processing failed for number " + total.intValue(), subExchange, subExchange.getException());
185 }
186
187 if (LOG.isTraceEnabled()) {
188 LOG.trace("Parallel processing complete for exchange: " + subExchange);
189 }
190 return subExchange;
191 }
192 });
193
194 total.incrementAndGet();
195 }
196
197 for (int i = 0; i < total.intValue(); i++) {
198 Future<Exchange> future = completion.take();
199 Exchange subExchange = future.get();
200 if (aggregationStrategy != null) {
201 doAggregate(result, subExchange);
202 }
203 }
204
205 if (LOG.isDebugEnabled()) {
206 LOG.debug("Done parallel processing " + total + " exchanges");
207 }
208 }
209
210 protected void doProcessSequential(AtomicExchange result, Iterable<ProcessorExchangePair> pairs) throws Exception {
211 int total = 0;
212
213 for (ProcessorExchangePair pair : pairs) {
214 Processor producer = pair.getProcessor();
215 Exchange subExchange = pair.getExchange();
216 updateNewExchange(subExchange, total, pairs);
217
218 doProcess(producer, subExchange);
219
220 // should we stop in case of an exception occurred during processing?
221 if (stopOnException && subExchange.getException() != null) {
222 throw new CamelExchangeException("Sequential processing failed for number " + total, subExchange, subExchange.getException());
223 }
224
225 if (LOG.isTraceEnabled()) {
226 LOG.trace("Sequential processing complete for number " + total + " exchange: " + subExchange);
227 }
228
229 if (aggregationStrategy != null) {
230 doAggregate(result, subExchange);
231 }
232 total++;
233 }
234
235 if (LOG.isDebugEnabled()) {
236 LOG.debug("Done sequential processing " + total + " exchanges");
237 }
238 }
239
240 private void doProcess(Processor producer, Exchange exchange) {
241 TracedRouteNodes traced = exchange.getUnitOfWork() != null ? exchange.getUnitOfWork().getTracedRouteNodes() : null;
242
243 try {
244 // prepare tracing starting from a new block
245 if (traced != null) {
246 traced.pushBlock();
247 }
248
249 // set property which endpoint we send to
250 setToEndpoint(exchange, producer);
251
252 if (exchange.getUnitOfWork() != null && exchange.getUnitOfWork().getRouteContext() != null) {
253 // wrap the producer in error handler so we have fine grained error handling on
254 // the output side instead of the input side
255 // this is needed to support redelivery on that output alone and not doing redelivery
256 // for the entire multicast block again which will start from scratch again
257 RouteContext routeContext = exchange.getUnitOfWork().getRouteContext();
258 ErrorHandlerBuilder builder = routeContext.getRoute().getErrorHandlerBuilder();
259
260 // create error handler (create error handler directly to keep it light weight,
261 // instead of using ProcessorDefinition.wrapInErrorHandler)
262 producer = builder.createErrorHandler(routeContext, producer);
263 }
264
265 // let the producer process it
266 producer.process(exchange);
267 } catch (Exception e) {
268 exchange.setException(e);
269 } finally {
270 // pop the block so by next round we have the same staring point and thus the tracing looks accurate
271 if (traced != null) {
272 traced.popBlock();
273 }
274 }
275 }
276
277 /**
278 * Aggregate the {@link Exchange} with the current result
279 *
280 * @param result the current result
281 * @param exchange the exchange to be added to the result
282 */
283 protected synchronized void doAggregate(AtomicExchange result, Exchange exchange) {
284 if (aggregationStrategy != null) {
285 // prepare the exchanges for aggregation
286 Exchange oldExchange = result.get();
287 ExchangeHelper.prepareAggregation(oldExchange, exchange);
288 result.set(aggregationStrategy.aggregate(oldExchange, exchange));
289 }
290 }
291
292 protected void updateNewExchange(Exchange exchange, int index, Iterable<ProcessorExchangePair> allPairs) {
293 exchange.setProperty(Exchange.MULTICAST_INDEX, index);
294 }
295
296 protected Iterable<ProcessorExchangePair> createProcessorExchangePairs(Exchange exchange) {
297 List<ProcessorExchangePair> result = new ArrayList<ProcessorExchangePair>(processors.size());
298
299 for (Processor processor : processors) {
300 Exchange copy = exchange.copy();
301 result.add(new ProcessorExchangePair(processor, copy));
302 }
303 return result;
304 }
305
306 protected void doStop() throws Exception {
307 if (executorService != null) {
308 executorService.shutdown();
309 executorService.awaitTermination(0, TimeUnit.SECONDS);
310 executorService = null;
311 }
312 ServiceHelper.stopServices(processors);
313 }
314
315 protected void doStart() throws Exception {
316 ServiceHelper.startServices(processors);
317 }
318
319 private static void setToEndpoint(Exchange exchange, Processor processor) {
320 if (processor instanceof Producer) {
321 Producer producer = (Producer) processor;
322 exchange.setProperty(Exchange.TO_ENDPOINT, producer.getEndpoint().getEndpointUri());
323 }
324 }
325
326 /**
327 * Is the multicast processor working in streaming mode?
328 *
329 * In streaming mode:
330 * <ul>
331 * <li>we use {@link Iterable} to ensure we can send messages as soon as the data becomes available</li>
332 * <li>for parallel processing, we start aggregating responses as they get send back to the processor;
333 * this means the {@link org.apache.camel.processor.aggregate.AggregationStrategy} has to take care of handling out-of-order arrival of exchanges</li>
334 * </ul>
335 */
336 public boolean isStreaming() {
337 return streaming;
338 }
339
340 /**
341 * Should the multicast processor stop processing further exchanges in case of an exception occurred?
342 */
343 public boolean isStopOnException() {
344 return stopOnException;
345 }
346
347 /**
348 * Returns the producers to multicast to
349 */
350 public Collection<Processor> getProcessors() {
351 return processors;
352 }
353
354 public AggregationStrategy getAggregationStrategy() {
355 return aggregationStrategy;
356 }
357
358 public boolean isParallelProcessing() {
359 return isParallelProcessing;
360 }
361
362 public ExecutorService getExecutorService() {
363 return executorService;
364 }
365
366 public void setExecutorService(ExecutorService executorService) {
367 this.executorService = executorService;
368 }
369
370 public List<Processor> next() {
371 if (!hasNext()) {
372 return null;
373 }
374 return new ArrayList<Processor>(processors);
375 }
376
377 public boolean hasNext() {
378 return processors != null && !processors.isEmpty();
379 }
380 }