Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.store.kahadb;
018
019import java.io.DataInputStream;
020import java.io.IOException;
021import java.io.InterruptedIOException;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Map.Entry;
030import java.util.Set;
031import java.util.concurrent.BlockingQueue;
032import java.util.concurrent.Callable;
033import java.util.concurrent.ExecutorService;
034import java.util.concurrent.FutureTask;
035import java.util.concurrent.LinkedBlockingQueue;
036import java.util.concurrent.Semaphore;
037import java.util.concurrent.ThreadFactory;
038import java.util.concurrent.ThreadPoolExecutor;
039import java.util.concurrent.TimeUnit;
040import java.util.concurrent.TimeoutException;
041import java.util.concurrent.atomic.AtomicBoolean;
042import java.util.concurrent.atomic.AtomicInteger;
043
044import org.apache.activemq.broker.ConnectionContext;
045import org.apache.activemq.broker.region.BaseDestination;
046import org.apache.activemq.broker.scheduler.JobSchedulerStore;
047import org.apache.activemq.command.ActiveMQDestination;
048import org.apache.activemq.command.ActiveMQQueue;
049import org.apache.activemq.command.ActiveMQTempQueue;
050import org.apache.activemq.command.ActiveMQTempTopic;
051import org.apache.activemq.command.ActiveMQTopic;
052import org.apache.activemq.command.Message;
053import org.apache.activemq.command.MessageAck;
054import org.apache.activemq.command.MessageId;
055import org.apache.activemq.command.ProducerId;
056import org.apache.activemq.command.SubscriptionInfo;
057import org.apache.activemq.command.TransactionId;
058import org.apache.activemq.openwire.OpenWireFormat;
059import org.apache.activemq.protobuf.Buffer;
060import org.apache.activemq.store.AbstractMessageStore;
061import org.apache.activemq.store.IndexListener;
062import org.apache.activemq.store.ListenableFuture;
063import org.apache.activemq.store.MessageRecoveryListener;
064import org.apache.activemq.store.MessageStore;
065import org.apache.activemq.store.PersistenceAdapter;
066import org.apache.activemq.store.TopicMessageStore;
067import org.apache.activemq.store.TransactionIdTransformer;
068import org.apache.activemq.store.TransactionStore;
069import org.apache.activemq.store.kahadb.data.KahaAddMessageCommand;
070import org.apache.activemq.store.kahadb.data.KahaDestination;
071import org.apache.activemq.store.kahadb.data.KahaDestination.DestinationType;
072import org.apache.activemq.store.kahadb.data.KahaLocation;
073import org.apache.activemq.store.kahadb.data.KahaRemoveDestinationCommand;
074import org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand;
075import org.apache.activemq.store.kahadb.data.KahaSubscriptionCommand;
076import org.apache.activemq.store.kahadb.data.KahaUpdateMessageCommand;
077import org.apache.activemq.store.kahadb.disk.journal.Location;
078import org.apache.activemq.store.kahadb.disk.page.Transaction;
079import org.apache.activemq.store.kahadb.scheduler.JobSchedulerStoreImpl;
080import org.apache.activemq.usage.MemoryUsage;
081import org.apache.activemq.usage.SystemUsage;
082import org.apache.activemq.util.ServiceStopper;
083import org.apache.activemq.util.ThreadPoolUtils;
084import org.apache.activemq.wireformat.WireFormat;
085import org.slf4j.Logger;
086import org.slf4j.LoggerFactory;
087
088public class KahaDBStore extends MessageDatabase implements PersistenceAdapter {
089    static final Logger LOG = LoggerFactory.getLogger(KahaDBStore.class);
090    private static final int MAX_ASYNC_JOBS = BaseDestination.MAX_AUDIT_DEPTH;
091
092    public static final String PROPERTY_CANCELED_TASK_MOD_METRIC = "org.apache.activemq.store.kahadb.CANCELED_TASK_MOD_METRIC";
093    public static final int cancelledTaskModMetric = Integer.parseInt(System.getProperty(
094            PROPERTY_CANCELED_TASK_MOD_METRIC, "0"), 10);
095    public static final String PROPERTY_ASYNC_EXECUTOR_MAX_THREADS = "org.apache.activemq.store.kahadb.ASYNC_EXECUTOR_MAX_THREADS";
096    private static final int asyncExecutorMaxThreads = Integer.parseInt(System.getProperty(
097            PROPERTY_ASYNC_EXECUTOR_MAX_THREADS, "1"), 10);;
098
099    protected ExecutorService queueExecutor;
100    protected ExecutorService topicExecutor;
101    protected final List<Map<AsyncJobKey, StoreTask>> asyncQueueMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
102    protected final List<Map<AsyncJobKey, StoreTask>> asyncTopicMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
103    final WireFormat wireFormat = new OpenWireFormat();
104    private SystemUsage usageManager;
105    private LinkedBlockingQueue<Runnable> asyncQueueJobQueue;
106    private LinkedBlockingQueue<Runnable> asyncTopicJobQueue;
107    Semaphore globalQueueSemaphore;
108    Semaphore globalTopicSemaphore;
109    private boolean concurrentStoreAndDispatchQueues = true;
110    // when true, message order may be compromised when cache is exhausted if store is out
111    // or order w.r.t cache
112    private boolean concurrentStoreAndDispatchTopics = false;
113    private final boolean concurrentStoreAndDispatchTransactions = false;
114    private int maxAsyncJobs = MAX_ASYNC_JOBS;
115    private final KahaDBTransactionStore transactionStore;
116    private TransactionIdTransformer transactionIdTransformer;
117
118    public KahaDBStore() {
119        this.transactionStore = new KahaDBTransactionStore(this);
120        this.transactionIdTransformer = new TransactionIdTransformer() {
121            @Override
122            public TransactionId transform(TransactionId txid) {
123                return txid;
124            }
125        };
126    }
127
128    @Override
129    public String toString() {
130        return "KahaDB:[" + directory.getAbsolutePath() + "]";
131    }
132
133    @Override
134    public void setBrokerName(String brokerName) {
135    }
136
137    @Override
138    public void setUsageManager(SystemUsage usageManager) {
139        this.usageManager = usageManager;
140    }
141
142    public SystemUsage getUsageManager() {
143        return this.usageManager;
144    }
145
146    /**
147     * @return the concurrentStoreAndDispatch
148     */
149    public boolean isConcurrentStoreAndDispatchQueues() {
150        return this.concurrentStoreAndDispatchQueues;
151    }
152
153    /**
154     * @param concurrentStoreAndDispatch
155     *            the concurrentStoreAndDispatch to set
156     */
157    public void setConcurrentStoreAndDispatchQueues(boolean concurrentStoreAndDispatch) {
158        this.concurrentStoreAndDispatchQueues = concurrentStoreAndDispatch;
159    }
160
161    /**
162     * @return the concurrentStoreAndDispatch
163     */
164    public boolean isConcurrentStoreAndDispatchTopics() {
165        return this.concurrentStoreAndDispatchTopics;
166    }
167
168    /**
169     * @param concurrentStoreAndDispatch
170     *            the concurrentStoreAndDispatch to set
171     */
172    public void setConcurrentStoreAndDispatchTopics(boolean concurrentStoreAndDispatch) {
173        this.concurrentStoreAndDispatchTopics = concurrentStoreAndDispatch;
174    }
175
176    public boolean isConcurrentStoreAndDispatchTransactions() {
177        return this.concurrentStoreAndDispatchTransactions;
178    }
179
180    /**
181     * @return the maxAsyncJobs
182     */
183    public int getMaxAsyncJobs() {
184        return this.maxAsyncJobs;
185    }
186
187    /**
188     * @param maxAsyncJobs
189     *            the maxAsyncJobs to set
190     */
191    public void setMaxAsyncJobs(int maxAsyncJobs) {
192        this.maxAsyncJobs = maxAsyncJobs;
193    }
194
195    @Override
196    public void doStart() throws Exception {
197        if (brokerService != null) {
198            metadata.openwireVersion = brokerService.getStoreOpenWireVersion();
199            wireFormat.setVersion(metadata.openwireVersion);
200
201            if (LOG.isDebugEnabled()) {
202                LOG.debug("Store OpenWire version configured as: {}", metadata.openwireVersion);
203            }
204
205        }
206        super.doStart();
207
208        if (brokerService != null) {
209            // In case the recovered store used a different OpenWire version log a warning
210            // to assist in determining why journal reads fail.
211            if (metadata.openwireVersion != brokerService.getStoreOpenWireVersion()) {
212                LOG.warn("Recovered Store uses a different OpenWire version[{}] " +
213                         "than the version configured[{}].",
214                         metadata.openwireVersion, brokerService.getStoreOpenWireVersion());
215            }
216        }
217
218        this.globalQueueSemaphore = new Semaphore(getMaxAsyncJobs());
219        this.globalTopicSemaphore = new Semaphore(getMaxAsyncJobs());
220        this.asyncQueueJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
221        this.asyncTopicJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
222        this.queueExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
223            asyncQueueJobQueue, new ThreadFactory() {
224                @Override
225                public Thread newThread(Runnable runnable) {
226                    Thread thread = new Thread(runnable, "ConcurrentQueueStoreAndDispatch");
227                    thread.setDaemon(true);
228                    return thread;
229                }
230            });
231        this.topicExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
232            asyncTopicJobQueue, new ThreadFactory() {
233                @Override
234                public Thread newThread(Runnable runnable) {
235                    Thread thread = new Thread(runnable, "ConcurrentTopicStoreAndDispatch");
236                    thread.setDaemon(true);
237                    return thread;
238                }
239            });
240    }
241
242    @Override
243    public void doStop(ServiceStopper stopper) throws Exception {
244        // drain down async jobs
245        LOG.info("Stopping async queue tasks");
246        if (this.globalQueueSemaphore != null) {
247            this.globalQueueSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
248        }
249        synchronized (this.asyncQueueMaps) {
250            for (Map<AsyncJobKey, StoreTask> m : asyncQueueMaps) {
251                synchronized (m) {
252                    for (StoreTask task : m.values()) {
253                        task.cancel();
254                    }
255                }
256            }
257            this.asyncQueueMaps.clear();
258        }
259        LOG.info("Stopping async topic tasks");
260        if (this.globalTopicSemaphore != null) {
261            this.globalTopicSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
262        }
263        synchronized (this.asyncTopicMaps) {
264            for (Map<AsyncJobKey, StoreTask> m : asyncTopicMaps) {
265                synchronized (m) {
266                    for (StoreTask task : m.values()) {
267                        task.cancel();
268                    }
269                }
270            }
271            this.asyncTopicMaps.clear();
272        }
273        if (this.globalQueueSemaphore != null) {
274            this.globalQueueSemaphore.drainPermits();
275        }
276        if (this.globalTopicSemaphore != null) {
277            this.globalTopicSemaphore.drainPermits();
278        }
279        if (this.queueExecutor != null) {
280            ThreadPoolUtils.shutdownNow(queueExecutor);
281            queueExecutor = null;
282        }
283        if (this.topicExecutor != null) {
284            ThreadPoolUtils.shutdownNow(topicExecutor);
285            topicExecutor = null;
286        }
287        LOG.info("Stopped KahaDB");
288        super.doStop(stopper);
289    }
290
291    private Location findMessageLocation(final String key, final KahaDestination destination) throws IOException {
292        return pageFile.tx().execute(new Transaction.CallableClosure<Location, IOException>() {
293            @Override
294            public Location execute(Transaction tx) throws IOException {
295                StoredDestination sd = getStoredDestination(destination, tx);
296                Long sequence = sd.messageIdIndex.get(tx, key);
297                if (sequence == null) {
298                    return null;
299                }
300                return sd.orderIndex.get(tx, sequence).location;
301            }
302        });
303    }
304
305    protected StoreQueueTask removeQueueTask(KahaDBMessageStore store, MessageId id) {
306        StoreQueueTask task = null;
307        synchronized (store.asyncTaskMap) {
308            task = (StoreQueueTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
309        }
310        return task;
311    }
312
313    protected void addQueueTask(KahaDBMessageStore store, StoreQueueTask task) throws IOException {
314        synchronized (store.asyncTaskMap) {
315            store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
316        }
317        this.queueExecutor.execute(task);
318    }
319
320    protected StoreTopicTask removeTopicTask(KahaDBTopicMessageStore store, MessageId id) {
321        StoreTopicTask task = null;
322        synchronized (store.asyncTaskMap) {
323            task = (StoreTopicTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
324        }
325        return task;
326    }
327
328    protected void addTopicTask(KahaDBTopicMessageStore store, StoreTopicTask task) throws IOException {
329        synchronized (store.asyncTaskMap) {
330            store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
331        }
332        this.topicExecutor.execute(task);
333    }
334
335    @Override
336    public TransactionStore createTransactionStore() throws IOException {
337        return this.transactionStore;
338    }
339
340    public boolean getForceRecoverIndex() {
341        return this.forceRecoverIndex;
342    }
343
344    public void setForceRecoverIndex(boolean forceRecoverIndex) {
345        this.forceRecoverIndex = forceRecoverIndex;
346    }
347
348    public class KahaDBMessageStore extends AbstractMessageStore {
349        protected final Map<AsyncJobKey, StoreTask> asyncTaskMap = new HashMap<AsyncJobKey, StoreTask>();
350        protected KahaDestination dest;
351        private final int maxAsyncJobs;
352        private final Semaphore localDestinationSemaphore;
353
354        double doneTasks, canceledTasks = 0;
355
356        public KahaDBMessageStore(ActiveMQDestination destination) {
357            super(destination);
358            this.dest = convert(destination);
359            this.maxAsyncJobs = getMaxAsyncJobs();
360            this.localDestinationSemaphore = new Semaphore(this.maxAsyncJobs);
361        }
362
363        @Override
364        public ActiveMQDestination getDestination() {
365            return destination;
366        }
367
368        @Override
369        public ListenableFuture<Object> asyncAddQueueMessage(final ConnectionContext context, final Message message)
370                throws IOException {
371            if (isConcurrentStoreAndDispatchQueues()) {
372                StoreQueueTask result = new StoreQueueTask(this, context, message);
373                ListenableFuture<Object> future = result.getFuture();
374                message.getMessageId().setFutureOrSequenceLong(future);
375                message.setRecievedByDFBridge(true); // flag message as concurrentStoreAndDispatch
376                result.aquireLocks();
377                addQueueTask(this, result);
378                if (indexListener != null) {
379                    // allow concurrent dispatch by setting entry locator,
380                    indexListener.onAdd(new IndexListener.MessageContext(context, message, null));
381                }
382                return future;
383            } else {
384                return super.asyncAddQueueMessage(context, message);
385            }
386        }
387
388        @Override
389        public void removeAsyncMessage(ConnectionContext context, MessageAck ack) throws IOException {
390            if (isConcurrentStoreAndDispatchQueues()) {
391                AsyncJobKey key = new AsyncJobKey(ack.getLastMessageId(), getDestination());
392                StoreQueueTask task = null;
393                synchronized (asyncTaskMap) {
394                    task = (StoreQueueTask) asyncTaskMap.get(key);
395                }
396                if (task != null) {
397                    if (ack.isInTransaction() || !task.cancel()) {
398                        try {
399                            task.future.get();
400                        } catch (InterruptedException e) {
401                            throw new InterruptedIOException(e.toString());
402                        } catch (Exception ignored) {
403                            LOG.debug("removeAsync: cannot cancel, waiting for add resulted in ex", ignored);
404                        }
405                        removeMessage(context, ack);
406                    } else {
407                        synchronized (asyncTaskMap) {
408                            asyncTaskMap.remove(key);
409                        }
410                    }
411                } else {
412                    removeMessage(context, ack);
413                }
414            } else {
415                removeMessage(context, ack);
416            }
417        }
418
419        @Override
420        public void addMessage(final ConnectionContext context, final Message message) throws IOException {
421            final KahaAddMessageCommand command = new KahaAddMessageCommand();
422            command.setDestination(dest);
423            command.setMessageId(message.getMessageId().toProducerKey());
424            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(message.getTransactionId())));
425            command.setPriority(message.getPriority());
426            command.setPrioritySupported(isPrioritizedMessages());
427            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
428            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
429                store(command, isEnableJournalDiskSyncs() && message.isResponseRequired(), new IndexAware() {
430                    // sync add? (for async, future present from getFutureOrSequenceLong)
431                    Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
432
433                    public void sequenceAssignedWithIndexLocked(final long sequence) {
434                        message.getMessageId().setFutureOrSequenceLong(sequence);
435                        if (indexListener != null) {
436                            if (possibleFuture == null) {
437                                trackPendingAdd(dest, sequence);
438                                indexListener.onAdd(new IndexListener.MessageContext(context, message, new Runnable() {
439                                    @Override
440                                    public void run() {
441                                        trackPendingAddComplete(dest, sequence);
442                                    }
443                                }));
444                            }
445                        }
446                    }
447                }, null);
448        }
449
450        @Override
451        public void updateMessage(Message message) throws IOException {
452            if (LOG.isTraceEnabled()) {
453                LOG.trace("updating: " + message.getMessageId() + " with deliveryCount: " + message.getRedeliveryCounter());
454            }
455            KahaUpdateMessageCommand updateMessageCommand = new KahaUpdateMessageCommand();
456            KahaAddMessageCommand command = new KahaAddMessageCommand();
457            command.setDestination(dest);
458            command.setMessageId(message.getMessageId().toProducerKey());
459            command.setPriority(message.getPriority());
460            command.setPrioritySupported(prioritizedMessages);
461            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
462            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
463            updateMessageCommand.setMessage(command);
464            store(updateMessageCommand, isEnableJournalDiskSyncs(), null, null);
465        }
466
467        @Override
468        public void removeMessage(ConnectionContext context, MessageAck ack) throws IOException {
469            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
470            command.setDestination(dest);
471            command.setMessageId(ack.getLastMessageId().toProducerKey());
472            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())));
473
474            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
475            command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
476            store(command, isEnableJournalDiskSyncs() && ack.isResponseRequired(), null, null);
477        }
478
479        @Override
480        public void removeAllMessages(ConnectionContext context) throws IOException {
481            KahaRemoveDestinationCommand command = new KahaRemoveDestinationCommand();
482            command.setDestination(dest);
483            store(command, true, null, null);
484        }
485
486        @Override
487        public Message getMessage(MessageId identity) throws IOException {
488            final String key = identity.toProducerKey();
489
490            // Hopefully one day the page file supports concurrent read
491            // operations... but for now we must
492            // externally synchronize...
493            Location location;
494            indexLock.writeLock().lock();
495            try {
496                location = findMessageLocation(key, dest);
497            } finally {
498                indexLock.writeLock().unlock();
499            }
500            if (location == null) {
501                return null;
502            }
503
504            return loadMessage(location);
505        }
506
507        @Override
508        public int getMessageCount() throws IOException {
509            try {
510                lockAsyncJobQueue();
511                indexLock.writeLock().lock();
512                try {
513                    return pageFile.tx().execute(new Transaction.CallableClosure<Integer, IOException>() {
514                        @Override
515                        public Integer execute(Transaction tx) throws IOException {
516                            // Iterate through all index entries to get a count
517                            // of messages in the destination.
518                            StoredDestination sd = getStoredDestination(dest, tx);
519                            int rc = 0;
520                            for (Iterator<Entry<Location, Long>> iterator = sd.locationIndex.iterator(tx); iterator.hasNext();) {
521                                iterator.next();
522                                rc++;
523                            }
524                            return rc;
525                        }
526                    });
527                } finally {
528                    indexLock.writeLock().unlock();
529                }
530            } finally {
531                unlockAsyncJobQueue();
532            }
533        }
534
535        @Override
536        public boolean isEmpty() throws IOException {
537            indexLock.writeLock().lock();
538            try {
539                return pageFile.tx().execute(new Transaction.CallableClosure<Boolean, IOException>() {
540                    @Override
541                    public Boolean execute(Transaction tx) throws IOException {
542                        // Iterate through all index entries to get a count of
543                        // messages in the destination.
544                        StoredDestination sd = getStoredDestination(dest, tx);
545                        return sd.locationIndex.isEmpty(tx);
546                    }
547                });
548            } finally {
549                indexLock.writeLock().unlock();
550            }
551        }
552
553        @Override
554        public void recover(final MessageRecoveryListener listener) throws Exception {
555            // recovery may involve expiry which will modify
556            indexLock.writeLock().lock();
557            try {
558                pageFile.tx().execute(new Transaction.Closure<Exception>() {
559                    @Override
560                    public void execute(Transaction tx) throws Exception {
561                        StoredDestination sd = getStoredDestination(dest, tx);
562                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
563                        sd.orderIndex.resetCursorPosition();
564                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); listener.hasSpace() && iterator
565                                .hasNext(); ) {
566                            Entry<Long, MessageKeys> entry = iterator.next();
567                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
568                                continue;
569                            }
570                            Message msg = loadMessage(entry.getValue().location);
571                            listener.recoverMessage(msg);
572                        }
573                    }
574                });
575            } finally {
576                indexLock.writeLock().unlock();
577            }
578        }
579
580        @Override
581        public void recoverNextMessages(final int maxReturned, final MessageRecoveryListener listener) throws Exception {
582            indexLock.writeLock().lock();
583            try {
584                pageFile.tx().execute(new Transaction.Closure<Exception>() {
585                    @Override
586                    public void execute(Transaction tx) throws Exception {
587                        StoredDestination sd = getStoredDestination(dest, tx);
588                        Entry<Long, MessageKeys> entry = null;
589                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
590                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator.hasNext(); ) {
591                            entry = iterator.next();
592                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
593                                continue;
594                            }
595                            Message msg = loadMessage(entry.getValue().location);
596                            msg.getMessageId().setFutureOrSequenceLong(entry.getKey());
597                            listener.recoverMessage(msg);
598                            counter++;
599                            if (counter >= maxReturned) {
600                                break;
601                            }
602                        }
603                        sd.orderIndex.stoppedIterating();
604                    }
605                });
606            } finally {
607                indexLock.writeLock().unlock();
608            }
609        }
610
611        protected int recoverRolledBackAcks(StoredDestination sd, Transaction tx, int maxReturned, MessageRecoveryListener listener) throws Exception {
612            int counter = 0;
613            String id;
614            for (Iterator<String> iterator = rolledBackAcks.iterator(); iterator.hasNext(); ) {
615                id = iterator.next();
616                iterator.remove();
617                Long sequence = sd.messageIdIndex.get(tx, id);
618                if (sequence != null) {
619                    if (sd.orderIndex.alreadyDispatched(sequence)) {
620                        listener.recoverMessage(loadMessage(sd.orderIndex.get(tx, sequence).location));
621                        counter++;
622                        if (counter >= maxReturned) {
623                            break;
624                        }
625                    } else {
626                        LOG.info("rolledback ack message {} with seq {} will be picked up in future batch {}", id, sequence, sd.orderIndex.cursor);
627                    }
628                } else {
629                    LOG.warn("Failed to locate rolled back ack message {} in {}", id, sd);
630                }
631            }
632            return counter;
633        }
634
635
636        @Override
637        public void resetBatching() {
638            if (pageFile.isLoaded()) {
639                indexLock.writeLock().lock();
640                try {
641                    pageFile.tx().execute(new Transaction.Closure<Exception>() {
642                        @Override
643                        public void execute(Transaction tx) throws Exception {
644                            StoredDestination sd = getExistingStoredDestination(dest, tx);
645                            if (sd != null) {
646                                sd.orderIndex.resetCursorPosition();}
647                            }
648                        });
649                } catch (Exception e) {
650                    LOG.error("Failed to reset batching",e);
651                } finally {
652                    indexLock.writeLock().unlock();
653                }
654            }
655        }
656
657        @Override
658        public void setBatch(final MessageId identity) throws IOException {
659            indexLock.writeLock().lock();
660            try {
661                pageFile.tx().execute(new Transaction.Closure<IOException>() {
662                    @Override
663                    public void execute(Transaction tx) throws IOException {
664                        StoredDestination sd = getStoredDestination(dest, tx);
665                        Long location = (Long) identity.getFutureOrSequenceLong();
666                        Long pending = sd.orderIndex.minPendingAdd();
667                        if (pending != null) {
668                            location = Math.min(location, pending-1);
669                        }
670                        sd.orderIndex.setBatch(tx, location);
671                    }
672                });
673            } finally {
674                indexLock.writeLock().unlock();
675            }
676        }
677
678        @Override
679        public void setMemoryUsage(MemoryUsage memoryUsage) {
680        }
681        @Override
682        public void start() throws Exception {
683            super.start();
684        }
685        @Override
686        public void stop() throws Exception {
687            super.stop();
688        }
689
690        protected void lockAsyncJobQueue() {
691            try {
692                if (!this.localDestinationSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS)) {
693                    throw new TimeoutException(this +" timeout waiting for localDestSem:" + this.localDestinationSemaphore);
694                }
695            } catch (Exception e) {
696                LOG.error("Failed to lock async jobs for " + this.destination, e);
697            }
698        }
699
700        protected void unlockAsyncJobQueue() {
701            this.localDestinationSemaphore.release(this.maxAsyncJobs);
702        }
703
704        protected void acquireLocalAsyncLock() {
705            try {
706                this.localDestinationSemaphore.acquire();
707            } catch (InterruptedException e) {
708                LOG.error("Failed to aquire async lock for " + this.destination, e);
709            }
710        }
711
712        protected void releaseLocalAsyncLock() {
713            this.localDestinationSemaphore.release();
714        }
715
716        @Override
717        public String toString(){
718            return "permits:" + this.localDestinationSemaphore.availablePermits() + ",sd=" + storedDestinations.get(key(dest));
719        }
720    }
721
722    class KahaDBTopicMessageStore extends KahaDBMessageStore implements TopicMessageStore {
723        private final AtomicInteger subscriptionCount = new AtomicInteger();
724        public KahaDBTopicMessageStore(ActiveMQTopic destination) throws IOException {
725            super(destination);
726            this.subscriptionCount.set(getAllSubscriptions().length);
727            asyncTopicMaps.add(asyncTaskMap);
728        }
729
730        @Override
731        public ListenableFuture<Object> asyncAddTopicMessage(final ConnectionContext context, final Message message)
732                throws IOException {
733            if (isConcurrentStoreAndDispatchTopics()) {
734                StoreTopicTask result = new StoreTopicTask(this, context, message, subscriptionCount.get());
735                result.aquireLocks();
736                addTopicTask(this, result);
737                return result.getFuture();
738            } else {
739                return super.asyncAddTopicMessage(context, message);
740            }
741        }
742
743        @Override
744        public void acknowledge(ConnectionContext context, String clientId, String subscriptionName,
745                                MessageId messageId, MessageAck ack) throws IOException {
746            String subscriptionKey = subscriptionKey(clientId, subscriptionName).toString();
747            if (isConcurrentStoreAndDispatchTopics()) {
748                AsyncJobKey key = new AsyncJobKey(messageId, getDestination());
749                StoreTopicTask task = null;
750                synchronized (asyncTaskMap) {
751                    task = (StoreTopicTask) asyncTaskMap.get(key);
752                }
753                if (task != null) {
754                    if (task.addSubscriptionKey(subscriptionKey)) {
755                        removeTopicTask(this, messageId);
756                        if (task.cancel()) {
757                            synchronized (asyncTaskMap) {
758                                asyncTaskMap.remove(key);
759                            }
760                        }
761                    }
762                } else {
763                    doAcknowledge(context, subscriptionKey, messageId, ack);
764                }
765            } else {
766                doAcknowledge(context, subscriptionKey, messageId, ack);
767            }
768        }
769
770        protected void doAcknowledge(ConnectionContext context, String subscriptionKey, MessageId messageId, MessageAck ack)
771                throws IOException {
772            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
773            command.setDestination(dest);
774            command.setSubscriptionKey(subscriptionKey);
775            command.setMessageId(messageId.toProducerKey());
776            command.setTransactionInfo(ack != null ? TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())) : null);
777            if (ack != null && ack.isUnmatchedAck()) {
778                command.setAck(UNMATCHED);
779            } else {
780                org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
781                command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
782            }
783            store(command, false, null, null);
784        }
785
786        @Override
787        public void addSubscription(SubscriptionInfo subscriptionInfo, boolean retroactive) throws IOException {
788            String subscriptionKey = subscriptionKey(subscriptionInfo.getClientId(), subscriptionInfo
789                    .getSubscriptionName());
790            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
791            command.setDestination(dest);
792            command.setSubscriptionKey(subscriptionKey.toString());
793            command.setRetroactive(retroactive);
794            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(subscriptionInfo);
795            command.setSubscriptionInfo(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
796            store(command, isEnableJournalDiskSyncs() && true, null, null);
797            this.subscriptionCount.incrementAndGet();
798        }
799
800        @Override
801        public void deleteSubscription(String clientId, String subscriptionName) throws IOException {
802            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
803            command.setDestination(dest);
804            command.setSubscriptionKey(subscriptionKey(clientId, subscriptionName).toString());
805            store(command, isEnableJournalDiskSyncs() && true, null, null);
806            this.subscriptionCount.decrementAndGet();
807        }
808
809        @Override
810        public SubscriptionInfo[] getAllSubscriptions() throws IOException {
811
812            final ArrayList<SubscriptionInfo> subscriptions = new ArrayList<SubscriptionInfo>();
813            indexLock.writeLock().lock();
814            try {
815                pageFile.tx().execute(new Transaction.Closure<IOException>() {
816                    @Override
817                    public void execute(Transaction tx) throws IOException {
818                        StoredDestination sd = getStoredDestination(dest, tx);
819                        for (Iterator<Entry<String, KahaSubscriptionCommand>> iterator = sd.subscriptions.iterator(tx); iterator
820                                .hasNext();) {
821                            Entry<String, KahaSubscriptionCommand> entry = iterator.next();
822                            SubscriptionInfo info = (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(entry
823                                    .getValue().getSubscriptionInfo().newInput()));
824                            subscriptions.add(info);
825
826                        }
827                    }
828                });
829            } finally {
830                indexLock.writeLock().unlock();
831            }
832
833            SubscriptionInfo[] rc = new SubscriptionInfo[subscriptions.size()];
834            subscriptions.toArray(rc);
835            return rc;
836        }
837
838        @Override
839        public SubscriptionInfo lookupSubscription(String clientId, String subscriptionName) throws IOException {
840            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
841            indexLock.writeLock().lock();
842            try {
843                return pageFile.tx().execute(new Transaction.CallableClosure<SubscriptionInfo, IOException>() {
844                    @Override
845                    public SubscriptionInfo execute(Transaction tx) throws IOException {
846                        StoredDestination sd = getStoredDestination(dest, tx);
847                        KahaSubscriptionCommand command = sd.subscriptions.get(tx, subscriptionKey);
848                        if (command == null) {
849                            return null;
850                        }
851                        return (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(command
852                                .getSubscriptionInfo().newInput()));
853                    }
854                });
855            } finally {
856                indexLock.writeLock().unlock();
857            }
858        }
859
860        @Override
861        public int getMessageCount(String clientId, String subscriptionName) throws IOException {
862            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
863            indexLock.writeLock().lock();
864            try {
865                return pageFile.tx().execute(new Transaction.CallableClosure<Integer, IOException>() {
866                    @Override
867                    public Integer execute(Transaction tx) throws IOException {
868                        StoredDestination sd = getStoredDestination(dest, tx);
869                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
870                        if (cursorPos == null) {
871                            // The subscription might not exist.
872                            return 0;
873                        }
874
875                        return (int) getStoredMessageCount(tx, sd, subscriptionKey);
876                    }
877                });
878            } finally {
879                indexLock.writeLock().unlock();
880            }
881        }
882
883        @Override
884        public void recoverSubscription(String clientId, String subscriptionName, final MessageRecoveryListener listener)
885                throws Exception {
886            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
887            @SuppressWarnings("unused")
888            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
889            indexLock.writeLock().lock();
890            try {
891                pageFile.tx().execute(new Transaction.Closure<Exception>() {
892                    @Override
893                    public void execute(Transaction tx) throws Exception {
894                        StoredDestination sd = getStoredDestination(dest, tx);
895                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
896                        sd.orderIndex.setBatch(tx, cursorPos);
897                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
898                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator
899                                .hasNext();) {
900                            Entry<Long, MessageKeys> entry = iterator.next();
901                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
902                                continue;
903                            }
904                            listener.recoverMessage(loadMessage(entry.getValue().location));
905                        }
906                        sd.orderIndex.resetCursorPosition();
907                    }
908                });
909            } finally {
910                indexLock.writeLock().unlock();
911            }
912        }
913
914        @Override
915        public void recoverNextMessages(String clientId, String subscriptionName, final int maxReturned,
916                final MessageRecoveryListener listener) throws Exception {
917            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
918            @SuppressWarnings("unused")
919            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
920            indexLock.writeLock().lock();
921            try {
922                pageFile.tx().execute(new Transaction.Closure<Exception>() {
923                    @Override
924                    public void execute(Transaction tx) throws Exception {
925                        StoredDestination sd = getStoredDestination(dest, tx);
926                        sd.orderIndex.resetCursorPosition();
927                        MessageOrderCursor moc = sd.subscriptionCursors.get(subscriptionKey);
928                        if (moc == null) {
929                            LastAck pos = getLastAck(tx, sd, subscriptionKey);
930                            if (pos == null) {
931                                // sub deleted
932                                return;
933                            }
934                            sd.orderIndex.setBatch(tx, pos);
935                            moc = sd.orderIndex.cursor;
936                        } else {
937                            sd.orderIndex.cursor.sync(moc);
938                        }
939
940                        Entry<Long, MessageKeys> entry = null;
941                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
942                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx, moc); iterator
943                                .hasNext();) {
944                            entry = iterator.next();
945                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
946                                continue;
947                            }
948                            if (listener.recoverMessage(loadMessage(entry.getValue().location))) {
949                                counter++;
950                            }
951                            if (counter >= maxReturned || listener.hasSpace() == false) {
952                                break;
953                            }
954                        }
955                        sd.orderIndex.stoppedIterating();
956                        if (entry != null) {
957                            MessageOrderCursor copy = sd.orderIndex.cursor.copy();
958                            sd.subscriptionCursors.put(subscriptionKey, copy);
959                        }
960                    }
961                });
962            } finally {
963                indexLock.writeLock().unlock();
964            }
965        }
966
967        @Override
968        public void resetBatching(String clientId, String subscriptionName) {
969            try {
970                final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
971                indexLock.writeLock().lock();
972                try {
973                    pageFile.tx().execute(new Transaction.Closure<IOException>() {
974                        @Override
975                        public void execute(Transaction tx) throws IOException {
976                            StoredDestination sd = getStoredDestination(dest, tx);
977                            sd.subscriptionCursors.remove(subscriptionKey);
978                        }
979                    });
980                }finally {
981                    indexLock.writeLock().unlock();
982                }
983            } catch (IOException e) {
984                throw new RuntimeException(e);
985            }
986        }
987    }
988
989    String subscriptionKey(String clientId, String subscriptionName) {
990        return clientId + ":" + subscriptionName;
991    }
992
993    @Override
994    public MessageStore createQueueMessageStore(ActiveMQQueue destination) throws IOException {
995        return this.transactionStore.proxy(new KahaDBMessageStore(destination));
996    }
997
998    @Override
999    public TopicMessageStore createTopicMessageStore(ActiveMQTopic destination) throws IOException {
1000        return this.transactionStore.proxy(new KahaDBTopicMessageStore(destination));
1001    }
1002
1003    /**
1004     * Cleanup method to remove any state associated with the given destination.
1005     * This method does not stop the message store (it might not be cached).
1006     *
1007     * @param destination
1008     *            Destination to forget
1009     */
1010    @Override
1011    public void removeQueueMessageStore(ActiveMQQueue destination) {
1012    }
1013
1014    /**
1015     * Cleanup method to remove any state associated with the given destination
1016     * This method does not stop the message store (it might not be cached).
1017     *
1018     * @param destination
1019     *            Destination to forget
1020     */
1021    @Override
1022    public void removeTopicMessageStore(ActiveMQTopic destination) {
1023    }
1024
1025    @Override
1026    public void deleteAllMessages() throws IOException {
1027        deleteAllMessages = true;
1028    }
1029
1030    @Override
1031    public Set<ActiveMQDestination> getDestinations() {
1032        try {
1033            final HashSet<ActiveMQDestination> rc = new HashSet<ActiveMQDestination>();
1034            indexLock.writeLock().lock();
1035            try {
1036                pageFile.tx().execute(new Transaction.Closure<IOException>() {
1037                    @Override
1038                    public void execute(Transaction tx) throws IOException {
1039                        for (Iterator<Entry<String, StoredDestination>> iterator = metadata.destinations.iterator(tx); iterator
1040                                .hasNext();) {
1041                            Entry<String, StoredDestination> entry = iterator.next();
1042                            if (!isEmptyTopic(entry, tx)) {
1043                                rc.add(convert(entry.getKey()));
1044                            }
1045                        }
1046                    }
1047
1048                    private boolean isEmptyTopic(Entry<String, StoredDestination> entry, Transaction tx)
1049                            throws IOException {
1050                        boolean isEmptyTopic = false;
1051                        ActiveMQDestination dest = convert(entry.getKey());
1052                        if (dest.isTopic()) {
1053                            StoredDestination loadedStore = getStoredDestination(convert(dest), tx);
1054                            if (loadedStore.subscriptionAcks.isEmpty(tx)) {
1055                                isEmptyTopic = true;
1056                            }
1057                        }
1058                        return isEmptyTopic;
1059                    }
1060                });
1061            }finally {
1062                indexLock.writeLock().unlock();
1063            }
1064            return rc;
1065        } catch (IOException e) {
1066            throw new RuntimeException(e);
1067        }
1068    }
1069
1070    @Override
1071    public long getLastMessageBrokerSequenceId() throws IOException {
1072        return 0;
1073    }
1074
1075    @Override
1076    public long getLastProducerSequenceId(ProducerId id) {
1077        indexLock.readLock().lock();
1078        try {
1079            return metadata.producerSequenceIdTracker.getLastSeqId(id);
1080        } finally {
1081            indexLock.readLock().unlock();
1082        }
1083    }
1084
1085    @Override
1086    public long size() {
1087        try {
1088            return journalSize.get() + getPageFile().getDiskSize();
1089        } catch (IOException e) {
1090            throw new RuntimeException(e);
1091        }
1092    }
1093
1094    @Override
1095    public void beginTransaction(ConnectionContext context) throws IOException {
1096        throw new IOException("Not yet implemented.");
1097    }
1098    @Override
1099    public void commitTransaction(ConnectionContext context) throws IOException {
1100        throw new IOException("Not yet implemented.");
1101    }
1102    @Override
1103    public void rollbackTransaction(ConnectionContext context) throws IOException {
1104        throw new IOException("Not yet implemented.");
1105    }
1106
1107    @Override
1108    public void checkpoint(boolean sync) throws IOException {
1109        super.checkpointCleanup(sync);
1110    }
1111
1112    // /////////////////////////////////////////////////////////////////
1113    // Internal helper methods.
1114    // /////////////////////////////////////////////////////////////////
1115
1116    /**
1117     * @param location
1118     * @return
1119     * @throws IOException
1120     */
1121    Message loadMessage(Location location) throws IOException {
1122        JournalCommand<?> command = load(location);
1123        KahaAddMessageCommand addMessage = null;
1124        switch (command.type()) {
1125            case KAHA_UPDATE_MESSAGE_COMMAND:
1126                addMessage = ((KahaUpdateMessageCommand)command).getMessage();
1127                break;
1128            default:
1129                addMessage = (KahaAddMessageCommand) command;
1130        }
1131        Message msg = (Message) wireFormat.unmarshal(new DataInputStream(addMessage.getMessage().newInput()));
1132        return msg;
1133    }
1134
1135    // /////////////////////////////////////////////////////////////////
1136    // Internal conversion methods.
1137    // /////////////////////////////////////////////////////////////////
1138
1139    KahaLocation convert(Location location) {
1140        KahaLocation rc = new KahaLocation();
1141        rc.setLogId(location.getDataFileId());
1142        rc.setOffset(location.getOffset());
1143        return rc;
1144    }
1145
1146    KahaDestination convert(ActiveMQDestination dest) {
1147        KahaDestination rc = new KahaDestination();
1148        rc.setName(dest.getPhysicalName());
1149        switch (dest.getDestinationType()) {
1150        case ActiveMQDestination.QUEUE_TYPE:
1151            rc.setType(DestinationType.QUEUE);
1152            return rc;
1153        case ActiveMQDestination.TOPIC_TYPE:
1154            rc.setType(DestinationType.TOPIC);
1155            return rc;
1156        case ActiveMQDestination.TEMP_QUEUE_TYPE:
1157            rc.setType(DestinationType.TEMP_QUEUE);
1158            return rc;
1159        case ActiveMQDestination.TEMP_TOPIC_TYPE:
1160            rc.setType(DestinationType.TEMP_TOPIC);
1161            return rc;
1162        default:
1163            return null;
1164        }
1165    }
1166
1167    ActiveMQDestination convert(String dest) {
1168        int p = dest.indexOf(":");
1169        if (p < 0) {
1170            throw new IllegalArgumentException("Not in the valid destination format");
1171        }
1172        int type = Integer.parseInt(dest.substring(0, p));
1173        String name = dest.substring(p + 1);
1174        return convert(type, name);
1175    }
1176
1177    private ActiveMQDestination convert(KahaDestination commandDestination) {
1178        return convert(commandDestination.getType().getNumber(), commandDestination.getName());
1179    }
1180
1181    private ActiveMQDestination convert(int type, String name) {
1182        switch (KahaDestination.DestinationType.valueOf(type)) {
1183        case QUEUE:
1184            return new ActiveMQQueue(name);
1185        case TOPIC:
1186            return new ActiveMQTopic(name);
1187        case TEMP_QUEUE:
1188            return new ActiveMQTempQueue(name);
1189        case TEMP_TOPIC:
1190            return new ActiveMQTempTopic(name);
1191        default:
1192            throw new IllegalArgumentException("Not in the valid destination format");
1193        }
1194    }
1195
1196    public TransactionIdTransformer getTransactionIdTransformer() {
1197        return transactionIdTransformer;
1198    }
1199
1200    public void setTransactionIdTransformer(TransactionIdTransformer transactionIdTransformer) {
1201        this.transactionIdTransformer = transactionIdTransformer;
1202    }
1203
1204    static class AsyncJobKey {
1205        MessageId id;
1206        ActiveMQDestination destination;
1207
1208        AsyncJobKey(MessageId id, ActiveMQDestination destination) {
1209            this.id = id;
1210            this.destination = destination;
1211        }
1212
1213        @Override
1214        public boolean equals(Object obj) {
1215            if (obj == this) {
1216                return true;
1217            }
1218            return obj instanceof AsyncJobKey && id.equals(((AsyncJobKey) obj).id)
1219                    && destination.equals(((AsyncJobKey) obj).destination);
1220        }
1221
1222        @Override
1223        public int hashCode() {
1224            return id.hashCode() + destination.hashCode();
1225        }
1226
1227        @Override
1228        public String toString() {
1229            return destination.getPhysicalName() + "-" + id;
1230        }
1231    }
1232
1233    public interface StoreTask {
1234        public boolean cancel();
1235
1236        public void aquireLocks();
1237
1238        public void releaseLocks();
1239    }
1240
1241    class StoreQueueTask implements Runnable, StoreTask {
1242        protected final Message message;
1243        protected final ConnectionContext context;
1244        protected final KahaDBMessageStore store;
1245        protected final InnerFutureTask future;
1246        protected final AtomicBoolean done = new AtomicBoolean();
1247        protected final AtomicBoolean locked = new AtomicBoolean();
1248
1249        public StoreQueueTask(KahaDBMessageStore store, ConnectionContext context, Message message) {
1250            this.store = store;
1251            this.context = context;
1252            this.message = message;
1253            this.future = new InnerFutureTask(this);
1254        }
1255
1256        public ListenableFuture<Object> getFuture() {
1257            return this.future;
1258        }
1259
1260        @Override
1261        public boolean cancel() {
1262            if (this.done.compareAndSet(false, true)) {
1263                return this.future.cancel(false);
1264            }
1265            return false;
1266        }
1267
1268        @Override
1269        public void aquireLocks() {
1270            if (this.locked.compareAndSet(false, true)) {
1271                try {
1272                    globalQueueSemaphore.acquire();
1273                    store.acquireLocalAsyncLock();
1274                    message.incrementReferenceCount();
1275                } catch (InterruptedException e) {
1276                    LOG.warn("Failed to aquire lock", e);
1277                }
1278            }
1279
1280        }
1281
1282        @Override
1283        public void releaseLocks() {
1284            if (this.locked.compareAndSet(true, false)) {
1285                store.releaseLocalAsyncLock();
1286                globalQueueSemaphore.release();
1287                message.decrementReferenceCount();
1288            }
1289        }
1290
1291        @Override
1292        public void run() {
1293            this.store.doneTasks++;
1294            try {
1295                if (this.done.compareAndSet(false, true)) {
1296                    this.store.addMessage(context, message);
1297                    removeQueueTask(this.store, this.message.getMessageId());
1298                    this.future.complete();
1299                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1300                    System.err.println(this.store.dest.getName() + " cancelled: "
1301                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1302                    this.store.canceledTasks = this.store.doneTasks = 0;
1303                }
1304            } catch (Exception e) {
1305                this.future.setException(e);
1306            }
1307        }
1308
1309        protected Message getMessage() {
1310            return this.message;
1311        }
1312
1313        private class InnerFutureTask extends FutureTask<Object> implements ListenableFuture<Object>  {
1314
1315            private Runnable listener;
1316            public InnerFutureTask(Runnable runnable) {
1317                super(runnable, null);
1318
1319            }
1320
1321            public void setException(final Exception e) {
1322                super.setException(e);
1323            }
1324
1325            public void complete() {
1326                super.set(null);
1327            }
1328
1329            @Override
1330            public void done() {
1331                fireListener();
1332            }
1333
1334            @Override
1335            public void addListener(Runnable listener) {
1336                this.listener = listener;
1337                if (isDone()) {
1338                    fireListener();
1339                }
1340            }
1341
1342            private void fireListener() {
1343                if (listener != null) {
1344                    try {
1345                        listener.run();
1346                    } catch (Exception ignored) {
1347                        LOG.warn("Unexpected exception from future {} listener callback {}", this, listener, ignored);
1348                    }
1349                }
1350            }
1351        }
1352    }
1353
1354    class StoreTopicTask extends StoreQueueTask {
1355        private final int subscriptionCount;
1356        private final List<String> subscriptionKeys = new ArrayList<String>(1);
1357        private final KahaDBTopicMessageStore topicStore;
1358        public StoreTopicTask(KahaDBTopicMessageStore store, ConnectionContext context, Message message,
1359                int subscriptionCount) {
1360            super(store, context, message);
1361            this.topicStore = store;
1362            this.subscriptionCount = subscriptionCount;
1363
1364        }
1365
1366        @Override
1367        public void aquireLocks() {
1368            if (this.locked.compareAndSet(false, true)) {
1369                try {
1370                    globalTopicSemaphore.acquire();
1371                    store.acquireLocalAsyncLock();
1372                    message.incrementReferenceCount();
1373                } catch (InterruptedException e) {
1374                    LOG.warn("Failed to aquire lock", e);
1375                }
1376            }
1377        }
1378
1379        @Override
1380        public void releaseLocks() {
1381            if (this.locked.compareAndSet(true, false)) {
1382                message.decrementReferenceCount();
1383                store.releaseLocalAsyncLock();
1384                globalTopicSemaphore.release();
1385            }
1386        }
1387
1388        /**
1389         * add a key
1390         *
1391         * @param key
1392         * @return true if all acknowledgements received
1393         */
1394        public boolean addSubscriptionKey(String key) {
1395            synchronized (this.subscriptionKeys) {
1396                this.subscriptionKeys.add(key);
1397            }
1398            return this.subscriptionKeys.size() >= this.subscriptionCount;
1399        }
1400
1401        @Override
1402        public void run() {
1403            this.store.doneTasks++;
1404            try {
1405                if (this.done.compareAndSet(false, true)) {
1406                    this.topicStore.addMessage(context, message);
1407                    // apply any acks we have
1408                    synchronized (this.subscriptionKeys) {
1409                        for (String key : this.subscriptionKeys) {
1410                            this.topicStore.doAcknowledge(context, key, this.message.getMessageId(), null);
1411
1412                        }
1413                    }
1414                    removeTopicTask(this.topicStore, this.message.getMessageId());
1415                    this.future.complete();
1416                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1417                    System.err.println(this.store.dest.getName() + " cancelled: "
1418                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1419                    this.store.canceledTasks = this.store.doneTasks = 0;
1420                }
1421            } catch (Exception e) {
1422                this.future.setException(e);
1423            }
1424        }
1425    }
1426
1427    public class StoreTaskExecutor extends ThreadPoolExecutor {
1428
1429        public StoreTaskExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit timeUnit, BlockingQueue<Runnable> queue, ThreadFactory threadFactory) {
1430            super(corePoolSize, maximumPoolSize, keepAliveTime, timeUnit, queue, threadFactory);
1431        }
1432
1433        @Override
1434        protected void afterExecute(Runnable runnable, Throwable throwable) {
1435            super.afterExecute(runnable, throwable);
1436
1437            if (runnable instanceof StoreTask) {
1438               ((StoreTask)runnable).releaseLocks();
1439            }
1440        }
1441    }
1442
1443    @Override
1444    public JobSchedulerStore createJobSchedulerStore() throws IOException, UnsupportedOperationException {
1445        return new JobSchedulerStoreImpl();
1446    }
1447}