Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.store.kahadb;
018
019import java.io.DataInputStream;
020import java.io.IOException;
021import java.io.InterruptedIOException;
022import java.util.ArrayList;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.LinkedList;
027import java.util.List;
028import java.util.Map;
029import java.util.Map.Entry;
030import java.util.Set;
031import java.util.concurrent.BlockingQueue;
032import java.util.concurrent.ExecutorService;
033import java.util.concurrent.FutureTask;
034import java.util.concurrent.LinkedBlockingQueue;
035import java.util.concurrent.Semaphore;
036import java.util.concurrent.ThreadFactory;
037import java.util.concurrent.ThreadPoolExecutor;
038import java.util.concurrent.TimeUnit;
039import java.util.concurrent.TimeoutException;
040import java.util.concurrent.atomic.AtomicBoolean;
041import java.util.concurrent.atomic.AtomicInteger;
042
043import org.apache.activemq.broker.ConnectionContext;
044import org.apache.activemq.broker.region.BaseDestination;
045import org.apache.activemq.broker.scheduler.JobSchedulerStore;
046import org.apache.activemq.command.ActiveMQDestination;
047import org.apache.activemq.command.ActiveMQQueue;
048import org.apache.activemq.command.ActiveMQTempQueue;
049import org.apache.activemq.command.ActiveMQTempTopic;
050import org.apache.activemq.command.ActiveMQTopic;
051import org.apache.activemq.command.Message;
052import org.apache.activemq.command.MessageAck;
053import org.apache.activemq.command.MessageId;
054import org.apache.activemq.command.ProducerId;
055import org.apache.activemq.command.SubscriptionInfo;
056import org.apache.activemq.command.TransactionId;
057import org.apache.activemq.openwire.OpenWireFormat;
058import org.apache.activemq.protobuf.Buffer;
059import org.apache.activemq.store.AbstractMessageStore;
060import org.apache.activemq.store.IndexListener;
061import org.apache.activemq.store.ListenableFuture;
062import org.apache.activemq.store.MessageRecoveryListener;
063import org.apache.activemq.store.MessageStore;
064import org.apache.activemq.store.MessageStoreStatistics;
065import org.apache.activemq.store.PersistenceAdapter;
066import org.apache.activemq.store.TopicMessageStore;
067import org.apache.activemq.store.TransactionIdTransformer;
068import org.apache.activemq.store.TransactionStore;
069import org.apache.activemq.store.kahadb.data.KahaAddMessageCommand;
070import org.apache.activemq.store.kahadb.data.KahaDestination;
071import org.apache.activemq.store.kahadb.data.KahaDestination.DestinationType;
072import org.apache.activemq.store.kahadb.data.KahaLocation;
073import org.apache.activemq.store.kahadb.data.KahaRemoveDestinationCommand;
074import org.apache.activemq.store.kahadb.data.KahaRemoveMessageCommand;
075import org.apache.activemq.store.kahadb.data.KahaSubscriptionCommand;
076import org.apache.activemq.store.kahadb.data.KahaUpdateMessageCommand;
077import org.apache.activemq.store.kahadb.disk.journal.Location;
078import org.apache.activemq.store.kahadb.disk.page.Transaction;
079import org.apache.activemq.store.kahadb.scheduler.JobSchedulerStoreImpl;
080import org.apache.activemq.usage.MemoryUsage;
081import org.apache.activemq.usage.SystemUsage;
082import org.apache.activemq.util.ServiceStopper;
083import org.apache.activemq.util.ThreadPoolUtils;
084import org.apache.activemq.wireformat.WireFormat;
085import org.slf4j.Logger;
086import org.slf4j.LoggerFactory;
087
088public class KahaDBStore extends MessageDatabase implements PersistenceAdapter {
089    static final Logger LOG = LoggerFactory.getLogger(KahaDBStore.class);
090    private static final int MAX_ASYNC_JOBS = BaseDestination.MAX_AUDIT_DEPTH;
091
092    public static final String PROPERTY_CANCELED_TASK_MOD_METRIC = "org.apache.activemq.store.kahadb.CANCELED_TASK_MOD_METRIC";
093    public static final int cancelledTaskModMetric = Integer.parseInt(System.getProperty(
094            PROPERTY_CANCELED_TASK_MOD_METRIC, "0"), 10);
095    public static final String PROPERTY_ASYNC_EXECUTOR_MAX_THREADS = "org.apache.activemq.store.kahadb.ASYNC_EXECUTOR_MAX_THREADS";
096    private static final int asyncExecutorMaxThreads = Integer.parseInt(System.getProperty(
097            PROPERTY_ASYNC_EXECUTOR_MAX_THREADS, "1"), 10);;
098
099    protected ExecutorService queueExecutor;
100    protected ExecutorService topicExecutor;
101    protected final List<Map<AsyncJobKey, StoreTask>> asyncQueueMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
102    protected final List<Map<AsyncJobKey, StoreTask>> asyncTopicMaps = new LinkedList<Map<AsyncJobKey, StoreTask>>();
103    final WireFormat wireFormat = new OpenWireFormat();
104    private SystemUsage usageManager;
105    private LinkedBlockingQueue<Runnable> asyncQueueJobQueue;
106    private LinkedBlockingQueue<Runnable> asyncTopicJobQueue;
107    Semaphore globalQueueSemaphore;
108    Semaphore globalTopicSemaphore;
109    private boolean concurrentStoreAndDispatchQueues = true;
110    // when true, message order may be compromised when cache is exhausted if store is out
111    // or order w.r.t cache
112    private boolean concurrentStoreAndDispatchTopics = false;
113    private final boolean concurrentStoreAndDispatchTransactions = false;
114    private int maxAsyncJobs = MAX_ASYNC_JOBS;
115    private final KahaDBTransactionStore transactionStore;
116    private TransactionIdTransformer transactionIdTransformer;
117
118    public KahaDBStore() {
119        this.transactionStore = new KahaDBTransactionStore(this);
120        this.transactionIdTransformer = new TransactionIdTransformer() {
121            @Override
122            public TransactionId transform(TransactionId txid) {
123                return txid;
124            }
125        };
126    }
127
128    @Override
129    public String toString() {
130        return "KahaDB:[" + directory.getAbsolutePath() + "]";
131    }
132
133    @Override
134    public void setBrokerName(String brokerName) {
135    }
136
137    @Override
138    public void setUsageManager(SystemUsage usageManager) {
139        this.usageManager = usageManager;
140    }
141
142    public SystemUsage getUsageManager() {
143        return this.usageManager;
144    }
145
146    /**
147     * @return the concurrentStoreAndDispatch
148     */
149    public boolean isConcurrentStoreAndDispatchQueues() {
150        return this.concurrentStoreAndDispatchQueues;
151    }
152
153    /**
154     * @param concurrentStoreAndDispatch
155     *            the concurrentStoreAndDispatch to set
156     */
157    public void setConcurrentStoreAndDispatchQueues(boolean concurrentStoreAndDispatch) {
158        this.concurrentStoreAndDispatchQueues = concurrentStoreAndDispatch;
159    }
160
161    /**
162     * @return the concurrentStoreAndDispatch
163     */
164    public boolean isConcurrentStoreAndDispatchTopics() {
165        return this.concurrentStoreAndDispatchTopics;
166    }
167
168    /**
169     * @param concurrentStoreAndDispatch
170     *            the concurrentStoreAndDispatch to set
171     */
172    public void setConcurrentStoreAndDispatchTopics(boolean concurrentStoreAndDispatch) {
173        this.concurrentStoreAndDispatchTopics = concurrentStoreAndDispatch;
174    }
175
176    public boolean isConcurrentStoreAndDispatchTransactions() {
177        return this.concurrentStoreAndDispatchTransactions;
178    }
179
180    /**
181     * @return the maxAsyncJobs
182     */
183    public int getMaxAsyncJobs() {
184        return this.maxAsyncJobs;
185    }
186
187    /**
188     * @param maxAsyncJobs
189     *            the maxAsyncJobs to set
190     */
191    public void setMaxAsyncJobs(int maxAsyncJobs) {
192        this.maxAsyncJobs = maxAsyncJobs;
193    }
194
195    @Override
196    public void doStart() throws Exception {
197        if (brokerService != null) {
198            metadata.openwireVersion = brokerService.getStoreOpenWireVersion();
199            wireFormat.setVersion(metadata.openwireVersion);
200
201            if (LOG.isDebugEnabled()) {
202                LOG.debug("Store OpenWire version configured as: {}", metadata.openwireVersion);
203            }
204
205        }
206        super.doStart();
207
208        if (brokerService != null) {
209            // In case the recovered store used a different OpenWire version log a warning
210            // to assist in determining why journal reads fail.
211            if (metadata.openwireVersion != brokerService.getStoreOpenWireVersion()) {
212                LOG.warn("Existing Store uses a different OpenWire version[{}] " +
213                         "than the version configured[{}] reverting to the version " +
214                         "used by this store, some newer broker features may not work" +
215                         "as expected.",
216                         metadata.openwireVersion, brokerService.getStoreOpenWireVersion());
217
218                // Update the broker service instance to the actual version in use.
219                wireFormat.setVersion(metadata.openwireVersion);
220                brokerService.setStoreOpenWireVersion(metadata.openwireVersion);
221            }
222        }
223
224        this.globalQueueSemaphore = new Semaphore(getMaxAsyncJobs());
225        this.globalTopicSemaphore = new Semaphore(getMaxAsyncJobs());
226        this.asyncQueueJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
227        this.asyncTopicJobQueue = new LinkedBlockingQueue<Runnable>(getMaxAsyncJobs());
228        this.queueExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
229            asyncQueueJobQueue, new ThreadFactory() {
230                @Override
231                public Thread newThread(Runnable runnable) {
232                    Thread thread = new Thread(runnable, "ConcurrentQueueStoreAndDispatch");
233                    thread.setDaemon(true);
234                    return thread;
235                }
236            });
237        this.topicExecutor = new StoreTaskExecutor(1, asyncExecutorMaxThreads, 0L, TimeUnit.MILLISECONDS,
238            asyncTopicJobQueue, new ThreadFactory() {
239                @Override
240                public Thread newThread(Runnable runnable) {
241                    Thread thread = new Thread(runnable, "ConcurrentTopicStoreAndDispatch");
242                    thread.setDaemon(true);
243                    return thread;
244                }
245            });
246    }
247
248    @Override
249    public void doStop(ServiceStopper stopper) throws Exception {
250        // drain down async jobs
251        LOG.info("Stopping async queue tasks");
252        if (this.globalQueueSemaphore != null) {
253            this.globalQueueSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
254        }
255        synchronized (this.asyncQueueMaps) {
256            for (Map<AsyncJobKey, StoreTask> m : asyncQueueMaps) {
257                synchronized (m) {
258                    for (StoreTask task : m.values()) {
259                        task.cancel();
260                    }
261                }
262            }
263            this.asyncQueueMaps.clear();
264        }
265        LOG.info("Stopping async topic tasks");
266        if (this.globalTopicSemaphore != null) {
267            this.globalTopicSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS);
268        }
269        synchronized (this.asyncTopicMaps) {
270            for (Map<AsyncJobKey, StoreTask> m : asyncTopicMaps) {
271                synchronized (m) {
272                    for (StoreTask task : m.values()) {
273                        task.cancel();
274                    }
275                }
276            }
277            this.asyncTopicMaps.clear();
278        }
279        if (this.globalQueueSemaphore != null) {
280            this.globalQueueSemaphore.drainPermits();
281        }
282        if (this.globalTopicSemaphore != null) {
283            this.globalTopicSemaphore.drainPermits();
284        }
285        if (this.queueExecutor != null) {
286            ThreadPoolUtils.shutdownNow(queueExecutor);
287            queueExecutor = null;
288        }
289        if (this.topicExecutor != null) {
290            ThreadPoolUtils.shutdownNow(topicExecutor);
291            topicExecutor = null;
292        }
293        LOG.info("Stopped KahaDB");
294        super.doStop(stopper);
295    }
296
297    private Location findMessageLocation(final String key, final KahaDestination destination) throws IOException {
298        return pageFile.tx().execute(new Transaction.CallableClosure<Location, IOException>() {
299            @Override
300            public Location execute(Transaction tx) throws IOException {
301                StoredDestination sd = getStoredDestination(destination, tx);
302                Long sequence = sd.messageIdIndex.get(tx, key);
303                if (sequence == null) {
304                    return null;
305                }
306                return sd.orderIndex.get(tx, sequence).location;
307            }
308        });
309    }
310
311    protected StoreQueueTask removeQueueTask(KahaDBMessageStore store, MessageId id) {
312        StoreQueueTask task = null;
313        synchronized (store.asyncTaskMap) {
314            task = (StoreQueueTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
315        }
316        return task;
317    }
318
319    protected void addQueueTask(KahaDBMessageStore store, StoreQueueTask task) throws IOException {
320        synchronized (store.asyncTaskMap) {
321            store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
322        }
323        this.queueExecutor.execute(task);
324    }
325
326    protected StoreTopicTask removeTopicTask(KahaDBTopicMessageStore store, MessageId id) {
327        StoreTopicTask task = null;
328        synchronized (store.asyncTaskMap) {
329            task = (StoreTopicTask) store.asyncTaskMap.remove(new AsyncJobKey(id, store.getDestination()));
330        }
331        return task;
332    }
333
334    protected void addTopicTask(KahaDBTopicMessageStore store, StoreTopicTask task) throws IOException {
335        synchronized (store.asyncTaskMap) {
336            store.asyncTaskMap.put(new AsyncJobKey(task.getMessage().getMessageId(), store.getDestination()), task);
337        }
338        this.topicExecutor.execute(task);
339    }
340
341    @Override
342    public TransactionStore createTransactionStore() throws IOException {
343        return this.transactionStore;
344    }
345
346    public boolean getForceRecoverIndex() {
347        return this.forceRecoverIndex;
348    }
349
350    public void setForceRecoverIndex(boolean forceRecoverIndex) {
351        this.forceRecoverIndex = forceRecoverIndex;
352    }
353
354    public class KahaDBMessageStore extends AbstractMessageStore {
355        protected final Map<AsyncJobKey, StoreTask> asyncTaskMap = new HashMap<AsyncJobKey, StoreTask>();
356        protected KahaDestination dest;
357        private final int maxAsyncJobs;
358        private final Semaphore localDestinationSemaphore;
359
360        double doneTasks, canceledTasks = 0;
361
362        public KahaDBMessageStore(ActiveMQDestination destination) {
363            super(destination);
364            this.dest = convert(destination);
365            this.maxAsyncJobs = getMaxAsyncJobs();
366            this.localDestinationSemaphore = new Semaphore(this.maxAsyncJobs);
367        }
368
369        @Override
370        public ActiveMQDestination getDestination() {
371            return destination;
372        }
373
374        @Override
375        public ListenableFuture<Object> asyncAddQueueMessage(final ConnectionContext context, final Message message)
376                throws IOException {
377            if (isConcurrentStoreAndDispatchQueues()) {
378                StoreQueueTask result = new StoreQueueTask(this, context, message);
379                ListenableFuture<Object> future = result.getFuture();
380                message.getMessageId().setFutureOrSequenceLong(future);
381                message.setRecievedByDFBridge(true); // flag message as concurrentStoreAndDispatch
382                result.aquireLocks();
383                addQueueTask(this, result);
384                if (indexListener != null) {
385                    indexListener.onAdd(new IndexListener.MessageContext(context, message, null));
386                }
387                return future;
388            } else {
389                return super.asyncAddQueueMessage(context, message);
390            }
391        }
392
393        @Override
394        public void removeAsyncMessage(ConnectionContext context, MessageAck ack) throws IOException {
395            if (isConcurrentStoreAndDispatchQueues()) {
396                AsyncJobKey key = new AsyncJobKey(ack.getLastMessageId(), getDestination());
397                StoreQueueTask task = null;
398                synchronized (asyncTaskMap) {
399                    task = (StoreQueueTask) asyncTaskMap.get(key);
400                }
401                if (task != null) {
402                    if (ack.isInTransaction() || !task.cancel()) {
403                        try {
404                            task.future.get();
405                        } catch (InterruptedException e) {
406                            throw new InterruptedIOException(e.toString());
407                        } catch (Exception ignored) {
408                            LOG.debug("removeAsync: cannot cancel, waiting for add resulted in ex", ignored);
409                        }
410                        removeMessage(context, ack);
411                    } else {
412                        synchronized (asyncTaskMap) {
413                            asyncTaskMap.remove(key);
414                        }
415                    }
416                } else {
417                    removeMessage(context, ack);
418                }
419            } else {
420                removeMessage(context, ack);
421            }
422        }
423
424        @Override
425        public void addMessage(final ConnectionContext context, final Message message) throws IOException {
426            final KahaAddMessageCommand command = new KahaAddMessageCommand();
427            command.setDestination(dest);
428            command.setMessageId(message.getMessageId().toProducerKey());
429            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(message.getTransactionId())));
430            command.setPriority(message.getPriority());
431            command.setPrioritySupported(isPrioritizedMessages());
432            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
433            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
434            store(command, isEnableJournalDiskSyncs() && message.isResponseRequired(), new IndexAware() {
435                // sync add? (for async, future present from getFutureOrSequenceLong)
436                Object possibleFuture = message.getMessageId().getFutureOrSequenceLong();
437
438                @Override
439                public void sequenceAssignedWithIndexLocked(final long sequence) {
440                    message.getMessageId().setFutureOrSequenceLong(sequence);
441                    if (indexListener != null) {
442                        if (possibleFuture == null) {
443                            trackPendingAdd(dest, sequence);
444                            indexListener.onAdd(new IndexListener.MessageContext(context, message, new Runnable() {
445                                @Override
446                                public void run() {
447                                    trackPendingAddComplete(dest, sequence);
448                                }
449                            }));
450                        }
451                    }
452                }
453            }, null);
454        }
455
456        @Override
457        public void updateMessage(Message message) throws IOException {
458            if (LOG.isTraceEnabled()) {
459                LOG.trace("updating: " + message.getMessageId() + " with deliveryCount: " + message.getRedeliveryCounter());
460            }
461            KahaUpdateMessageCommand updateMessageCommand = new KahaUpdateMessageCommand();
462            KahaAddMessageCommand command = new KahaAddMessageCommand();
463            command.setDestination(dest);
464            command.setMessageId(message.getMessageId().toProducerKey());
465            command.setPriority(message.getPriority());
466            command.setPrioritySupported(prioritizedMessages);
467            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(message);
468            command.setMessage(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
469            updateMessageCommand.setMessage(command);
470            store(updateMessageCommand, isEnableJournalDiskSyncs(), null, null);
471        }
472
473        @Override
474        public void removeMessage(ConnectionContext context, MessageAck ack) throws IOException {
475            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
476            command.setDestination(dest);
477            command.setMessageId(ack.getLastMessageId().toProducerKey());
478            command.setTransactionInfo(TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())));
479
480            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
481            command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
482            store(command, isEnableJournalDiskSyncs() && ack.isResponseRequired(), null, null);
483        }
484
485        @Override
486        public void removeAllMessages(ConnectionContext context) throws IOException {
487            KahaRemoveDestinationCommand command = new KahaRemoveDestinationCommand();
488            command.setDestination(dest);
489            store(command, true, null, null);
490        }
491
492        @Override
493        public Message getMessage(MessageId identity) throws IOException {
494            final String key = identity.toProducerKey();
495
496            // Hopefully one day the page file supports concurrent read
497            // operations... but for now we must
498            // externally synchronize...
499            Location location;
500            indexLock.writeLock().lock();
501            try {
502                location = findMessageLocation(key, dest);
503            } finally {
504                indexLock.writeLock().unlock();
505            }
506            if (location == null) {
507                return null;
508            }
509
510            return loadMessage(location);
511        }
512
513        @Override
514        public boolean isEmpty() throws IOException {
515            indexLock.writeLock().lock();
516            try {
517                return pageFile.tx().execute(new Transaction.CallableClosure<Boolean, IOException>() {
518                    @Override
519                    public Boolean execute(Transaction tx) throws IOException {
520                        // Iterate through all index entries to get a count of
521                        // messages in the destination.
522                        StoredDestination sd = getStoredDestination(dest, tx);
523                        return sd.locationIndex.isEmpty(tx);
524                    }
525                });
526            } finally {
527                indexLock.writeLock().unlock();
528            }
529        }
530
531        @Override
532        public void recover(final MessageRecoveryListener listener) throws Exception {
533            // recovery may involve expiry which will modify
534            indexLock.writeLock().lock();
535            try {
536                pageFile.tx().execute(new Transaction.Closure<Exception>() {
537                    @Override
538                    public void execute(Transaction tx) throws Exception {
539                        StoredDestination sd = getStoredDestination(dest, tx);
540                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
541                        sd.orderIndex.resetCursorPosition();
542                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); listener.hasSpace() && iterator
543                                .hasNext(); ) {
544                            Entry<Long, MessageKeys> entry = iterator.next();
545                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
546                                continue;
547                            }
548                            Message msg = loadMessage(entry.getValue().location);
549                            listener.recoverMessage(msg);
550                        }
551                    }
552                });
553            } finally {
554                indexLock.writeLock().unlock();
555            }
556        }
557
558        @Override
559        public void recoverNextMessages(final int maxReturned, final MessageRecoveryListener listener) throws Exception {
560            indexLock.writeLock().lock();
561            try {
562                pageFile.tx().execute(new Transaction.Closure<Exception>() {
563                    @Override
564                    public void execute(Transaction tx) throws Exception {
565                        StoredDestination sd = getStoredDestination(dest, tx);
566                        Entry<Long, MessageKeys> entry = null;
567                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
568                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator.hasNext(); ) {
569                            entry = iterator.next();
570                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
571                                continue;
572                            }
573                            Message msg = loadMessage(entry.getValue().location);
574                            msg.getMessageId().setFutureOrSequenceLong(entry.getKey());
575                            listener.recoverMessage(msg);
576                            counter++;
577                            if (counter >= maxReturned) {
578                                break;
579                            }
580                        }
581                        sd.orderIndex.stoppedIterating();
582                    }
583                });
584            } finally {
585                indexLock.writeLock().unlock();
586            }
587        }
588
589        protected int recoverRolledBackAcks(StoredDestination sd, Transaction tx, int maxReturned, MessageRecoveryListener listener) throws Exception {
590            int counter = 0;
591            String id;
592            for (Iterator<String> iterator = rolledBackAcks.iterator(); iterator.hasNext(); ) {
593                id = iterator.next();
594                iterator.remove();
595                Long sequence = sd.messageIdIndex.get(tx, id);
596                if (sequence != null) {
597                    if (sd.orderIndex.alreadyDispatched(sequence)) {
598                        listener.recoverMessage(loadMessage(sd.orderIndex.get(tx, sequence).location));
599                        counter++;
600                        if (counter >= maxReturned) {
601                            break;
602                        }
603                    } else {
604                        LOG.info("rolledback ack message {} with seq {} will be picked up in future batch {}", id, sequence, sd.orderIndex.cursor);
605                    }
606                } else {
607                    LOG.warn("Failed to locate rolled back ack message {} in {}", id, sd);
608                }
609            }
610            return counter;
611        }
612
613
614        @Override
615        public void resetBatching() {
616            if (pageFile.isLoaded()) {
617                indexLock.writeLock().lock();
618                try {
619                    pageFile.tx().execute(new Transaction.Closure<Exception>() {
620                        @Override
621                        public void execute(Transaction tx) throws Exception {
622                            StoredDestination sd = getExistingStoredDestination(dest, tx);
623                            if (sd != null) {
624                                sd.orderIndex.resetCursorPosition();}
625                            }
626                        });
627                } catch (Exception e) {
628                    LOG.error("Failed to reset batching",e);
629                } finally {
630                    indexLock.writeLock().unlock();
631                }
632            }
633        }
634
635        @Override
636        public void setBatch(final MessageId identity) throws IOException {
637            indexLock.writeLock().lock();
638            try {
639                pageFile.tx().execute(new Transaction.Closure<IOException>() {
640                    @Override
641                    public void execute(Transaction tx) throws IOException {
642                        StoredDestination sd = getStoredDestination(dest, tx);
643                        Long location = (Long) identity.getFutureOrSequenceLong();
644                        Long pending = sd.orderIndex.minPendingAdd();
645                        if (pending != null) {
646                            location = Math.min(location, pending-1);
647                        }
648                        sd.orderIndex.setBatch(tx, location);
649                    }
650                });
651            } finally {
652                indexLock.writeLock().unlock();
653            }
654        }
655
656        @Override
657        public void setMemoryUsage(MemoryUsage memoryUsage) {
658        }
659        @Override
660        public void start() throws Exception {
661            super.start();
662        }
663        @Override
664        public void stop() throws Exception {
665            super.stop();
666        }
667
668        protected void lockAsyncJobQueue() {
669            try {
670                if (!this.localDestinationSemaphore.tryAcquire(this.maxAsyncJobs, 60, TimeUnit.SECONDS)) {
671                    throw new TimeoutException(this +" timeout waiting for localDestSem:" + this.localDestinationSemaphore);
672                }
673            } catch (Exception e) {
674                LOG.error("Failed to lock async jobs for " + this.destination, e);
675            }
676        }
677
678        protected void unlockAsyncJobQueue() {
679            this.localDestinationSemaphore.release(this.maxAsyncJobs);
680        }
681
682        protected void acquireLocalAsyncLock() {
683            try {
684                this.localDestinationSemaphore.acquire();
685            } catch (InterruptedException e) {
686                LOG.error("Failed to aquire async lock for " + this.destination, e);
687            }
688        }
689
690        protected void releaseLocalAsyncLock() {
691            this.localDestinationSemaphore.release();
692        }
693
694        @Override
695        public String toString(){
696            return "permits:" + this.localDestinationSemaphore.availablePermits() + ",sd=" + storedDestinations.get(key(dest));
697        }
698
699        @Override
700        protected void recoverMessageStoreStatistics() throws IOException {
701            try {
702                MessageStoreStatistics recoveredStatistics;
703                lockAsyncJobQueue();
704                indexLock.writeLock().lock();
705                try {
706                    recoveredStatistics = pageFile.tx().execute(new Transaction.CallableClosure<MessageStoreStatistics, IOException>() {
707                        @Override
708                        public MessageStoreStatistics execute(Transaction tx) throws IOException {
709                            MessageStoreStatistics statistics = new MessageStoreStatistics();
710
711                            // Iterate through all index entries to get the size of each message
712                            StoredDestination sd = getStoredDestination(dest, tx);
713                            for (Iterator<Entry<Location, Long>> iterator = sd.locationIndex.iterator(tx); iterator.hasNext();) {
714                                int locationSize = iterator.next().getKey().getSize();
715                                statistics.getMessageCount().increment();
716                                statistics.getMessageSize().addSize(locationSize > 0 ? locationSize : 0);
717                            }
718                           return statistics;
719                        }
720                    });
721                    getMessageStoreStatistics().getMessageCount().setCount(recoveredStatistics.getMessageCount().getCount());
722                    getMessageStoreStatistics().getMessageSize().setTotalSize(recoveredStatistics.getMessageSize().getTotalSize());
723                } finally {
724                    indexLock.writeLock().unlock();
725                }
726            } finally {
727                unlockAsyncJobQueue();
728            }
729        }
730    }
731
732    class KahaDBTopicMessageStore extends KahaDBMessageStore implements TopicMessageStore {
733        private final AtomicInteger subscriptionCount = new AtomicInteger();
734        public KahaDBTopicMessageStore(ActiveMQTopic destination) throws IOException {
735            super(destination);
736            this.subscriptionCount.set(getAllSubscriptions().length);
737            if (isConcurrentStoreAndDispatchTopics()) {
738                asyncTopicMaps.add(asyncTaskMap);
739            }
740        }
741
742        @Override
743        public ListenableFuture<Object> asyncAddTopicMessage(final ConnectionContext context, final Message message)
744                throws IOException {
745            if (isConcurrentStoreAndDispatchTopics()) {
746                StoreTopicTask result = new StoreTopicTask(this, context, message, subscriptionCount.get());
747                result.aquireLocks();
748                addTopicTask(this, result);
749                return result.getFuture();
750            } else {
751                return super.asyncAddTopicMessage(context, message);
752            }
753        }
754
755        @Override
756        public void acknowledge(ConnectionContext context, String clientId, String subscriptionName,
757                                MessageId messageId, MessageAck ack) throws IOException {
758            String subscriptionKey = subscriptionKey(clientId, subscriptionName).toString();
759            if (isConcurrentStoreAndDispatchTopics()) {
760                AsyncJobKey key = new AsyncJobKey(messageId, getDestination());
761                StoreTopicTask task = null;
762                synchronized (asyncTaskMap) {
763                    task = (StoreTopicTask) asyncTaskMap.get(key);
764                }
765                if (task != null) {
766                    if (task.addSubscriptionKey(subscriptionKey)) {
767                        removeTopicTask(this, messageId);
768                        if (task.cancel()) {
769                            synchronized (asyncTaskMap) {
770                                asyncTaskMap.remove(key);
771                            }
772                        }
773                    }
774                } else {
775                    doAcknowledge(context, subscriptionKey, messageId, ack);
776                }
777            } else {
778                doAcknowledge(context, subscriptionKey, messageId, ack);
779            }
780        }
781
782        protected void doAcknowledge(ConnectionContext context, String subscriptionKey, MessageId messageId, MessageAck ack)
783                throws IOException {
784            KahaRemoveMessageCommand command = new KahaRemoveMessageCommand();
785            command.setDestination(dest);
786            command.setSubscriptionKey(subscriptionKey);
787            command.setMessageId(messageId.toProducerKey());
788            command.setTransactionInfo(ack != null ? TransactionIdConversion.convert(transactionIdTransformer.transform(ack.getTransactionId())) : null);
789            if (ack != null && ack.isUnmatchedAck()) {
790                command.setAck(UNMATCHED);
791            } else {
792                org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(ack);
793                command.setAck(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
794            }
795            store(command, false, null, null);
796        }
797
798        @Override
799        public void addSubscription(SubscriptionInfo subscriptionInfo, boolean retroactive) throws IOException {
800            String subscriptionKey = subscriptionKey(subscriptionInfo.getClientId(), subscriptionInfo
801                    .getSubscriptionName());
802            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
803            command.setDestination(dest);
804            command.setSubscriptionKey(subscriptionKey.toString());
805            command.setRetroactive(retroactive);
806            org.apache.activemq.util.ByteSequence packet = wireFormat.marshal(subscriptionInfo);
807            command.setSubscriptionInfo(new Buffer(packet.getData(), packet.getOffset(), packet.getLength()));
808            store(command, isEnableJournalDiskSyncs() && true, null, null);
809            this.subscriptionCount.incrementAndGet();
810        }
811
812        @Override
813        public void deleteSubscription(String clientId, String subscriptionName) throws IOException {
814            KahaSubscriptionCommand command = new KahaSubscriptionCommand();
815            command.setDestination(dest);
816            command.setSubscriptionKey(subscriptionKey(clientId, subscriptionName).toString());
817            store(command, isEnableJournalDiskSyncs() && true, null, null);
818            this.subscriptionCount.decrementAndGet();
819        }
820
821        @Override
822        public SubscriptionInfo[] getAllSubscriptions() throws IOException {
823
824            final ArrayList<SubscriptionInfo> subscriptions = new ArrayList<SubscriptionInfo>();
825            indexLock.writeLock().lock();
826            try {
827                pageFile.tx().execute(new Transaction.Closure<IOException>() {
828                    @Override
829                    public void execute(Transaction tx) throws IOException {
830                        StoredDestination sd = getStoredDestination(dest, tx);
831                        for (Iterator<Entry<String, KahaSubscriptionCommand>> iterator = sd.subscriptions.iterator(tx); iterator
832                                .hasNext();) {
833                            Entry<String, KahaSubscriptionCommand> entry = iterator.next();
834                            SubscriptionInfo info = (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(entry
835                                    .getValue().getSubscriptionInfo().newInput()));
836                            subscriptions.add(info);
837
838                        }
839                    }
840                });
841            } finally {
842                indexLock.writeLock().unlock();
843            }
844
845            SubscriptionInfo[] rc = new SubscriptionInfo[subscriptions.size()];
846            subscriptions.toArray(rc);
847            return rc;
848        }
849
850        @Override
851        public SubscriptionInfo lookupSubscription(String clientId, String subscriptionName) throws IOException {
852            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
853            indexLock.writeLock().lock();
854            try {
855                return pageFile.tx().execute(new Transaction.CallableClosure<SubscriptionInfo, IOException>() {
856                    @Override
857                    public SubscriptionInfo execute(Transaction tx) throws IOException {
858                        StoredDestination sd = getStoredDestination(dest, tx);
859                        KahaSubscriptionCommand command = sd.subscriptions.get(tx, subscriptionKey);
860                        if (command == null) {
861                            return null;
862                        }
863                        return (SubscriptionInfo) wireFormat.unmarshal(new DataInputStream(command
864                                .getSubscriptionInfo().newInput()));
865                    }
866                });
867            } finally {
868                indexLock.writeLock().unlock();
869            }
870        }
871
872        @Override
873        public int getMessageCount(String clientId, String subscriptionName) throws IOException {
874            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
875            indexLock.writeLock().lock();
876            try {
877                return pageFile.tx().execute(new Transaction.CallableClosure<Integer, IOException>() {
878                    @Override
879                    public Integer execute(Transaction tx) throws IOException {
880                        StoredDestination sd = getStoredDestination(dest, tx);
881                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
882                        if (cursorPos == null) {
883                            // The subscription might not exist.
884                            return 0;
885                        }
886
887                        return (int) getStoredMessageCount(tx, sd, subscriptionKey);
888                    }
889                });
890            } finally {
891                indexLock.writeLock().unlock();
892            }
893        }
894
895        @Override
896        public void recoverSubscription(String clientId, String subscriptionName, final MessageRecoveryListener listener)
897                throws Exception {
898            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
899            @SuppressWarnings("unused")
900            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
901            indexLock.writeLock().lock();
902            try {
903                pageFile.tx().execute(new Transaction.Closure<Exception>() {
904                    @Override
905                    public void execute(Transaction tx) throws Exception {
906                        StoredDestination sd = getStoredDestination(dest, tx);
907                        LastAck cursorPos = getLastAck(tx, sd, subscriptionKey);
908                        sd.orderIndex.setBatch(tx, cursorPos);
909                        recoverRolledBackAcks(sd, tx, Integer.MAX_VALUE, listener);
910                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx); iterator
911                                .hasNext();) {
912                            Entry<Long, MessageKeys> entry = iterator.next();
913                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
914                                continue;
915                            }
916                            listener.recoverMessage(loadMessage(entry.getValue().location));
917                        }
918                        sd.orderIndex.resetCursorPosition();
919                    }
920                });
921            } finally {
922                indexLock.writeLock().unlock();
923            }
924        }
925
926        @Override
927        public void recoverNextMessages(String clientId, String subscriptionName, final int maxReturned,
928                final MessageRecoveryListener listener) throws Exception {
929            final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
930            @SuppressWarnings("unused")
931            final SubscriptionInfo info = lookupSubscription(clientId, subscriptionName);
932            indexLock.writeLock().lock();
933            try {
934                pageFile.tx().execute(new Transaction.Closure<Exception>() {
935                    @Override
936                    public void execute(Transaction tx) throws Exception {
937                        StoredDestination sd = getStoredDestination(dest, tx);
938                        sd.orderIndex.resetCursorPosition();
939                        MessageOrderCursor moc = sd.subscriptionCursors.get(subscriptionKey);
940                        if (moc == null) {
941                            LastAck pos = getLastAck(tx, sd, subscriptionKey);
942                            if (pos == null) {
943                                // sub deleted
944                                return;
945                            }
946                            sd.orderIndex.setBatch(tx, pos);
947                            moc = sd.orderIndex.cursor;
948                        } else {
949                            sd.orderIndex.cursor.sync(moc);
950                        }
951
952                        Entry<Long, MessageKeys> entry = null;
953                        int counter = recoverRolledBackAcks(sd, tx, maxReturned, listener);
954                        for (Iterator<Entry<Long, MessageKeys>> iterator = sd.orderIndex.iterator(tx, moc); iterator
955                                .hasNext();) {
956                            entry = iterator.next();
957                            if (ackedAndPrepared.contains(entry.getValue().messageId)) {
958                                continue;
959                            }
960                            if (listener.recoverMessage(loadMessage(entry.getValue().location))) {
961                                counter++;
962                            }
963                            if (counter >= maxReturned || listener.hasSpace() == false) {
964                                break;
965                            }
966                        }
967                        sd.orderIndex.stoppedIterating();
968                        if (entry != null) {
969                            MessageOrderCursor copy = sd.orderIndex.cursor.copy();
970                            sd.subscriptionCursors.put(subscriptionKey, copy);
971                        }
972                    }
973                });
974            } finally {
975                indexLock.writeLock().unlock();
976            }
977        }
978
979        @Override
980        public void resetBatching(String clientId, String subscriptionName) {
981            try {
982                final String subscriptionKey = subscriptionKey(clientId, subscriptionName);
983                indexLock.writeLock().lock();
984                try {
985                    pageFile.tx().execute(new Transaction.Closure<IOException>() {
986                        @Override
987                        public void execute(Transaction tx) throws IOException {
988                            StoredDestination sd = getStoredDestination(dest, tx);
989                            sd.subscriptionCursors.remove(subscriptionKey);
990                        }
991                    });
992                }finally {
993                    indexLock.writeLock().unlock();
994                }
995            } catch (IOException e) {
996                throw new RuntimeException(e);
997            }
998        }
999    }
1000
1001    String subscriptionKey(String clientId, String subscriptionName) {
1002        return clientId + ":" + subscriptionName;
1003    }
1004
1005    @Override
1006    public MessageStore createQueueMessageStore(ActiveMQQueue destination) throws IOException {
1007        String key = key(convert(destination));
1008        MessageStore store = storeCache.get(key(convert(destination)));
1009        if (store == null) {
1010            final MessageStore queueStore = this.transactionStore.proxy(new KahaDBMessageStore(destination));
1011            store = storeCache.putIfAbsent(key, queueStore);
1012            if (store == null) {
1013                store = queueStore;
1014            }
1015        }
1016
1017        return store;
1018    }
1019
1020    @Override
1021    public TopicMessageStore createTopicMessageStore(ActiveMQTopic destination) throws IOException {
1022        String key = key(convert(destination));
1023        MessageStore store = storeCache.get(key(convert(destination)));
1024        if (store == null) {
1025            final TopicMessageStore topicStore = this.transactionStore.proxy(new KahaDBTopicMessageStore(destination));
1026            store = storeCache.putIfAbsent(key, topicStore);
1027            if (store == null) {
1028                store = topicStore;
1029            }
1030        }
1031
1032        return (TopicMessageStore) store;
1033    }
1034
1035    /**
1036     * Cleanup method to remove any state associated with the given destination.
1037     * This method does not stop the message store (it might not be cached).
1038     *
1039     * @param destination
1040     *            Destination to forget
1041     */
1042    @Override
1043    public void removeQueueMessageStore(ActiveMQQueue destination) {
1044    }
1045
1046    /**
1047     * Cleanup method to remove any state associated with the given destination
1048     * This method does not stop the message store (it might not be cached).
1049     *
1050     * @param destination
1051     *            Destination to forget
1052     */
1053    @Override
1054    public void removeTopicMessageStore(ActiveMQTopic destination) {
1055    }
1056
1057    @Override
1058    public void deleteAllMessages() throws IOException {
1059        deleteAllMessages = true;
1060    }
1061
1062    @Override
1063    public Set<ActiveMQDestination> getDestinations() {
1064        try {
1065            final HashSet<ActiveMQDestination> rc = new HashSet<ActiveMQDestination>();
1066            indexLock.writeLock().lock();
1067            try {
1068                pageFile.tx().execute(new Transaction.Closure<IOException>() {
1069                    @Override
1070                    public void execute(Transaction tx) throws IOException {
1071                        for (Iterator<Entry<String, StoredDestination>> iterator = metadata.destinations.iterator(tx); iterator
1072                                .hasNext();) {
1073                            Entry<String, StoredDestination> entry = iterator.next();
1074                            //Removing isEmpty topic check - see AMQ-5875
1075                            rc.add(convert(entry.getKey()));
1076                        }
1077                    }
1078                });
1079            }finally {
1080                indexLock.writeLock().unlock();
1081            }
1082            return rc;
1083        } catch (IOException e) {
1084            throw new RuntimeException(e);
1085        }
1086    }
1087
1088    @Override
1089    public long getLastMessageBrokerSequenceId() throws IOException {
1090        return 0;
1091    }
1092
1093    @Override
1094    public long getLastProducerSequenceId(ProducerId id) {
1095        indexLock.readLock().lock();
1096        try {
1097            return metadata.producerSequenceIdTracker.getLastSeqId(id);
1098        } finally {
1099            indexLock.readLock().unlock();
1100        }
1101    }
1102
1103    @Override
1104    public long size() {
1105        try {
1106            return journalSize.get() + getPageFile().getDiskSize();
1107        } catch (IOException e) {
1108            throw new RuntimeException(e);
1109        }
1110    }
1111
1112    @Override
1113    public void beginTransaction(ConnectionContext context) throws IOException {
1114        throw new IOException("Not yet implemented.");
1115    }
1116    @Override
1117    public void commitTransaction(ConnectionContext context) throws IOException {
1118        throw new IOException("Not yet implemented.");
1119    }
1120    @Override
1121    public void rollbackTransaction(ConnectionContext context) throws IOException {
1122        throw new IOException("Not yet implemented.");
1123    }
1124
1125    @Override
1126    public void checkpoint(boolean sync) throws IOException {
1127        super.checkpointCleanup(sync);
1128    }
1129
1130    // /////////////////////////////////////////////////////////////////
1131    // Internal helper methods.
1132    // /////////////////////////////////////////////////////////////////
1133
1134    /**
1135     * @param location
1136     * @return
1137     * @throws IOException
1138     */
1139    Message loadMessage(Location location) throws IOException {
1140        JournalCommand<?> command = load(location);
1141        KahaAddMessageCommand addMessage = null;
1142        switch (command.type()) {
1143            case KAHA_UPDATE_MESSAGE_COMMAND:
1144                addMessage = ((KahaUpdateMessageCommand)command).getMessage();
1145                break;
1146            default:
1147                addMessage = (KahaAddMessageCommand) command;
1148        }
1149        Message msg = (Message) wireFormat.unmarshal(new DataInputStream(addMessage.getMessage().newInput()));
1150        return msg;
1151    }
1152
1153    // /////////////////////////////////////////////////////////////////
1154    // Internal conversion methods.
1155    // /////////////////////////////////////////////////////////////////
1156
1157    KahaLocation convert(Location location) {
1158        KahaLocation rc = new KahaLocation();
1159        rc.setLogId(location.getDataFileId());
1160        rc.setOffset(location.getOffset());
1161        return rc;
1162    }
1163
1164    KahaDestination convert(ActiveMQDestination dest) {
1165        KahaDestination rc = new KahaDestination();
1166        rc.setName(dest.getPhysicalName());
1167        switch (dest.getDestinationType()) {
1168        case ActiveMQDestination.QUEUE_TYPE:
1169            rc.setType(DestinationType.QUEUE);
1170            return rc;
1171        case ActiveMQDestination.TOPIC_TYPE:
1172            rc.setType(DestinationType.TOPIC);
1173            return rc;
1174        case ActiveMQDestination.TEMP_QUEUE_TYPE:
1175            rc.setType(DestinationType.TEMP_QUEUE);
1176            return rc;
1177        case ActiveMQDestination.TEMP_TOPIC_TYPE:
1178            rc.setType(DestinationType.TEMP_TOPIC);
1179            return rc;
1180        default:
1181            return null;
1182        }
1183    }
1184
1185    ActiveMQDestination convert(String dest) {
1186        int p = dest.indexOf(":");
1187        if (p < 0) {
1188            throw new IllegalArgumentException("Not in the valid destination format");
1189        }
1190        int type = Integer.parseInt(dest.substring(0, p));
1191        String name = dest.substring(p + 1);
1192        return convert(type, name);
1193    }
1194
1195    private ActiveMQDestination convert(KahaDestination commandDestination) {
1196        return convert(commandDestination.getType().getNumber(), commandDestination.getName());
1197    }
1198
1199    private ActiveMQDestination convert(int type, String name) {
1200        switch (KahaDestination.DestinationType.valueOf(type)) {
1201        case QUEUE:
1202            return new ActiveMQQueue(name);
1203        case TOPIC:
1204            return new ActiveMQTopic(name);
1205        case TEMP_QUEUE:
1206            return new ActiveMQTempQueue(name);
1207        case TEMP_TOPIC:
1208            return new ActiveMQTempTopic(name);
1209        default:
1210            throw new IllegalArgumentException("Not in the valid destination format");
1211        }
1212    }
1213
1214    public TransactionIdTransformer getTransactionIdTransformer() {
1215        return transactionIdTransformer;
1216    }
1217
1218    public void setTransactionIdTransformer(TransactionIdTransformer transactionIdTransformer) {
1219        this.transactionIdTransformer = transactionIdTransformer;
1220    }
1221
1222    static class AsyncJobKey {
1223        MessageId id;
1224        ActiveMQDestination destination;
1225
1226        AsyncJobKey(MessageId id, ActiveMQDestination destination) {
1227            this.id = id;
1228            this.destination = destination;
1229        }
1230
1231        @Override
1232        public boolean equals(Object obj) {
1233            if (obj == this) {
1234                return true;
1235            }
1236            return obj instanceof AsyncJobKey && id.equals(((AsyncJobKey) obj).id)
1237                    && destination.equals(((AsyncJobKey) obj).destination);
1238        }
1239
1240        @Override
1241        public int hashCode() {
1242            return id.hashCode() + destination.hashCode();
1243        }
1244
1245        @Override
1246        public String toString() {
1247            return destination.getPhysicalName() + "-" + id;
1248        }
1249    }
1250
1251    public interface StoreTask {
1252        public boolean cancel();
1253
1254        public void aquireLocks();
1255
1256        public void releaseLocks();
1257    }
1258
1259    class StoreQueueTask implements Runnable, StoreTask {
1260        protected final Message message;
1261        protected final ConnectionContext context;
1262        protected final KahaDBMessageStore store;
1263        protected final InnerFutureTask future;
1264        protected final AtomicBoolean done = new AtomicBoolean();
1265        protected final AtomicBoolean locked = new AtomicBoolean();
1266
1267        public StoreQueueTask(KahaDBMessageStore store, ConnectionContext context, Message message) {
1268            this.store = store;
1269            this.context = context;
1270            this.message = message;
1271            this.future = new InnerFutureTask(this);
1272        }
1273
1274        public ListenableFuture<Object> getFuture() {
1275            return this.future;
1276        }
1277
1278        @Override
1279        public boolean cancel() {
1280            if (this.done.compareAndSet(false, true)) {
1281                return this.future.cancel(false);
1282            }
1283            return false;
1284        }
1285
1286        @Override
1287        public void aquireLocks() {
1288            if (this.locked.compareAndSet(false, true)) {
1289                try {
1290                    globalQueueSemaphore.acquire();
1291                    store.acquireLocalAsyncLock();
1292                    message.incrementReferenceCount();
1293                } catch (InterruptedException e) {
1294                    LOG.warn("Failed to aquire lock", e);
1295                }
1296            }
1297
1298        }
1299
1300        @Override
1301        public void releaseLocks() {
1302            if (this.locked.compareAndSet(true, false)) {
1303                store.releaseLocalAsyncLock();
1304                globalQueueSemaphore.release();
1305                message.decrementReferenceCount();
1306            }
1307        }
1308
1309        @Override
1310        public void run() {
1311            this.store.doneTasks++;
1312            try {
1313                if (this.done.compareAndSet(false, true)) {
1314                    this.store.addMessage(context, message);
1315                    removeQueueTask(this.store, this.message.getMessageId());
1316                    this.future.complete();
1317                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1318                    System.err.println(this.store.dest.getName() + " cancelled: "
1319                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1320                    this.store.canceledTasks = this.store.doneTasks = 0;
1321                }
1322            } catch (Exception e) {
1323                this.future.setException(e);
1324            }
1325        }
1326
1327        protected Message getMessage() {
1328            return this.message;
1329        }
1330
1331        private class InnerFutureTask extends FutureTask<Object> implements ListenableFuture<Object>  {
1332
1333            private Runnable listener;
1334            public InnerFutureTask(Runnable runnable) {
1335                super(runnable, null);
1336
1337            }
1338
1339            public void setException(final Exception e) {
1340                super.setException(e);
1341            }
1342
1343            public void complete() {
1344                super.set(null);
1345            }
1346
1347            @Override
1348            public void done() {
1349                fireListener();
1350            }
1351
1352            @Override
1353            public void addListener(Runnable listener) {
1354                this.listener = listener;
1355                if (isDone()) {
1356                    fireListener();
1357                }
1358            }
1359
1360            private void fireListener() {
1361                if (listener != null) {
1362                    try {
1363                        listener.run();
1364                    } catch (Exception ignored) {
1365                        LOG.warn("Unexpected exception from future {} listener callback {}", this, listener, ignored);
1366                    }
1367                }
1368            }
1369        }
1370    }
1371
1372    class StoreTopicTask extends StoreQueueTask {
1373        private final int subscriptionCount;
1374        private final List<String> subscriptionKeys = new ArrayList<String>(1);
1375        private final KahaDBTopicMessageStore topicStore;
1376        public StoreTopicTask(KahaDBTopicMessageStore store, ConnectionContext context, Message message,
1377                int subscriptionCount) {
1378            super(store, context, message);
1379            this.topicStore = store;
1380            this.subscriptionCount = subscriptionCount;
1381
1382        }
1383
1384        @Override
1385        public void aquireLocks() {
1386            if (this.locked.compareAndSet(false, true)) {
1387                try {
1388                    globalTopicSemaphore.acquire();
1389                    store.acquireLocalAsyncLock();
1390                    message.incrementReferenceCount();
1391                } catch (InterruptedException e) {
1392                    LOG.warn("Failed to aquire lock", e);
1393                }
1394            }
1395        }
1396
1397        @Override
1398        public void releaseLocks() {
1399            if (this.locked.compareAndSet(true, false)) {
1400                message.decrementReferenceCount();
1401                store.releaseLocalAsyncLock();
1402                globalTopicSemaphore.release();
1403            }
1404        }
1405
1406        /**
1407         * add a key
1408         *
1409         * @param key
1410         * @return true if all acknowledgements received
1411         */
1412        public boolean addSubscriptionKey(String key) {
1413            synchronized (this.subscriptionKeys) {
1414                this.subscriptionKeys.add(key);
1415            }
1416            return this.subscriptionKeys.size() >= this.subscriptionCount;
1417        }
1418
1419        @Override
1420        public void run() {
1421            this.store.doneTasks++;
1422            try {
1423                if (this.done.compareAndSet(false, true)) {
1424                    this.topicStore.addMessage(context, message);
1425                    // apply any acks we have
1426                    synchronized (this.subscriptionKeys) {
1427                        for (String key : this.subscriptionKeys) {
1428                            this.topicStore.doAcknowledge(context, key, this.message.getMessageId(), null);
1429
1430                        }
1431                    }
1432                    removeTopicTask(this.topicStore, this.message.getMessageId());
1433                    this.future.complete();
1434                } else if (cancelledTaskModMetric > 0 && this.store.canceledTasks++ % cancelledTaskModMetric == 0) {
1435                    System.err.println(this.store.dest.getName() + " cancelled: "
1436                            + (this.store.canceledTasks / this.store.doneTasks) * 100);
1437                    this.store.canceledTasks = this.store.doneTasks = 0;
1438                }
1439            } catch (Exception e) {
1440                this.future.setException(e);
1441            }
1442        }
1443    }
1444
1445    public class StoreTaskExecutor extends ThreadPoolExecutor {
1446
1447        public StoreTaskExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit timeUnit, BlockingQueue<Runnable> queue, ThreadFactory threadFactory) {
1448            super(corePoolSize, maximumPoolSize, keepAliveTime, timeUnit, queue, threadFactory);
1449        }
1450
1451        @Override
1452        protected void afterExecute(Runnable runnable, Throwable throwable) {
1453            super.afterExecute(runnable, throwable);
1454
1455            if (runnable instanceof StoreTask) {
1456               ((StoreTask)runnable).releaseLocks();
1457            }
1458        }
1459    }
1460
1461    @Override
1462    public JobSchedulerStore createJobSchedulerStore() throws IOException, UnsupportedOperationException {
1463        return new JobSchedulerStoreImpl();
1464    }
1465}