001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.store.kahadb;
018
019import java.io.File;
020import java.io.FileFilter;
021import java.io.IOException;
022import java.nio.charset.Charset;
023import java.util.Collections;
024import java.util.HashMap;
025import java.util.HashSet;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import java.util.concurrent.CopyOnWriteArrayList;
030
031import javax.transaction.xa.Xid;
032
033import org.apache.activemq.broker.BrokerService;
034import org.apache.activemq.broker.BrokerServiceAware;
035import org.apache.activemq.broker.ConnectionContext;
036import org.apache.activemq.broker.Lockable;
037import org.apache.activemq.broker.LockableServiceSupport;
038import org.apache.activemq.broker.Locker;
039import org.apache.activemq.broker.scheduler.JobSchedulerStore;
040import org.apache.activemq.command.ActiveMQDestination;
041import org.apache.activemq.command.ActiveMQQueue;
042import org.apache.activemq.command.ActiveMQTopic;
043import org.apache.activemq.command.LocalTransactionId;
044import org.apache.activemq.command.ProducerId;
045import org.apache.activemq.command.TransactionId;
046import org.apache.activemq.command.XATransactionId;
047import org.apache.activemq.filter.AnyDestination;
048import org.apache.activemq.filter.DestinationMap;
049import org.apache.activemq.filter.DestinationMapEntry;
050import org.apache.activemq.store.MessageStore;
051import org.apache.activemq.store.NoLocalSubscriptionAware;
052import org.apache.activemq.store.PersistenceAdapter;
053import org.apache.activemq.store.SharedFileLocker;
054import org.apache.activemq.store.TopicMessageStore;
055import org.apache.activemq.store.TransactionIdTransformer;
056import org.apache.activemq.store.TransactionIdTransformerAware;
057import org.apache.activemq.store.TransactionStore;
058import org.apache.activemq.store.kahadb.scheduler.JobSchedulerStoreImpl;
059import org.apache.activemq.usage.SystemUsage;
060import org.apache.activemq.util.IOExceptionSupport;
061import org.apache.activemq.util.IOHelper;
062import org.apache.activemq.util.IntrospectionSupport;
063import org.apache.activemq.util.ServiceStopper;
064import org.slf4j.Logger;
065import org.slf4j.LoggerFactory;
066
067import static org.apache.activemq.store.kahadb.MessageDatabase.DEFAULT_DIRECTORY;
068
069/**
070 * An implementation of {@link org.apache.activemq.store.PersistenceAdapter}  that supports
071 * distribution of destinations across multiple kahaDB persistence adapters
072 *
073 * @org.apache.xbean.XBean element="mKahaDB"
074 */
075public class MultiKahaDBPersistenceAdapter extends LockableServiceSupport implements PersistenceAdapter,
076    BrokerServiceAware, NoLocalSubscriptionAware {
077
078    static final Logger LOG = LoggerFactory.getLogger(MultiKahaDBPersistenceAdapter.class);
079
080    final static ActiveMQDestination matchAll = new AnyDestination(new ActiveMQDestination[]{new ActiveMQQueue(">"), new ActiveMQTopic(">")});
081    final int LOCAL_FORMAT_ID_MAGIC = Integer.valueOf(System.getProperty("org.apache.activemq.store.kahadb.MultiKahaDBTransactionStore.localXaFormatId", "61616"));
082
083    final class DelegateDestinationMap extends DestinationMap {
084        @Override
085        public void setEntries(List<DestinationMapEntry>  entries) {
086            super.setEntries(entries);
087        }
088    };
089    final DelegateDestinationMap destinationMap = new DelegateDestinationMap();
090
091    List<PersistenceAdapter> adapters = new CopyOnWriteArrayList<PersistenceAdapter>();
092    private File directory = new File(IOHelper.getDefaultDataDirectory() + File.separator + "mKahaDB");
093
094    MultiKahaDBTransactionStore transactionStore = new MultiKahaDBTransactionStore(this);
095
096    // all local store transactions are XA, 2pc if more than one adapter involved
097    TransactionIdTransformer transactionIdTransformer = new TransactionIdTransformer() {
098        @Override
099        public TransactionId transform(TransactionId txid) {
100            if (txid == null) {
101                return null;
102            }
103            if (txid.isLocalTransaction()) {
104                final LocalTransactionId t = (LocalTransactionId) txid;
105                return new XATransactionId(new Xid() {
106                    @Override
107                    public int getFormatId() {
108                        return LOCAL_FORMAT_ID_MAGIC;
109                    }
110
111                    @Override
112                    public byte[] getGlobalTransactionId() {
113                        return t.getConnectionId().getValue().getBytes(Charset.forName("utf-8"));
114                    }
115
116                    @Override
117                    public byte[] getBranchQualifier() {
118                        return Long.toString(t.getValue()).getBytes(Charset.forName("utf-8"));
119                    }
120                });
121            } else {
122                return txid;
123            }
124        }
125    };
126
127    /**
128     * Sets the  FilteredKahaDBPersistenceAdapter entries
129     *
130     * @org.apache.xbean.ElementType class="org.apache.activemq.store.kahadb.FilteredKahaDBPersistenceAdapter"
131     */
132    @SuppressWarnings({ "rawtypes", "unchecked" })
133    public void setFilteredPersistenceAdapters(List entries) {
134        for (Object entry : entries) {
135            FilteredKahaDBPersistenceAdapter filteredAdapter = (FilteredKahaDBPersistenceAdapter) entry;
136            PersistenceAdapter adapter = filteredAdapter.getPersistenceAdapter();
137            if (filteredAdapter.getDestination() == null) {
138                filteredAdapter.setDestination(matchAll);
139            }
140
141            if (filteredAdapter.isPerDestination()) {
142                configureDirectory(adapter, null);
143                // per destination adapters will be created on demand or during recovery
144                continue;
145            } else {
146                configureDirectory(adapter, nameFromDestinationFilter(filteredAdapter.getDestination()));
147            }
148
149            configureAdapter(adapter);
150            adapters.add(adapter);
151        }
152        destinationMap.setEntries(entries);
153    }
154
155    public static String nameFromDestinationFilter(ActiveMQDestination destination) {
156        if (destination.getQualifiedName().length() > IOHelper.getMaxFileNameLength()) {
157            LOG.warn("Destination name is longer than 'MaximumFileNameLength' system property, " +
158                     "potential problem with recovery can result from name truncation.");
159        }
160
161        return IOHelper.toFileSystemSafeName(destination.getQualifiedName());
162    }
163
164    public boolean isLocalXid(TransactionId xid) {
165        return xid instanceof XATransactionId &&
166                ((XATransactionId)xid).getFormatId() == LOCAL_FORMAT_ID_MAGIC;
167    }
168
169    @Override
170    public void beginTransaction(ConnectionContext context) throws IOException {
171        throw new IllegalStateException();
172    }
173
174    @Override
175    public void checkpoint(final boolean cleanup) throws IOException {
176        for (PersistenceAdapter persistenceAdapter : adapters) {
177            persistenceAdapter.checkpoint(cleanup);
178        }
179    }
180
181    @Override
182    public void commitTransaction(ConnectionContext context) throws IOException {
183        throw new IllegalStateException();
184    }
185
186    @Override
187    public MessageStore createQueueMessageStore(ActiveMQQueue destination) throws IOException {
188        PersistenceAdapter persistenceAdapter = getMatchingPersistenceAdapter(destination);
189        return transactionStore.proxy(persistenceAdapter.createTransactionStore(), persistenceAdapter.createQueueMessageStore(destination));
190    }
191
192    private PersistenceAdapter getMatchingPersistenceAdapter(ActiveMQDestination destination) throws IOException {
193        Object result = destinationMap.chooseValue(destination);
194        if (result == null) {
195            throw new RuntimeException("No matching persistence adapter configured for destination: " + destination + ", options:" + adapters);
196        }
197        FilteredKahaDBPersistenceAdapter filteredAdapter = (FilteredKahaDBPersistenceAdapter) result;
198        if (filteredAdapter.getDestination() == matchAll && filteredAdapter.isPerDestination()) {
199            filteredAdapter = addAdapter(filteredAdapter, destination);
200            if (LOG.isTraceEnabled()) {
201                LOG.trace("created per destination adapter for: " + destination  + ", " + result);
202            }
203        }
204        startAdapter(filteredAdapter.getPersistenceAdapter(), destination.getQualifiedName());
205        LOG.debug("destination {} matched persistence adapter {}", destination.getQualifiedName(), filteredAdapter.getPersistenceAdapter());
206        return filteredAdapter.getPersistenceAdapter();
207    }
208
209    private void startAdapter(PersistenceAdapter kahaDBPersistenceAdapter, String destination) {
210        try {
211            kahaDBPersistenceAdapter.start();
212        } catch (Exception e) {
213            RuntimeException detail = new RuntimeException("Failed to start per destination persistence adapter for destination: " + destination + ", options:" + adapters, e);
214            LOG.error(detail.toString(), e);
215            throw detail;
216        }
217    }
218
219    private void stopAdapter(PersistenceAdapter kahaDBPersistenceAdapter, String destination) {
220        try {
221            kahaDBPersistenceAdapter.stop();
222        } catch (Exception e) {
223            RuntimeException detail = new RuntimeException("Failed to stop per destination persistence adapter for destination: " + destination + ", options:" + adapters, e);
224            LOG.error(detail.toString(), e);
225            throw detail;
226        }
227    }
228
229    @Override
230    public TopicMessageStore createTopicMessageStore(ActiveMQTopic destination) throws IOException {
231        PersistenceAdapter persistenceAdapter = getMatchingPersistenceAdapter(destination);
232        return transactionStore.proxy(persistenceAdapter.createTransactionStore(), persistenceAdapter.createTopicMessageStore(destination));
233    }
234
235    @Override
236    public TransactionStore createTransactionStore() throws IOException {
237        return transactionStore;
238    }
239
240    @Override
241    public void deleteAllMessages() throws IOException {
242        for (PersistenceAdapter persistenceAdapter : adapters) {
243            persistenceAdapter.deleteAllMessages();
244        }
245        transactionStore.deleteAllMessages();
246        IOHelper.deleteChildren(getDirectory());
247        for (Object o : destinationMap.get(new AnyDestination(new ActiveMQDestination[]{new ActiveMQQueue(">"), new ActiveMQTopic(">")}))) {
248            if (o instanceof FilteredKahaDBPersistenceAdapter) {
249                FilteredKahaDBPersistenceAdapter filteredKahaDBPersistenceAdapter = (FilteredKahaDBPersistenceAdapter) o;
250                if (filteredKahaDBPersistenceAdapter.getPersistenceAdapter().getDirectory() != DEFAULT_DIRECTORY) {
251                    IOHelper.deleteChildren(filteredKahaDBPersistenceAdapter.getPersistenceAdapter().getDirectory());
252                }
253                if (filteredKahaDBPersistenceAdapter.getPersistenceAdapter() instanceof KahaDBPersistenceAdapter) {
254                    KahaDBPersistenceAdapter kahaDBPersistenceAdapter = (KahaDBPersistenceAdapter) filteredKahaDBPersistenceAdapter.getPersistenceAdapter();
255                    if (kahaDBPersistenceAdapter.getIndexDirectory() != null) {
256                        IOHelper.deleteChildren(kahaDBPersistenceAdapter.getIndexDirectory());
257                    }
258                }
259            }
260        }
261    }
262
263    @Override
264    public Set<ActiveMQDestination> getDestinations() {
265        Set<ActiveMQDestination> results = new HashSet<ActiveMQDestination>();
266        for (PersistenceAdapter persistenceAdapter : adapters) {
267            results.addAll(persistenceAdapter.getDestinations());
268        }
269        return results;
270    }
271
272    @Override
273    public long getLastMessageBrokerSequenceId() throws IOException {
274        long maxId = -1;
275        for (PersistenceAdapter persistenceAdapter : adapters) {
276            maxId = Math.max(maxId, persistenceAdapter.getLastMessageBrokerSequenceId());
277        }
278        return maxId;
279    }
280
281    @Override
282    public long getLastProducerSequenceId(ProducerId id) throws IOException {
283        long maxId = -1;
284        for (PersistenceAdapter persistenceAdapter : adapters) {
285            maxId = Math.max(maxId, persistenceAdapter.getLastProducerSequenceId(id));
286        }
287        return maxId;
288    }
289
290    @Override
291    public void allowIOResumption() {
292        for (PersistenceAdapter persistenceAdapter : adapters) {
293            persistenceAdapter.allowIOResumption();
294        }
295    }
296
297    @Override
298    public void removeQueueMessageStore(ActiveMQQueue destination) {
299        PersistenceAdapter adapter = null;
300        try {
301            adapter = getMatchingPersistenceAdapter(destination);
302        } catch (IOException e) {
303            throw new RuntimeException(e);
304        }
305        if (adapter instanceof PersistenceAdapter && adapter.getDestinations().isEmpty()) {
306            adapter.removeQueueMessageStore(destination);
307            removeMessageStore(adapter, destination);
308            destinationMap.remove(destination, adapter);
309        }
310    }
311
312    @Override
313    public void removeTopicMessageStore(ActiveMQTopic destination) {
314        PersistenceAdapter adapter = null;
315        try {
316            adapter = getMatchingPersistenceAdapter(destination);
317        } catch (IOException e) {
318            throw new RuntimeException(e);
319        }
320        if (adapter instanceof PersistenceAdapter && adapter.getDestinations().isEmpty()) {
321            adapter.removeTopicMessageStore(destination);
322            removeMessageStore(adapter, destination);
323            destinationMap.remove(destination, adapter);
324        }
325    }
326
327    private void removeMessageStore(PersistenceAdapter adapter, ActiveMQDestination destination) {
328        stopAdapter(adapter, destination.toString());
329        File adapterDir = adapter.getDirectory();
330        if (adapterDir != null) {
331            if (IOHelper.deleteFile(adapterDir)) {
332                if (LOG.isTraceEnabled()) {
333                    LOG.trace("deleted per destination adapter directory for: " + destination);
334                }
335            } else {
336                if (LOG.isTraceEnabled()) {
337                    LOG.trace("failed to deleted per destination adapter directory for: " + destination);
338                }
339            }
340        }
341    }
342
343    @Override
344    public void rollbackTransaction(ConnectionContext context) throws IOException {
345        throw new IllegalStateException();
346    }
347
348    @Override
349    public void setBrokerName(String brokerName) {
350        for (PersistenceAdapter persistenceAdapter : adapters) {
351            persistenceAdapter.setBrokerName(brokerName);
352        }
353    }
354
355    @Override
356    public void setUsageManager(SystemUsage usageManager) {
357        for (PersistenceAdapter persistenceAdapter : adapters) {
358            persistenceAdapter.setUsageManager(usageManager);
359        }
360    }
361
362    @Override
363    public long size() {
364        long size = 0;
365        for (PersistenceAdapter persistenceAdapter : adapters) {
366            size += persistenceAdapter.size();
367        }
368        return size;
369    }
370
371    @Override
372    public void doStart() throws Exception {
373        Object result = destinationMap.chooseValue(matchAll);
374        if (result != null) {
375            FilteredKahaDBPersistenceAdapter filteredAdapter = (FilteredKahaDBPersistenceAdapter) result;
376            if (filteredAdapter.getDestination() == matchAll && filteredAdapter.isPerDestination()) {
377                findAndRegisterExistingAdapters(filteredAdapter);
378            }
379        }
380        for (PersistenceAdapter persistenceAdapter : adapters) {
381            persistenceAdapter.start();
382        }
383    }
384
385    private void findAndRegisterExistingAdapters(FilteredKahaDBPersistenceAdapter template) throws IOException {
386        FileFilter destinationNames = new FileFilter() {
387            @Override
388            public boolean accept(File file) {
389                return file.getName().startsWith("queue#") || file.getName().startsWith("topic#");
390            }
391        };
392        File[] candidates = template.getPersistenceAdapter().getDirectory().listFiles(destinationNames);
393        if (candidates != null) {
394            for (File candidate : candidates) {
395                registerExistingAdapter(template, candidate);
396            }
397        }
398    }
399
400    private void registerExistingAdapter(FilteredKahaDBPersistenceAdapter filteredAdapter, File candidate) throws IOException {
401        PersistenceAdapter adapter = adapterFromTemplate(filteredAdapter, candidate.getName());
402        startAdapter(adapter, candidate.getName());
403        Set<ActiveMQDestination> destinations = adapter.getDestinations();
404        if (destinations.size() != 0) {
405            registerAdapter(filteredAdapter, adapter, destinations.toArray(new ActiveMQDestination[]{})[0]);
406        } else {
407            stopAdapter(adapter, candidate.getName());
408        }
409    }
410
411    private FilteredKahaDBPersistenceAdapter addAdapter(FilteredKahaDBPersistenceAdapter filteredAdapter, ActiveMQDestination destination) throws IOException {
412        PersistenceAdapter adapter = adapterFromTemplate(filteredAdapter, nameFromDestinationFilter(destination));
413        return registerAdapter(filteredAdapter, adapter, destination);
414    }
415
416    private PersistenceAdapter adapterFromTemplate(FilteredKahaDBPersistenceAdapter template, String destinationName) throws IOException {
417        PersistenceAdapter adapter = kahaDBFromTemplate(template.getPersistenceAdapter());
418        configureAdapter(adapter);
419        configureDirectory(adapter, destinationName);
420        configureIndexDirectory(adapter, template.getPersistenceAdapter(), destinationName);
421        return adapter;
422    }
423
424    private void configureIndexDirectory(PersistenceAdapter adapter, PersistenceAdapter template, String destinationName) {
425        if (template instanceof KahaDBPersistenceAdapter) {
426            KahaDBPersistenceAdapter kahaDBPersistenceAdapter = (KahaDBPersistenceAdapter) template;
427            if (kahaDBPersistenceAdapter.getIndexDirectory() != null) {
428                if (adapter instanceof KahaDBPersistenceAdapter) {
429                    File directory = kahaDBPersistenceAdapter.getIndexDirectory();
430                    if (destinationName != null) {
431                        directory = new File(directory, destinationName);
432                    }
433                    ((KahaDBPersistenceAdapter)adapter).setIndexDirectory(directory);
434                }
435            }
436        }
437    }
438
439    private void configureDirectory(PersistenceAdapter adapter, String fileName) {
440        File directory = null;
441        File defaultDir = DEFAULT_DIRECTORY;
442        try {
443            defaultDir = adapter.getClass().newInstance().getDirectory();
444        } catch (Exception e) {
445        }
446        if (defaultDir.equals(adapter.getDirectory())) {
447            // not set so inherit from mkahadb
448            directory = getDirectory();
449        } else {
450            directory = adapter.getDirectory();
451        }
452
453        if (fileName != null) {
454            directory = new File(directory, fileName);
455        }
456        adapter.setDirectory(directory);
457    }
458
459    private FilteredKahaDBPersistenceAdapter registerAdapter(FilteredKahaDBPersistenceAdapter template, PersistenceAdapter adapter, ActiveMQDestination destination) {
460        adapters.add(adapter);
461        FilteredKahaDBPersistenceAdapter result = new FilteredKahaDBPersistenceAdapter(template, destination, adapter);
462        destinationMap.put(destination, result);
463        return result;
464    }
465
466    private void configureAdapter(PersistenceAdapter adapter) {
467        // need a per store factory that will put the store in the branch qualifier to disiambiguate xid mbeans
468        ((TransactionIdTransformerAware)adapter).setTransactionIdTransformer(transactionIdTransformer);
469        if (isUseLock()) {
470            if( adapter instanceof Lockable ) {
471                ((Lockable)adapter).setUseLock(false);
472            }
473        }
474        if( adapter instanceof BrokerServiceAware ) {
475            ((BrokerServiceAware)adapter).setBrokerService(getBrokerService());
476        }
477    }
478
479    private PersistenceAdapter kahaDBFromTemplate(PersistenceAdapter template) throws IOException {
480        try {
481            Map<String, Object> configuration = new HashMap<String, Object>();
482            IntrospectionSupport.getProperties(template, configuration, null);
483            PersistenceAdapter adapter = template.getClass().newInstance();
484            IntrospectionSupport.setProperties(adapter, configuration);
485            return adapter;
486        } catch (Exception e) {
487            throw IOExceptionSupport.create(e);
488        }
489    }
490
491    @Override
492    protected void doStop(ServiceStopper stopper) throws Exception {
493        for (PersistenceAdapter persistenceAdapter : adapters) {
494            stopper.stop(persistenceAdapter);
495        }
496    }
497
498    @Override
499    public File getDirectory() {
500        return this.directory;
501    }
502
503    @Override
504    public void setDirectory(File directory) {
505        this.directory = directory;
506    }
507
508    @Override
509    public void init() throws Exception {
510    }
511
512    @Override
513    public void setBrokerService(BrokerService brokerService) {
514        super.setBrokerService(brokerService);
515        for (PersistenceAdapter persistenceAdapter : adapters) {
516            if( persistenceAdapter instanceof BrokerServiceAware ) {
517                ((BrokerServiceAware)persistenceAdapter).setBrokerService(getBrokerService());
518            }
519        }
520    }
521
522    public void setTransactionStore(MultiKahaDBTransactionStore transactionStore) {
523        this.transactionStore = transactionStore;
524    }
525
526    /**
527     * Set the max file length of the transaction journal
528     * When set using Xbean, values of the form "20 Mb", "1024kb", and "1g" can
529     * be used
530     *
531     * @org.apache.xbean.Property propertyEditor="org.apache.activemq.util.MemoryIntPropertyEditor"
532     */
533    public void setJournalMaxFileLength(int maxFileLength) {
534        transactionStore.setJournalMaxFileLength(maxFileLength);
535    }
536
537    public int getJournalMaxFileLength() {
538        return transactionStore.getJournalMaxFileLength();
539    }
540
541    /**
542     * Set the max write batch size of  the transaction journal
543     * When set using Xbean, values of the form "20 Mb", "1024kb", and "1g" can
544     * be used
545     *
546     * @org.apache.xbean.Property propertyEditor="org.apache.activemq.util.MemoryIntPropertyEditor"
547     */
548    public void setJournalWriteBatchSize(int journalWriteBatchSize) {
549        transactionStore.setJournalMaxWriteBatchSize(journalWriteBatchSize);
550    }
551
552    public int getJournalWriteBatchSize() {
553        return transactionStore.getJournalMaxWriteBatchSize();
554    }
555
556
557    public void setJournalCleanupInterval(long journalCleanupInterval) {
558        transactionStore.setJournalCleanupInterval(journalCleanupInterval);
559    }
560
561    public long getJournalCleanupInterval() {
562        return transactionStore.getJournalCleanupInterval();
563    }
564
565    public void setCheckForCorruption(boolean checkForCorruption) {
566        transactionStore.setCheckForCorruption(checkForCorruption);
567    }
568
569    public boolean isCheckForCorruption() {
570        return transactionStore.isCheckForCorruption();
571    }
572
573    public List<PersistenceAdapter> getAdapters() {
574        return Collections.unmodifiableList(adapters);
575    }
576
577    @Override
578    public String toString() {
579        String path = getDirectory() != null ? getDirectory().getAbsolutePath() : "DIRECTORY_NOT_SET";
580        return "MultiKahaDBPersistenceAdapter[" + path + "]" + adapters;
581    }
582
583    @Override
584    public Locker createDefaultLocker() throws IOException {
585        SharedFileLocker locker = new SharedFileLocker();
586        locker.configure(this);
587        return locker;
588    }
589
590    @Override
591    public JobSchedulerStore createJobSchedulerStore() throws IOException, UnsupportedOperationException {
592        return new JobSchedulerStoreImpl();
593    }
594
595    /* (non-Javadoc)
596     * @see org.apache.activemq.store.NoLocalSubscriptionAware#isPersistNoLocal()
597     */
598    @Override
599    public boolean isPersistNoLocal() {
600        // Prior to v11 the broker did not store the noLocal value for durable subs.
601        return brokerService.getStoreOpenWireVersion() >= 11;
602    }
603}