001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.camel.processor.idempotent;
018
019import java.io.File;
020import java.io.FileOutputStream;
021import java.io.IOException;
022import java.util.Map;
023import java.util.Scanner;
024import java.util.concurrent.atomic.AtomicBoolean;
025
026import org.apache.camel.api.management.ManagedAttribute;
027import org.apache.camel.api.management.ManagedOperation;
028import org.apache.camel.api.management.ManagedResource;
029import org.apache.camel.spi.IdempotentRepository;
030import org.apache.camel.support.ServiceSupport;
031import org.apache.camel.util.FileUtil;
032import org.apache.camel.util.IOHelper;
033import org.apache.camel.util.LRUCache;
034import org.apache.camel.util.LRUCacheFactory;
035import org.apache.camel.util.ObjectHelper;
036import org.slf4j.Logger;
037import org.slf4j.LoggerFactory;
038
039/**
040 * A file based implementation of {@link org.apache.camel.spi.IdempotentRepository}.
041 * <p/>
042 * Care should be taken to use a suitable underlying {@link java.util.Map} to avoid this class being a
043 * memory leak.
044 *
045 * @version 
046 */
047@ManagedResource(description = "File based idempotent repository")
048public class FileIdempotentRepository extends ServiceSupport implements IdempotentRepository<String> {
049    private static final Logger LOG = LoggerFactory.getLogger(FileIdempotentRepository.class);
050    private static final String STORE_DELIMITER = "\n";
051    private Map<String, Object> cache;
052    private File fileStore;
053    private long maxFileStoreSize = 1024 * 1000L; // 1mb store file
054    private final AtomicBoolean init = new AtomicBoolean();
055
056    public FileIdempotentRepository() {
057    }
058
059    public FileIdempotentRepository(File fileStore, Map<String, Object> set) {
060        this.fileStore = fileStore;
061        this.cache = set;
062    }
063
064    /**
065     * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache}
066     * as 1st level cache with a default of 1000 entries in the cache.
067     *
068     * @param fileStore  the file store
069     */
070    public static IdempotentRepository<String> fileIdempotentRepository(File fileStore) {
071        return fileIdempotentRepository(fileStore, 1000);
072    }
073
074    /**
075     * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache}
076     * as 1st level cache.
077     *
078     * @param fileStore  the file store
079     * @param cacheSize  the cache size
080     */
081    @SuppressWarnings("unchecked")
082    public static IdempotentRepository<String> fileIdempotentRepository(File fileStore, int cacheSize) {
083        return fileIdempotentRepository(fileStore, LRUCacheFactory.newLRUCache(cacheSize));
084    }
085
086    /**
087     * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache}
088     * as 1st level cache.
089     *
090     * @param fileStore  the file store
091     * @param cacheSize  the cache size
092     * @param maxFileStoreSize  the max size in bytes for the filestore file 
093     */
094    @SuppressWarnings("unchecked")
095    public static IdempotentRepository<String> fileIdempotentRepository(File fileStore, int cacheSize, long maxFileStoreSize) {
096        FileIdempotentRepository repository = new FileIdempotentRepository(fileStore, LRUCacheFactory.newLRUCache(cacheSize));
097        repository.setMaxFileStoreSize(maxFileStoreSize);
098        return repository;
099    }
100
101    /**
102     * Creates a new file based repository using the given {@link java.util.Map}
103     * as 1st level cache.
104     * <p/>
105     * Care should be taken to use a suitable underlying {@link java.util.Map} to avoid this class being a
106     * memory leak.
107     *
108     * @param store  the file store
109     * @param cache  the cache to use as 1st level cache
110     */
111    public static IdempotentRepository<String> fileIdempotentRepository(File store, Map<String, Object> cache) {
112        return new FileIdempotentRepository(store, cache);
113    }
114
115    @ManagedOperation(description = "Adds the key to the store")
116    public boolean add(String key) {
117        synchronized (cache) {
118            if (cache.containsKey(key)) {
119                return false;
120            } else {
121                cache.put(key, key);
122                if (fileStore.length() < maxFileStoreSize) {
123                    // just append to store
124                    appendToStore(key);
125                } else {
126                    // trunk store and flush the cache
127                    trunkStore();
128                }
129
130                return true;
131            }
132        }
133    }
134
135    @ManagedOperation(description = "Does the store contain the given key")
136    public boolean contains(String key) {
137        synchronized (cache) {
138            return cache.containsKey(key);
139        }
140    }
141
142    @ManagedOperation(description = "Remove the key from the store")
143    public boolean remove(String key) {
144        boolean answer;
145        synchronized (cache) {
146            answer = cache.remove(key) != null;
147            // trunk store and flush the cache on remove
148            trunkStore();
149        }
150        return answer;
151    }
152
153    public boolean confirm(String key) {
154        // noop
155        return true;
156    }
157    
158    @ManagedOperation(description = "Clear the store")
159    public void clear() {
160        synchronized (cache) {
161            cache.clear();
162            if (cache instanceof LRUCache) {
163                ((LRUCache) cache).cleanUp();
164            }
165        }
166    }
167
168    public File getFileStore() {
169        return fileStore;
170    }
171
172    public void setFileStore(File fileStore) {
173        this.fileStore = fileStore;
174    }
175
176    @ManagedAttribute(description = "The file path for the store")
177    public String getFilePath() {
178        return fileStore.getPath();
179    }
180
181    public Map<String, Object> getCache() {
182        return cache;
183    }
184
185    public void setCache(Map<String, Object> cache) {
186        this.cache = cache;
187    }
188
189    @ManagedAttribute(description = "The maximum file size for the file store in bytes")
190    public long getMaxFileStoreSize() {
191        return maxFileStoreSize;
192    }
193
194    /**
195     * Sets the maximum file size for the file store in bytes.
196     * <p/>
197     * The default is 1mb.
198     */
199    @ManagedAttribute(description = "The maximum file size for the file store in bytes")
200    public void setMaxFileStoreSize(long maxFileStoreSize) {
201        this.maxFileStoreSize = maxFileStoreSize;
202    }
203
204    /**
205     * Sets the cache size
206     */
207    @SuppressWarnings("unchecked")
208    public void setCacheSize(int size) {
209        if (cache != null) {
210            cache.clear();
211        }
212        cache = LRUCacheFactory.newLRUCache(size);
213    }
214
215    @ManagedAttribute(description = "The current cache size")
216    public int getCacheSize() {
217        if (cache != null) {
218            return cache.size();
219        }
220        return 0;
221    }
222
223    /**
224     * Reset and clears the store to force it to reload from file
225     */
226    @ManagedOperation(description = "Reset and reloads the file store")
227    public synchronized void reset() throws IOException {
228        synchronized (cache) {
229            // trunk and clear, before we reload the store
230            trunkStore();
231            cache.clear();
232            if (cache instanceof LRUCache) {
233                ((LRUCache) cache).cleanUp();
234            }
235            loadStore();
236        }
237    }
238
239    /**
240     * Appends the given message id to the file store
241     *
242     * @param messageId  the message id
243     */
244    protected void appendToStore(final String messageId) {
245        LOG.debug("Appending {} to idempotent filestore: {}", messageId, fileStore);
246        FileOutputStream fos = null;
247        try {
248            // create store parent directory if missing
249            File storeParentDirectory = fileStore.getParentFile();
250            if (storeParentDirectory != null && !storeParentDirectory.exists()) {
251                LOG.info("Parent directory of file store {} doesn't exist. Creating.", fileStore);
252                if (fileStore.getParentFile().mkdirs()) {
253                    LOG.info("Parent directory of file store {} successfully created.", fileStore);
254                } else {
255                    LOG.warn("Parent directory of file store {} cannot be created.", fileStore);
256                }
257            }
258            // create store if missing
259            if (!fileStore.exists()) {
260                FileUtil.createNewFile(fileStore);
261            }
262            // append to store
263            fos = new FileOutputStream(fileStore, true);
264            fos.write(messageId.getBytes());
265            fos.write(STORE_DELIMITER.getBytes());
266        } catch (IOException e) {
267            throw ObjectHelper.wrapRuntimeCamelException(e);
268        } finally {
269            IOHelper.close(fos, "Appending to file idempotent repository", LOG);
270        }
271    }
272
273    /**
274     * Trunks the file store when the max store size is hit by rewriting the 1st level cache
275     * to the file store.
276     */
277    protected void trunkStore() {
278        LOG.info("Trunking idempotent filestore: {}", fileStore);
279        FileOutputStream fos = null;
280        try {
281            fos = new FileOutputStream(fileStore);
282            for (String key : cache.keySet()) {
283                fos.write(key.getBytes());
284                fos.write(STORE_DELIMITER.getBytes());
285            }
286        } catch (IOException e) {
287            throw ObjectHelper.wrapRuntimeCamelException(e);
288        } finally {
289            IOHelper.close(fos, "Trunking file idempotent repository", LOG);
290        }
291    }
292
293    /**
294     * Loads the given file store into the 1st level cache
295     */
296    protected void loadStore() throws IOException {
297        // auto create starting directory if needed
298        if (!fileStore.exists()) {
299            LOG.debug("Creating filestore: {}", fileStore);
300            File parent = fileStore.getParentFile();
301            if (parent != null) {
302                parent.mkdirs();
303            }
304            boolean created = FileUtil.createNewFile(fileStore);
305            if (!created) {
306                throw new IOException("Cannot create filestore: " + fileStore);
307            }
308        }
309
310        LOG.trace("Loading to 1st level cache from idempotent filestore: {}", fileStore);
311
312        cache.clear();
313        Scanner scanner = null;
314        try {
315            scanner = new Scanner(fileStore);
316            scanner.useDelimiter(STORE_DELIMITER);
317            while (scanner.hasNextLine()) {
318                String line = scanner.nextLine();
319                cache.put(line, line);
320            }
321        } catch (IOException e) {
322            throw ObjectHelper.wrapRuntimeCamelException(e);
323        } finally {
324            if (scanner != null) {
325                scanner.close();
326            }
327        }
328
329        LOG.debug("Loaded {} to the 1st level cache from idempotent filestore: {}", cache.size(), fileStore);
330    }
331
332    @Override
333    @SuppressWarnings("unchecked")
334    protected void doStart() throws Exception {
335        ObjectHelper.notNull(fileStore, "fileStore", this);
336
337        if (this.cache == null) {
338            // default use a 1st level cache
339            this.cache = LRUCacheFactory.newLRUCache(1000);
340        }
341
342        // init store if not loaded before
343        if (init.compareAndSet(false, true)) {
344            loadStore();
345        }
346    }
347
348    @Override
349    protected void doStop() throws Exception {
350        // reset will trunk and clear the cache
351        trunkStore();
352        cache.clear();
353        if (cache instanceof LRUCache) {
354            ((LRUCache) cache).cleanUp();
355        }
356        init.set(false);
357    }
358
359}