001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.processor.idempotent; 018 019import java.io.File; 020import java.io.FileOutputStream; 021import java.io.IOException; 022import java.util.Map; 023import java.util.Scanner; 024import java.util.concurrent.atomic.AtomicBoolean; 025 026import org.apache.camel.api.management.ManagedAttribute; 027import org.apache.camel.api.management.ManagedOperation; 028import org.apache.camel.api.management.ManagedResource; 029import org.apache.camel.spi.IdempotentRepository; 030import org.apache.camel.support.ServiceSupport; 031import org.apache.camel.util.FileUtil; 032import org.apache.camel.util.IOHelper; 033import org.apache.camel.util.LRUCache; 034import org.apache.camel.util.LRUCacheFactory; 035import org.apache.camel.util.ObjectHelper; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039/** 040 * A file based implementation of {@link org.apache.camel.spi.IdempotentRepository}. 041 * <p/> 042 * Care should be taken to use a suitable underlying {@link java.util.Map} to avoid this class being a 043 * memory leak. 044 * 045 * @version 046 */ 047@ManagedResource(description = "File based idempotent repository") 048public class FileIdempotentRepository extends ServiceSupport implements IdempotentRepository<String> { 049 private static final Logger LOG = LoggerFactory.getLogger(FileIdempotentRepository.class); 050 private static final String STORE_DELIMITER = "\n"; 051 private Map<String, Object> cache; 052 private File fileStore; 053 private long maxFileStoreSize = 1024 * 1000L; // 1mb store file 054 private final AtomicBoolean init = new AtomicBoolean(); 055 056 public FileIdempotentRepository() { 057 } 058 059 public FileIdempotentRepository(File fileStore, Map<String, Object> set) { 060 this.fileStore = fileStore; 061 this.cache = set; 062 } 063 064 /** 065 * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache} 066 * as 1st level cache with a default of 1000 entries in the cache. 067 * 068 * @param fileStore the file store 069 */ 070 public static IdempotentRepository<String> fileIdempotentRepository(File fileStore) { 071 return fileIdempotentRepository(fileStore, 1000); 072 } 073 074 /** 075 * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache} 076 * as 1st level cache. 077 * 078 * @param fileStore the file store 079 * @param cacheSize the cache size 080 */ 081 @SuppressWarnings("unchecked") 082 public static IdempotentRepository<String> fileIdempotentRepository(File fileStore, int cacheSize) { 083 return fileIdempotentRepository(fileStore, LRUCacheFactory.newLRUCache(cacheSize)); 084 } 085 086 /** 087 * Creates a new file based repository using a {@link org.apache.camel.util.LRUCache} 088 * as 1st level cache. 089 * 090 * @param fileStore the file store 091 * @param cacheSize the cache size 092 * @param maxFileStoreSize the max size in bytes for the filestore file 093 */ 094 @SuppressWarnings("unchecked") 095 public static IdempotentRepository<String> fileIdempotentRepository(File fileStore, int cacheSize, long maxFileStoreSize) { 096 FileIdempotentRepository repository = new FileIdempotentRepository(fileStore, LRUCacheFactory.newLRUCache(cacheSize)); 097 repository.setMaxFileStoreSize(maxFileStoreSize); 098 return repository; 099 } 100 101 /** 102 * Creates a new file based repository using the given {@link java.util.Map} 103 * as 1st level cache. 104 * <p/> 105 * Care should be taken to use a suitable underlying {@link java.util.Map} to avoid this class being a 106 * memory leak. 107 * 108 * @param store the file store 109 * @param cache the cache to use as 1st level cache 110 */ 111 public static IdempotentRepository<String> fileIdempotentRepository(File store, Map<String, Object> cache) { 112 return new FileIdempotentRepository(store, cache); 113 } 114 115 @ManagedOperation(description = "Adds the key to the store") 116 public boolean add(String key) { 117 synchronized (cache) { 118 if (cache.containsKey(key)) { 119 return false; 120 } else { 121 cache.put(key, key); 122 if (fileStore.length() < maxFileStoreSize) { 123 // just append to store 124 appendToStore(key); 125 } else { 126 // trunk store and flush the cache 127 trunkStore(); 128 } 129 130 return true; 131 } 132 } 133 } 134 135 @ManagedOperation(description = "Does the store contain the given key") 136 public boolean contains(String key) { 137 synchronized (cache) { 138 return cache.containsKey(key); 139 } 140 } 141 142 @ManagedOperation(description = "Remove the key from the store") 143 public boolean remove(String key) { 144 boolean answer; 145 synchronized (cache) { 146 answer = cache.remove(key) != null; 147 // trunk store and flush the cache on remove 148 trunkStore(); 149 } 150 return answer; 151 } 152 153 public boolean confirm(String key) { 154 // noop 155 return true; 156 } 157 158 @ManagedOperation(description = "Clear the store") 159 public void clear() { 160 synchronized (cache) { 161 cache.clear(); 162 if (cache instanceof LRUCache) { 163 ((LRUCache) cache).cleanUp(); 164 } 165 } 166 } 167 168 public File getFileStore() { 169 return fileStore; 170 } 171 172 public void setFileStore(File fileStore) { 173 this.fileStore = fileStore; 174 } 175 176 @ManagedAttribute(description = "The file path for the store") 177 public String getFilePath() { 178 return fileStore.getPath(); 179 } 180 181 public Map<String, Object> getCache() { 182 return cache; 183 } 184 185 public void setCache(Map<String, Object> cache) { 186 this.cache = cache; 187 } 188 189 @ManagedAttribute(description = "The maximum file size for the file store in bytes") 190 public long getMaxFileStoreSize() { 191 return maxFileStoreSize; 192 } 193 194 /** 195 * Sets the maximum file size for the file store in bytes. 196 * <p/> 197 * The default is 1mb. 198 */ 199 @ManagedAttribute(description = "The maximum file size for the file store in bytes") 200 public void setMaxFileStoreSize(long maxFileStoreSize) { 201 this.maxFileStoreSize = maxFileStoreSize; 202 } 203 204 /** 205 * Sets the cache size 206 */ 207 @SuppressWarnings("unchecked") 208 public void setCacheSize(int size) { 209 if (cache != null) { 210 cache.clear(); 211 } 212 cache = LRUCacheFactory.newLRUCache(size); 213 } 214 215 @ManagedAttribute(description = "The current cache size") 216 public int getCacheSize() { 217 if (cache != null) { 218 return cache.size(); 219 } 220 return 0; 221 } 222 223 /** 224 * Reset and clears the store to force it to reload from file 225 */ 226 @ManagedOperation(description = "Reset and reloads the file store") 227 public synchronized void reset() throws IOException { 228 synchronized (cache) { 229 // trunk and clear, before we reload the store 230 trunkStore(); 231 cache.clear(); 232 if (cache instanceof LRUCache) { 233 ((LRUCache) cache).cleanUp(); 234 } 235 loadStore(); 236 } 237 } 238 239 /** 240 * Appends the given message id to the file store 241 * 242 * @param messageId the message id 243 */ 244 protected void appendToStore(final String messageId) { 245 LOG.debug("Appending {} to idempotent filestore: {}", messageId, fileStore); 246 FileOutputStream fos = null; 247 try { 248 // create store parent directory if missing 249 File storeParentDirectory = fileStore.getParentFile(); 250 if (storeParentDirectory != null && !storeParentDirectory.exists()) { 251 LOG.info("Parent directory of file store {} doesn't exist. Creating.", fileStore); 252 if (fileStore.getParentFile().mkdirs()) { 253 LOG.info("Parent directory of file store {} successfully created.", fileStore); 254 } else { 255 LOG.warn("Parent directory of file store {} cannot be created.", fileStore); 256 } 257 } 258 // create store if missing 259 if (!fileStore.exists()) { 260 FileUtil.createNewFile(fileStore); 261 } 262 // append to store 263 fos = new FileOutputStream(fileStore, true); 264 fos.write(messageId.getBytes()); 265 fos.write(STORE_DELIMITER.getBytes()); 266 } catch (IOException e) { 267 throw ObjectHelper.wrapRuntimeCamelException(e); 268 } finally { 269 IOHelper.close(fos, "Appending to file idempotent repository", LOG); 270 } 271 } 272 273 /** 274 * Trunks the file store when the max store size is hit by rewriting the 1st level cache 275 * to the file store. 276 */ 277 protected void trunkStore() { 278 LOG.info("Trunking idempotent filestore: {}", fileStore); 279 FileOutputStream fos = null; 280 try { 281 fos = new FileOutputStream(fileStore); 282 for (String key : cache.keySet()) { 283 fos.write(key.getBytes()); 284 fos.write(STORE_DELIMITER.getBytes()); 285 } 286 } catch (IOException e) { 287 throw ObjectHelper.wrapRuntimeCamelException(e); 288 } finally { 289 IOHelper.close(fos, "Trunking file idempotent repository", LOG); 290 } 291 } 292 293 /** 294 * Loads the given file store into the 1st level cache 295 */ 296 protected void loadStore() throws IOException { 297 // auto create starting directory if needed 298 if (!fileStore.exists()) { 299 LOG.debug("Creating filestore: {}", fileStore); 300 File parent = fileStore.getParentFile(); 301 if (parent != null) { 302 parent.mkdirs(); 303 } 304 boolean created = FileUtil.createNewFile(fileStore); 305 if (!created) { 306 throw new IOException("Cannot create filestore: " + fileStore); 307 } 308 } 309 310 LOG.trace("Loading to 1st level cache from idempotent filestore: {}", fileStore); 311 312 cache.clear(); 313 Scanner scanner = null; 314 try { 315 scanner = new Scanner(fileStore); 316 scanner.useDelimiter(STORE_DELIMITER); 317 while (scanner.hasNextLine()) { 318 String line = scanner.nextLine(); 319 cache.put(line, line); 320 } 321 } catch (IOException e) { 322 throw ObjectHelper.wrapRuntimeCamelException(e); 323 } finally { 324 if (scanner != null) { 325 scanner.close(); 326 } 327 } 328 329 LOG.debug("Loaded {} to the 1st level cache from idempotent filestore: {}", cache.size(), fileStore); 330 } 331 332 @Override 333 @SuppressWarnings("unchecked") 334 protected void doStart() throws Exception { 335 ObjectHelper.notNull(fileStore, "fileStore", this); 336 337 if (this.cache == null) { 338 // default use a 1st level cache 339 this.cache = LRUCacheFactory.newLRUCache(1000); 340 } 341 342 // init store if not loaded before 343 if (init.compareAndSet(false, true)) { 344 loadStore(); 345 } 346 } 347 348 @Override 349 protected void doStop() throws Exception { 350 // reset will trunk and clear the cache 351 trunkStore(); 352 cache.clear(); 353 if (cache instanceof LRUCache) { 354 ((LRUCache) cache).cleanUp(); 355 } 356 init.set(false); 357 } 358 359}