001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.camel.component.file;
018
019import java.io.IOException;
020import java.lang.reflect.Method;
021import java.util.ArrayList;
022import java.util.Comparator;
023import java.util.HashMap;
024import java.util.List;
025import java.util.Map;
026import java.util.regex.Matcher;
027import java.util.regex.Pattern;
028
029import org.apache.camel.CamelContext;
030import org.apache.camel.Component;
031import org.apache.camel.Exchange;
032import org.apache.camel.Expression;
033import org.apache.camel.ExpressionIllegalSyntaxException;
034import org.apache.camel.LoggingLevel;
035import org.apache.camel.Message;
036import org.apache.camel.Predicate;
037import org.apache.camel.Processor;
038import org.apache.camel.impl.ScheduledPollEndpoint;
039import org.apache.camel.processor.idempotent.MemoryIdempotentRepository;
040import org.apache.camel.spi.BrowsableEndpoint;
041import org.apache.camel.spi.ExceptionHandler;
042import org.apache.camel.spi.FactoryFinder;
043import org.apache.camel.spi.IdempotentRepository;
044import org.apache.camel.spi.Language;
045import org.apache.camel.spi.UriParam;
046import org.apache.camel.util.FileUtil;
047import org.apache.camel.util.IOHelper;
048import org.apache.camel.util.ObjectHelper;
049import org.apache.camel.util.ServiceHelper;
050import org.apache.camel.util.StringHelper;
051import org.slf4j.Logger;
052import org.slf4j.LoggerFactory;
053
054/**
055 * Base class for file endpoints
056 */
057public abstract class GenericFileEndpoint<T> extends ScheduledPollEndpoint implements BrowsableEndpoint {
058
059    protected static final String DEFAULT_STRATEGYFACTORY_CLASS = "org.apache.camel.component.file.strategy.GenericFileProcessStrategyFactory";
060    protected static final int DEFAULT_IDEMPOTENT_CACHE_SIZE = 1000;
061    protected static final int DEFAULT_IN_PROGRESS_CACHE_SIZE = 50000;
062
063    protected final Logger log = LoggerFactory.getLogger(getClass());
064
065    // common options
066
067    @UriParam(label = "advanced", defaultValue = "true")
068    protected boolean autoCreate = true;
069    @UriParam(label = "advanced", defaultValue = "" + FileUtil.BUFFER_SIZE)
070    protected int bufferSize = FileUtil.BUFFER_SIZE;
071    @UriParam
072    protected String charset;
073    @UriParam(javaType = "java.lang.String")
074    protected Expression fileName;
075    @UriParam
076    protected String doneFileName;
077
078    // producer options
079
080    @UriParam(label = "producer")
081    protected boolean flatten;
082    @UriParam(label = "producer", defaultValue = "Override")
083    protected GenericFileExist fileExist = GenericFileExist.Override;
084    @UriParam(label = "producer")
085    protected String tempPrefix;
086    @UriParam(label = "producer", javaType = "java.lang.String")
087    protected Expression tempFileName;
088    @UriParam(label = "producer,advanced", defaultValue = "true")
089    protected boolean eagerDeleteTargetFile = true;
090    @UriParam(label = "producer,advanced")
091    protected boolean keepLastModified;
092    @UriParam(label = "producer,advanced")
093    protected boolean allowNullBody;
094    @UriParam(label = "producer", defaultValue = "true")
095    protected boolean jailStartingDirectory = true;
096
097    // consumer options
098
099    @UriParam
100    protected GenericFileConfiguration configuration;
101    @UriParam(label = "consumer,advanced")
102    protected GenericFileProcessStrategy<T> processStrategy;
103    @UriParam(label = "consumer,advanced")
104    protected IdempotentRepository<String> inProgressRepository = MemoryIdempotentRepository.memoryIdempotentRepository(DEFAULT_IN_PROGRESS_CACHE_SIZE);
105    @UriParam(label = "consumer,advanced")
106    protected String localWorkDirectory;
107    @UriParam(label = "consumer,advanced")
108    protected boolean startingDirectoryMustExist;
109    @UriParam(label = "consumer,advanced")
110    protected boolean directoryMustExist;
111    @UriParam(label = "consumer")
112    protected boolean noop;
113    @UriParam(label = "consumer")
114    protected boolean recursive;
115    @UriParam(label = "consumer")
116    protected boolean delete;
117    @UriParam(label = "consumer")
118    protected boolean preSort;
119    @UriParam(label = "consumer,filter")
120    protected int maxMessagesPerPoll;
121    @UriParam(label = "consumer,filter", defaultValue = "true")
122    protected boolean eagerMaxMessagesPerPoll = true;
123    @UriParam(label = "consumer,filter", defaultValue = "" + Integer.MAX_VALUE)
124    protected int maxDepth = Integer.MAX_VALUE;
125    @UriParam(label = "consumer,filter")
126    protected int minDepth;
127    @UriParam(label = "consumer,filter")
128    protected String include;
129    @UriParam(label = "consumer,filter")
130    protected String exclude;
131    @UriParam(label = "consumer,filter", javaType = "java.lang.String")
132    protected Expression move;
133    @UriParam(label = "consumer", javaType = "java.lang.String")
134    protected Expression moveFailed;
135    @UriParam(label = "consumer", javaType = "java.lang.String")
136    protected Expression preMove;
137    @UriParam(label = "producer", javaType = "java.lang.String")
138    protected Expression moveExisting;
139    @UriParam(label = "consumer,filter", defaultValue = "false")
140    protected Boolean idempotent;
141    @UriParam(label = "consumer,filter", javaType = "java.lang.String")
142    protected Expression idempotentKey;
143    @UriParam(label = "consumer,filter")
144    protected IdempotentRepository<String> idempotentRepository;
145    @UriParam(label = "consumer,filter")
146    protected GenericFileFilter<T> filter;
147    @UriParam(label = "consumer,filter", javaType = "java.lang.String")
148    protected Predicate filterDirectory;
149    @UriParam(label = "consumer,filter", javaType = "java.lang.String")
150    protected Predicate filterFile;
151    @UriParam(label = "consumer,filter", defaultValue = "true")
152    protected boolean antFilterCaseSensitive = true;
153    protected volatile AntPathMatcherGenericFileFilter<T> antFilter;
154    @UriParam(label = "consumer,filter")
155    protected String antInclude;
156    @UriParam(label = "consumer,filter")
157    protected String antExclude;
158    @UriParam(label = "consumer,sort")
159    protected Comparator<GenericFile<T>> sorter;
160    @UriParam(label = "consumer,sort", javaType = "java.lang.String")
161    protected Comparator<Exchange> sortBy;
162    @UriParam(label = "consumer,sort")
163    protected boolean shuffle;
164    @UriParam(label = "consumer,lock", defaultValue = "none", enums = "none,markerFile,fileLock,rename,changed,idempotent,idempotent-changed,idempotent-rename")
165    protected String readLock = "none";
166    @UriParam(label = "consumer,lock", defaultValue = "1000")
167    protected long readLockCheckInterval = 1000;
168    @UriParam(label = "consumer,lock", defaultValue = "10000")
169    protected long readLockTimeout = 10000;
170    @UriParam(label = "consumer,lock", defaultValue = "true")
171    protected boolean readLockMarkerFile = true;
172    @UriParam(label = "consumer,lock", defaultValue = "true")
173    protected boolean readLockDeleteOrphanLockFiles = true;
174    @UriParam(label = "consumer,lock", defaultValue = "DEBUG")
175    protected LoggingLevel readLockLoggingLevel = LoggingLevel.DEBUG;
176    @UriParam(label = "consumer,lock", defaultValue = "1")
177    protected long readLockMinLength = 1;
178    @UriParam(label = "consumer,lock", defaultValue = "0")
179    protected long readLockMinAge;
180    @UriParam(label = "consumer,lock", defaultValue = "true")
181    protected boolean readLockRemoveOnRollback = true;
182    @UriParam(label = "consumer,lock")
183    protected boolean readLockRemoveOnCommit;
184    @UriParam(label = "consumer,lock")
185    protected GenericFileExclusiveReadLockStrategy<T> exclusiveReadLockStrategy;
186    @UriParam(label = "consumer,advanced")
187    protected ExceptionHandler onCompletionExceptionHandler;
188
189    private Pattern includePattern;
190    private Pattern excludePattern;
191
192    public GenericFileEndpoint() {
193    }
194
195    public GenericFileEndpoint(String endpointUri, Component component) {
196        super(endpointUri, component);
197    }
198
199    public boolean isSingleton() {
200        return true;
201    }
202
203    public abstract GenericFileConsumer<T> createConsumer(Processor processor) throws Exception;
204
205    public abstract GenericFileProducer<T> createProducer() throws Exception;
206
207    public abstract Exchange createExchange(GenericFile<T> file);
208
209    public abstract String getScheme();
210
211    public abstract char getFileSeparator();
212
213    public abstract boolean isAbsolute(String name);
214
215    /**
216     * Return the file name that will be auto-generated for the given message if
217     * none is provided
218     */
219    public String getGeneratedFileName(Message message) {
220        return StringHelper.sanitize(message.getMessageId());
221    }
222
223    public GenericFileProcessStrategy<T> getGenericFileProcessStrategy() {
224        if (processStrategy == null) {
225            processStrategy = createGenericFileStrategy();
226            log.debug("Using Generic file process strategy: {}", processStrategy);
227        }
228        return processStrategy;
229    }
230
231    /**
232     * This implementation will <b>not</b> load the file content.
233     * Any file locking is neither in use by this implementation..
234     */
235    @Override
236    public List<Exchange> getExchanges() {
237        final List<Exchange> answer = new ArrayList<Exchange>();
238
239        GenericFileConsumer<?> consumer = null;
240        try {
241            // create a new consumer which can poll the exchanges we want to browse
242            // do not provide a processor as we do some custom processing
243            consumer = createConsumer(null);
244            consumer.setCustomProcessor(new Processor() {
245                @Override
246                public void process(Exchange exchange) throws Exception {
247                    answer.add(exchange);
248                }
249            });
250            // do not start scheduler, as we invoke the poll manually
251            consumer.setStartScheduler(false);
252            // start consumer
253            ServiceHelper.startService(consumer);
254            // invoke poll which performs the custom processing, so we can browse the exchanges
255            consumer.poll();
256        } catch (Exception e) {
257            throw ObjectHelper.wrapRuntimeCamelException(e);
258        } finally {
259            try {
260                ServiceHelper.stopService(consumer);
261            } catch (Exception e) {
262                log.debug("Error stopping consumer used for browsing exchanges. This exception will be ignored", e);
263            }
264        }
265
266        return answer;
267    }
268
269    /**
270     * A strategy method to lazily create the file strategy
271     */
272    @SuppressWarnings("unchecked")
273    protected GenericFileProcessStrategy<T> createGenericFileStrategy() {
274        Class<?> factory = null;
275        try {
276            FactoryFinder finder = getCamelContext().getFactoryFinder("META-INF/services/org/apache/camel/component/");
277            log.trace("Using FactoryFinder: {}", finder);
278            factory = finder.findClass(getScheme(), "strategy.factory.", CamelContext.class);
279        } catch (ClassNotFoundException e) {
280            log.trace("'strategy.factory.class' not found", e);
281        } catch (IOException e) {
282            log.trace("No strategy factory defined in 'META-INF/services/org/apache/camel/component/'", e);
283        }
284
285        if (factory == null) {
286            // use default
287            try {
288                log.trace("Using ClassResolver to resolve class: {}", DEFAULT_STRATEGYFACTORY_CLASS);
289                factory = this.getCamelContext().getClassResolver().resolveClass(DEFAULT_STRATEGYFACTORY_CLASS);
290            } catch (Exception e) {
291                log.trace("Cannot load class: {}", DEFAULT_STRATEGYFACTORY_CLASS, e);
292            }
293            // fallback and us this class loader
294            try {
295                if (log.isTraceEnabled()) {
296                    log.trace("Using classloader: {} to resolve class: {}", this.getClass().getClassLoader(), DEFAULT_STRATEGYFACTORY_CLASS);
297                }
298                factory = this.getCamelContext().getClassResolver().resolveClass(DEFAULT_STRATEGYFACTORY_CLASS, this.getClass().getClassLoader());
299            } catch (Exception e) {
300                if (log.isTraceEnabled()) {
301                    log.trace("Cannot load class: {} using classloader: " + this.getClass().getClassLoader(), DEFAULT_STRATEGYFACTORY_CLASS, e);
302                }
303            }
304
305            if (factory == null) {
306                throw new TypeNotPresentException(DEFAULT_STRATEGYFACTORY_CLASS + " class not found", null);
307            }
308        }
309
310        try {
311            Method factoryMethod = factory.getMethod("createGenericFileProcessStrategy", CamelContext.class, Map.class);
312            Map<String, Object> params = getParamsAsMap();
313            log.debug("Parameters for Generic file process strategy {}", params);
314            return (GenericFileProcessStrategy<T>) ObjectHelper.invokeMethod(factoryMethod, null, getCamelContext(), params);
315        } catch (NoSuchMethodException e) {
316            throw new TypeNotPresentException(factory.getSimpleName() + ".createGenericFileProcessStrategy method not found", e);
317        }
318    }
319
320    public boolean isNoop() {
321        return noop;
322    }
323
324    /**
325     * If true, the file is not moved or deleted in any way.
326     * This option is good for readonly data, or for ETL type requirements.
327     * If noop=true, Camel will set idempotent=true as well, to avoid consuming the same files over and over again.
328     */
329    public void setNoop(boolean noop) {
330        this.noop = noop;
331    }
332
333    public boolean isRecursive() {
334        return recursive;
335    }
336
337    /**
338     * If a directory, will look for files in all the sub-directories as well.
339     */
340    public void setRecursive(boolean recursive) {
341        this.recursive = recursive;
342    }
343
344    public String getInclude() {
345        return include;
346    }
347
348    /**
349     * Is used to include files, if filename matches the regex pattern (matching is case in-sensitive).
350     * <p/>
351     * Notice if you use symbols such as plus sign and others you would need to configure
352     * this using the RAW() syntax if configuring this as an endpoint uri.
353     * See more details at <a href="http://camel.apache.org/how-do-i-configure-endpoints.html">configuring endpoint uris</a>
354     */
355    public void setInclude(String include) {
356        this.include = include;
357        this.includePattern = Pattern.compile(include, Pattern.CASE_INSENSITIVE);
358    }
359
360    public Pattern getIncludePattern() {
361        return includePattern;
362    }
363
364    public String getExclude() {
365        return exclude;
366    }
367
368    /**
369     * Is used to exclude files, if filename matches the regex pattern (matching is case in-senstive).
370     * <p/>
371     * Notice if you use symbols such as plus sign and others you would need to configure
372     * this using the RAW() syntax if configuring this as an endpoint uri.
373     * See more details at <a href="http://camel.apache.org/how-do-i-configure-endpoints.html">configuring endpoint uris</a>
374     */
375    public void setExclude(String exclude) {
376        this.exclude = exclude;
377        this.excludePattern = Pattern.compile(exclude, Pattern.CASE_INSENSITIVE);
378    }
379
380    public Pattern getExcludePattern() {
381        return this.excludePattern;
382    }
383
384    public String getAntInclude() {
385        return antInclude;
386    }
387
388    /**
389     * Ant style filter inclusion.
390     * Multiple inclusions may be specified in comma-delimited format.
391     */
392    public void setAntInclude(String antInclude) {
393        this.antInclude = antInclude;
394    }
395
396    public String getAntExclude() {
397        return antExclude;
398    }
399
400    /**
401     * Ant style filter exclusion. If both antInclude and antExclude are used, antExclude takes precedence over antInclude.
402     * Multiple exclusions may be specified in comma-delimited format.
403     */
404    public void setAntExclude(String antExclude) {
405        this.antExclude = antExclude;
406    }
407
408    public boolean isAntFilterCaseSensitive() {
409        return antFilterCaseSensitive;
410    }
411
412    /**
413     * Sets case sensitive flag on ant filter
414     */
415    public void setAntFilterCaseSensitive(boolean antFilterCaseSensitive) {
416        this.antFilterCaseSensitive = antFilterCaseSensitive;
417    }
418
419    public GenericFileFilter<T> getAntFilter() {
420        return antFilter;
421    }
422    
423    public boolean isPreSort() {
424        return preSort;
425    }
426
427    /**
428     * When pre-sort is enabled then the consumer will sort the file and directory names during polling, 
429     * that was retrieved from the file system. You may want to do this in case you need to operate on the files 
430     * in a sorted order. The pre-sort is executed before the consumer starts to filter, and accept files 
431     * to process by Camel. This option is default=false meaning disabled.
432     */
433    public void setPreSort(boolean preSort) {
434        this.preSort = preSort;
435    }
436
437    public boolean isDelete() {
438        return delete;
439    }
440
441    /**
442     * If true, the file will be deleted after it is processed successfully.
443     */
444    public void setDelete(boolean delete) {
445        this.delete = delete;
446    }
447
448    public boolean isFlatten() {
449        return flatten;
450    }
451
452    /**
453     * Flatten is used to flatten the file name path to strip any leading paths, so it's just the file name.
454     * This allows you to consume recursively into sub-directories, but when you eg write the files to another directory
455     * they will be written in a single directory.
456     * Setting this to true on the producer enforces that any file name in CamelFileName header
457     * will be stripped for any leading paths.
458     */
459    public void setFlatten(boolean flatten) {
460        this.flatten = flatten;
461    }
462
463    public Expression getMove() {
464        return move;
465    }
466
467    /**
468     * Expression (such as Simple Language) used to dynamically set the filename when moving it after processing.
469     * To move files into a .done subdirectory just enter .done.
470     */
471    public void setMove(Expression move) {
472        this.move = move;
473    }
474
475    /**
476     * @see #setMove(org.apache.camel.Expression)
477     */
478    public void setMove(String fileLanguageExpression) {
479        String expression = configureMoveOrPreMoveExpression(fileLanguageExpression);
480        this.move = createFileLanguageExpression(expression);
481    }
482
483    public Expression getMoveFailed() {
484        return moveFailed;
485    }
486
487    /**
488     * Sets the move failure expression based on Simple language.
489     * For example, to move files into a .error subdirectory use: .error.
490     * Note: When moving the files to the fail location Camel will handle the error and will not pick up the file again.
491     */
492    public void setMoveFailed(Expression moveFailed) {
493        this.moveFailed = moveFailed;
494    }
495
496    public void setMoveFailed(String fileLanguageExpression) {
497        String expression = configureMoveOrPreMoveExpression(fileLanguageExpression);
498        this.moveFailed = createFileLanguageExpression(expression);
499    }
500
501    public Predicate getFilterDirectory() {
502        return filterDirectory;
503    }
504
505    /**
506     * Filters the directory based on Simple language.
507     * For example to filter on current date, you can use a simple date pattern such as ${date:now:yyyMMdd}
508     */
509    public void setFilterDirectory(Predicate filterDirectory) {
510        this.filterDirectory = filterDirectory;
511    }
512
513    /**
514     * Filters the directory based on Simple language.
515     * For example to filter on current date, you can use a simple date pattern such as ${date:now:yyyMMdd}
516     * @see #setFilterDirectory(Predicate)
517     */
518    public void setFilterDirectory(String expression) {
519        this.filterDirectory = createFileLanguagePredicate(expression);
520    }
521
522    public Predicate getFilterFile() {
523        return filterFile;
524    }
525
526    /**
527     * Filters the file based on Simple language.
528     * For example to filter on file size, you can use ${file:size} > 5000
529     */
530    public void setFilterFile(Predicate filterFile) {
531        this.filterFile = filterFile;
532    }
533
534    /**
535     * Filters the file based on Simple language.
536     * For example to filter on file size, you can use ${file:size} > 5000
537     * @see #setFilterFile(Predicate)
538     */
539    public void setFilterFile(String expression) {
540        this.filterFile = createFileLanguagePredicate(expression);
541    }
542
543    public Expression getPreMove() {
544        return preMove;
545    }
546
547    /**
548     * Expression (such as File Language) used to dynamically set the filename when moving it before processing.
549     * For example to move in-progress files into the order directory set this value to order.
550     */
551    public void setPreMove(Expression preMove) {
552        this.preMove = preMove;
553    }
554
555    public void setPreMove(String fileLanguageExpression) {
556        String expression = configureMoveOrPreMoveExpression(fileLanguageExpression);
557        this.preMove = createFileLanguageExpression(expression);
558    }
559
560    public Expression getMoveExisting() {
561        return moveExisting;
562    }
563
564    /**
565     * Expression (such as File Language) used to compute file name to use when fileExist=Move is configured.
566     * To move files into a backup subdirectory just enter backup.
567     * This option only supports the following File Language tokens: "file:name", "file:name.ext", "file:name.noext", "file:onlyname",
568     * "file:onlyname.noext", "file:ext", and "file:parent". Notice the "file:parent" is not supported by the FTP component,
569     * as the FTP component can only move any existing files to a relative directory based on current dir as base.
570     */
571    public void setMoveExisting(Expression moveExisting) {
572        this.moveExisting = moveExisting;
573    }
574
575    public void setMoveExisting(String fileLanguageExpression) {
576        String expression = configureMoveOrPreMoveExpression(fileLanguageExpression);
577        this.moveExisting = createFileLanguageExpression(expression);
578    }
579
580    public Expression getFileName() {
581        return fileName;
582    }
583
584    /**
585     * Use Expression such as File Language to dynamically set the filename.
586     * For consumers, it's used as a filename filter.
587     * For producers, it's used to evaluate the filename to write.
588     * If an expression is set, it take precedence over the CamelFileName header. (Note: The header itself can also be an Expression).
589     * The expression options support both String and Expression types.
590     * If the expression is a String type, it is always evaluated using the File Language.
591     * If the expression is an Expression type, the specified Expression type is used - this allows you,
592     * for instance, to use OGNL expressions. For the consumer, you can use it to filter filenames,
593     * so you can for instance consume today's file using the File Language syntax: mydata-${date:now:yyyyMMdd}.txt.
594     * The producers support the CamelOverruleFileName header which takes precedence over any existing CamelFileName header;
595     * the CamelOverruleFileName is a header that is used only once, and makes it easier as this avoids to temporary
596     * store CamelFileName and have to restore it afterwards.
597     */
598    public void setFileName(Expression fileName) {
599        this.fileName = fileName;
600    }
601
602    public void setFileName(String fileLanguageExpression) {
603        this.fileName = createFileLanguageExpression(fileLanguageExpression);
604    }
605
606    public String getDoneFileName() {
607        return doneFileName;
608    }
609
610    /**
611     * Producer: If provided, then Camel will write a 2nd done file when the original file has been written.
612     * The done file will be empty. This option configures what file name to use.
613     * Either you can specify a fixed name. Or you can use dynamic placeholders.
614     * The done file will always be written in the same folder as the original file.
615     * <p/>
616     * Consumer: If provided, Camel will only consume files if a done file exists.
617     * This option configures what file name to use. Either you can specify a fixed name.
618     * Or you can use dynamic placeholders.The done file is always expected in the same folder
619     * as the original file.
620     * <p/>
621     * Only ${file.name} and ${file.name.noext} is supported as dynamic placeholders.
622     */
623    public void setDoneFileName(String doneFileName) {
624        this.doneFileName = doneFileName;
625    }
626
627    public Boolean isIdempotent() {
628        return idempotent != null ? idempotent : false;
629    }
630
631    public String getCharset() {
632        return charset;
633    }
634
635    /**
636     * This option is used to specify the encoding of the file.
637     * You can use this on the consumer, to specify the encodings of the files, which allow Camel to know the charset
638     * it should load the file content in case the file content is being accessed.
639     * Likewise when writing a file, you can use this option to specify which charset to write the file as well.
640     * Do mind that when writing the file Camel may have to read the message content into memory to be able to
641     * convert the data into the configured charset, so do not use this if you have big messages.
642     */
643    public void setCharset(String charset) {
644        IOHelper.validateCharset(charset);
645        this.charset = charset;
646    }
647
648    protected boolean isIdempotentSet() {
649        return idempotent != null;
650    }
651
652    /**
653     * Option to use the Idempotent Consumer EIP pattern to let Camel skip already processed files.
654     * Will by default use a memory based LRUCache that holds 1000 entries. If noop=true then idempotent will be enabled
655     * as well to avoid consuming the same files over and over again.
656     */
657    public void setIdempotent(Boolean idempotent) {
658        this.idempotent = idempotent;
659    }
660
661    public Expression getIdempotentKey() {
662        return idempotentKey;
663    }
664
665    /**
666     * To use a custom idempotent key. By default the absolute path of the file is used.
667     * You can use the File Language, for example to use the file name and file size, you can do:
668     * <tt>idempotentKey=${file:name}-${file:size}</tt>
669     */
670    public void setIdempotentKey(Expression idempotentKey) {
671        this.idempotentKey = idempotentKey;
672    }
673
674    public void setIdempotentKey(String expression) {
675        this.idempotentKey = createFileLanguageExpression(expression);
676    }
677
678    public IdempotentRepository<String> getIdempotentRepository() {
679        return idempotentRepository;
680    }
681
682    /**
683     * A pluggable repository org.apache.camel.spi.IdempotentRepository which by default use MemoryMessageIdRepository
684     * if none is specified and idempotent is true.
685     */
686    public void setIdempotentRepository(IdempotentRepository<String> idempotentRepository) {
687        this.idempotentRepository = idempotentRepository;
688    }
689
690    public GenericFileFilter<T> getFilter() {
691        return filter;
692    }
693
694    /**
695     * Pluggable filter as a org.apache.camel.component.file.GenericFileFilter class.
696     * Will skip files if filter returns false in its accept() method.
697     */
698    public void setFilter(GenericFileFilter<T> filter) {
699        this.filter = filter;
700    }
701
702    public Comparator<GenericFile<T>> getSorter() {
703        return sorter;
704    }
705
706    /**
707     * Pluggable sorter as a java.util.Comparator<org.apache.camel.component.file.GenericFile> class.
708     */
709    public void setSorter(Comparator<GenericFile<T>> sorter) {
710        this.sorter = sorter;
711    }
712
713    public Comparator<Exchange> getSortBy() {
714        return sortBy;
715    }
716
717    /**
718     * Built-in sort by using the File Language.
719     * Supports nested sorts, so you can have a sort by file name and as a 2nd group sort by modified date.
720     */
721    public void setSortBy(Comparator<Exchange> sortBy) {
722        this.sortBy = sortBy;
723    }
724
725    public void setSortBy(String expression) {
726        setSortBy(expression, false);
727    }
728
729    public void setSortBy(String expression, boolean reverse) {
730        setSortBy(GenericFileDefaultSorter.sortByFileLanguage(getCamelContext(), expression, reverse));
731    }
732
733    public boolean isShuffle() {
734        return shuffle;
735    }
736
737    /**
738     * To shuffle the list of files (sort in random order)
739     */
740    public void setShuffle(boolean shuffle) {
741        this.shuffle = shuffle;
742    }
743
744    public String getTempPrefix() {
745        return tempPrefix;
746    }
747
748    /**
749     * This option is used to write the file using a temporary name and then, after the write is complete,
750     * rename it to the real name. Can be used to identify files being written and also avoid consumers
751     * (not using exclusive read locks) reading in progress files. Is often used by FTP when uploading big files.
752     */
753    public void setTempPrefix(String tempPrefix) {
754        this.tempPrefix = tempPrefix;
755        // use only name as we set a prefix in from on the name
756        setTempFileName(tempPrefix + "${file:onlyname}");
757    }
758
759    public Expression getTempFileName() {
760        return tempFileName;
761    }
762
763    /**
764     * The same as tempPrefix option but offering a more fine grained control on the naming of the temporary filename as it uses the File Language.
765     */
766    public void setTempFileName(Expression tempFileName) {
767        this.tempFileName = tempFileName;
768    }
769
770    public void setTempFileName(String tempFileNameExpression) {
771        this.tempFileName = createFileLanguageExpression(tempFileNameExpression);
772    }
773
774    public boolean isEagerDeleteTargetFile() {
775        return eagerDeleteTargetFile;
776    }
777
778    /**
779     * Whether or not to eagerly delete any existing target file.
780     * This option only applies when you use fileExists=Override and the tempFileName option as well.
781     * You can use this to disable (set it to false) deleting the target file before the temp file is written.
782     * For example you may write big files and want the target file to exists during the temp file is being written.
783     * This ensure the target file is only deleted until the very last moment, just before the temp file is being
784     * renamed to the target filename. This option is also used to control whether to delete any existing files when
785     * fileExist=Move is enabled, and an existing file exists.
786     * If this option copyAndDeleteOnRenameFails false, then an exception will be thrown if an existing file existed,
787     * if its true, then the existing file is deleted before the move operation.
788     */
789    public void setEagerDeleteTargetFile(boolean eagerDeleteTargetFile) {
790        this.eagerDeleteTargetFile = eagerDeleteTargetFile;
791    }
792
793    public GenericFileConfiguration getConfiguration() {
794        if (configuration == null) {
795            configuration = new GenericFileConfiguration();
796        }
797        return configuration;
798    }
799
800    public void setConfiguration(GenericFileConfiguration configuration) {
801        this.configuration = configuration;
802    }
803
804    public GenericFileExclusiveReadLockStrategy<T> getExclusiveReadLockStrategy() {
805        return exclusiveReadLockStrategy;
806    }
807
808    /**
809     * Pluggable read-lock as a org.apache.camel.component.file.GenericFileExclusiveReadLockStrategy implementation.
810     */
811    public void setExclusiveReadLockStrategy(GenericFileExclusiveReadLockStrategy<T> exclusiveReadLockStrategy) {
812        this.exclusiveReadLockStrategy = exclusiveReadLockStrategy;
813    }
814
815    public String getReadLock() {
816        return readLock;
817    }
818
819    /**
820     * Used by consumer, to only poll the files if it has exclusive read-lock on the file (i.e. the file is not in-progress or being written).
821     * Camel will wait until the file lock is granted.
822     * <p/>
823     * This option provides the build in strategies:
824     * <ul>
825     *     <li>none - No read lock is in use
826     *     <li>markerFile - Camel creates a marker file (fileName.camelLock) and then holds a lock on it. This option is not available for the FTP component
827     *     <li>changed - Changed is using file length/modification timestamp to detect whether the file is currently being copied or not. Will at least use 1 sec
828     *     to determine this, so this option cannot consume files as fast as the others, but can be more reliable as the JDK IO API cannot
829     *     always determine whether a file is currently being used by another process. The option readLockCheckInterval can be used to set the check frequency.</li>
830     *     <li>fileLock - is for using java.nio.channels.FileLock. This option is not avail for the FTP component. This approach should be avoided when accessing
831     *     a remote file system via a mount/share unless that file system supports distributed file locks.</li>
832     *     <li>rename - rename is for using a try to rename the file as a test if we can get exclusive read-lock.</li>
833     *     <li>idempotent - (only for file component) idempotent is for using a idempotentRepository as the read-lock.
834     *     This allows to use read locks that supports clustering if the idempotent repository implementation supports that.</li>
835     *     <li>idempotent-changed - (only for file component) idempotent-changed is for using a idempotentRepository and changed as the combined read-lock.
836     *     This allows to use read locks that supports clustering if the idempotent repository implementation supports that.</li>
837     *     <li>idempotent-rename - (only for file component) idempotent-rename is for using a idempotentRepository and rename as the combined read-lock.
838     *     This allows to use read locks that supports clustering if the idempotent repository implementation supports that.</li>
839     * </ul>
840     * Notice: The various read locks is not all suited to work in clustered mode, where concurrent consumers on different nodes is competing
841     * for the same files on a shared file system. The markerFile using a close to atomic operation to create the empty marker file,
842     * but its not guaranteed to work in a cluster. The fileLock may work better but then the file system need to support distributed file locks, and so on.
843     * Using the idempotent read lock can support clustering if the idempotent repository supports clustering, such as Hazelcast Component or Infinispan.
844     */
845    public void setReadLock(String readLock) {
846        this.readLock = readLock;
847    }
848
849    public long getReadLockCheckInterval() {
850        return readLockCheckInterval;
851    }
852
853    /**
854     * Interval in millis for the read-lock, if supported by the read lock.
855     * This interval is used for sleeping between attempts to acquire the read lock.
856     * For example when using the changed read lock, you can set a higher interval period to cater for slow writes.
857     * The default of 1 sec. may be too fast if the producer is very slow writing the file.
858     * <p/>
859     * Notice: For FTP the default readLockCheckInterval is 5000.
860     * <p/>
861     * The readLockTimeout value must be higher than readLockCheckInterval, but a rule of thumb is to have a timeout
862     * that is at least 2 or more times higher than the readLockCheckInterval. This is needed to ensure that amble
863     * time is allowed for the read lock process to try to grab the lock before the timeout was hit.
864     */
865    public void setReadLockCheckInterval(long readLockCheckInterval) {
866        this.readLockCheckInterval = readLockCheckInterval;
867    }
868
869    public long getReadLockTimeout() {
870        return readLockTimeout;
871    }
872
873    /**
874     * Optional timeout in millis for the read-lock, if supported by the read-lock.
875     * If the read-lock could not be granted and the timeout triggered, then Camel will skip the file.
876     * At next poll Camel, will try the file again, and this time maybe the read-lock could be granted.
877     * Use a value of 0 or lower to indicate forever. Currently fileLock, changed and rename support the timeout.
878     * <p/>
879     * Notice: For FTP the default readLockTimeout value is 20000 instead of 10000.
880     * <p/>
881     * The readLockTimeout value must be higher than readLockCheckInterval, but a rule of thumb is to have a timeout
882     * that is at least 2 or more times higher than the readLockCheckInterval. This is needed to ensure that amble
883     * time is allowed for the read lock process to try to grab the lock before the timeout was hit.
884     */
885    public void setReadLockTimeout(long readLockTimeout) {
886        this.readLockTimeout = readLockTimeout;
887    }
888
889    public boolean isReadLockMarkerFile() {
890        return readLockMarkerFile;
891    }
892
893    /**
894     * Whether to use marker file with the changed, rename, or exclusive read lock types.
895     * By default a marker file is used as well to guard against other processes picking up the same files.
896     * This behavior can be turned off by setting this option to false.
897     * For example if you do not want to write marker files to the file systems by the Camel application.
898     */
899    public void setReadLockMarkerFile(boolean readLockMarkerFile) {
900        this.readLockMarkerFile = readLockMarkerFile;
901    }
902
903    public boolean isReadLockDeleteOrphanLockFiles() {
904        return readLockDeleteOrphanLockFiles;
905    }
906
907    /**
908     * Whether or not read lock with marker files should upon startup delete any orphan read lock files, which may
909     * have been left on the file system, if Camel was not properly shutdown (such as a JVM crash).
910     * <p/>
911     * If turning this option to <tt>false</tt> then any orphaned lock file will cause Camel to not attempt to pickup
912     * that file, this could also be due another node is concurrently reading files from the same shared directory.
913     */
914    public void setReadLockDeleteOrphanLockFiles(boolean readLockDeleteOrphanLockFiles) {
915        this.readLockDeleteOrphanLockFiles = readLockDeleteOrphanLockFiles;
916    }
917
918    public LoggingLevel getReadLockLoggingLevel() {
919        return readLockLoggingLevel;
920    }
921
922    /**
923     * Logging level used when a read lock could not be acquired.
924     * By default a WARN is logged.
925     * You can change this level, for example to OFF to not have any logging.
926     * This option is only applicable for readLock of types: changed, fileLock, idempotent, idempotent-changed, idempotent-rename, rename.
927     */
928    public void setReadLockLoggingLevel(LoggingLevel readLockLoggingLevel) {
929        this.readLockLoggingLevel = readLockLoggingLevel;
930    }
931
932    public long getReadLockMinLength() {
933        return readLockMinLength;
934    }
935
936    /**
937     * This option applied only for readLock=changed. This option allows you to configure a minimum file length.
938     * By default Camel expects the file to contain data, and thus the default value is 1.
939     * You can set this option to zero, to allow consuming zero-length files.
940     */
941    public void setReadLockMinLength(long readLockMinLength) {
942        this.readLockMinLength = readLockMinLength;
943    }
944
945    public long getReadLockMinAge() {
946        return readLockMinAge;
947    }
948
949    /**
950     * This option applied only for readLock=change.
951     * This option allows to specify a minimum age the file must be before attempting to acquire the read lock.
952     * For example use readLockMinAge=300s to require the file is at last 5 minutes old.
953     * This can speedup the changed read lock as it will only attempt to acquire files which are at least that given age.
954     */
955    public void setReadLockMinAge(long readLockMinAge) {
956        this.readLockMinAge = readLockMinAge;
957    }
958
959    public boolean isReadLockRemoveOnRollback() {
960        return readLockRemoveOnRollback;
961    }
962
963    /**
964     * This option applied only for readLock=idempotent.
965     * This option allows to specify whether to remove the file name entry from the idempotent repository
966     * when processing the file failed and a rollback happens.
967     * If this option is false, then the file name entry is confirmed (as if the file did a commit).
968     */
969    public void setReadLockRemoveOnRollback(boolean readLockRemoveOnRollback) {
970        this.readLockRemoveOnRollback = readLockRemoveOnRollback;
971    }
972
973    public boolean isReadLockRemoveOnCommit() {
974        return readLockRemoveOnCommit;
975    }
976
977    /**
978     * This option applied only for readLock=idempotent.
979     * This option allows to specify whether to remove the file name entry from the idempotent repository
980     * when processing the file is succeeded and a commit happens.
981     * <p/>
982     * By default the file is not removed which ensures that any race-condition do not occur so another active
983     * node may attempt to grab the file. Instead the idempotent repository may support eviction strategies
984     * that you can configure to evict the file name entry after X minutes - this ensures no problems with race conditions.
985     */
986    public void setReadLockRemoveOnCommit(boolean readLockRemoveOnCommit) {
987        this.readLockRemoveOnCommit = readLockRemoveOnCommit;
988    }
989
990    public int getBufferSize() {
991        return bufferSize;
992    }
993
994    /**
995     * Write buffer sized in bytes.
996     */
997    public void setBufferSize(int bufferSize) {
998        if (bufferSize <= 0) {
999            throw new IllegalArgumentException("BufferSize must be a positive value, was " + bufferSize);
1000        }
1001        this.bufferSize = bufferSize;
1002    }
1003
1004    public GenericFileExist getFileExist() {
1005        return fileExist;
1006    }
1007
1008    /**
1009     * What to do if a file already exists with the same name.
1010     *
1011     * Override, which is the default, replaces the existing file.
1012     * <ul>
1013     *   <li>Append - adds content to the existing file.</li>
1014     *   <li>Fail - throws a GenericFileOperationException, indicating that there is already an existing file.</li>
1015     *   <li>Ignore - silently ignores the problem and does not override the existing file, but assumes everything is okay.</li>
1016     *   <li>Move - option requires to use the moveExisting option to be configured as well.
1017     *   The option eagerDeleteTargetFile can be used to control what to do if an moving the file, and there exists already an existing file,
1018     *   otherwise causing the move operation to fail.
1019     *   The Move option will move any existing files, before writing the target file.</li>
1020     *   <li>TryRename is only applicable if tempFileName option is in use. This allows to try renaming the file from the temporary name to the actual name,
1021     *   without doing any exists check. This check may be faster on some file systems and especially FTP servers.</li>
1022     * </ul>
1023     */
1024    public void setFileExist(GenericFileExist fileExist) {
1025        this.fileExist = fileExist;
1026    }
1027
1028    public boolean isAutoCreate() {
1029        return autoCreate;
1030    }
1031
1032    /**
1033     * Automatically create missing directories in the file's pathname. For the file consumer, that means creating the starting directory.
1034     * For the file producer, it means the directory the files should be written to.
1035     */
1036    public void setAutoCreate(boolean autoCreate) {
1037        this.autoCreate = autoCreate;
1038    }
1039
1040    public boolean isStartingDirectoryMustExist() {
1041        return startingDirectoryMustExist;
1042    }
1043
1044    /**
1045     * Whether the starting directory must exist. Mind that the autoCreate option is default enabled,
1046     * which means the starting directory is normally auto created if it doesn't exist.
1047     * You can disable autoCreate and enable this to ensure the starting directory must exist. Will thrown an exception if the directory doesn't exist.
1048     */
1049    public void setStartingDirectoryMustExist(boolean startingDirectoryMustExist) {
1050        this.startingDirectoryMustExist = startingDirectoryMustExist;
1051    }
1052
1053    public boolean isDirectoryMustExist() {
1054        return directoryMustExist;
1055    }
1056
1057    /**
1058     * Similar to startingDirectoryMustExist but this applies during polling recursive sub directories.
1059     */
1060    public void setDirectoryMustExist(boolean directoryMustExist) {
1061        this.directoryMustExist = directoryMustExist;
1062    }
1063
1064    public GenericFileProcessStrategy<T> getProcessStrategy() {
1065        return processStrategy;
1066    }
1067
1068    /**
1069     * A pluggable org.apache.camel.component.file.GenericFileProcessStrategy allowing you to implement your own readLock option or similar.
1070     * Can also be used when special conditions must be met before a file can be consumed, such as a special ready file exists.
1071     * If this option is set then the readLock option does not apply.
1072     */
1073    public void setProcessStrategy(GenericFileProcessStrategy<T> processStrategy) {
1074        this.processStrategy = processStrategy;
1075    }
1076
1077    public String getLocalWorkDirectory() {
1078        return localWorkDirectory;
1079    }
1080
1081    /**
1082     * When consuming, a local work directory can be used to store the remote file content directly in local files,
1083     * to avoid loading the content into memory. This is beneficial, if you consume a very big remote file and thus can conserve memory.
1084     */
1085    public void setLocalWorkDirectory(String localWorkDirectory) {
1086        this.localWorkDirectory = localWorkDirectory;
1087    }
1088
1089    public int getMaxMessagesPerPoll() {
1090        return maxMessagesPerPoll;
1091    }
1092
1093    /**
1094     * To define a maximum messages to gather per poll.
1095     * By default no maximum is set. Can be used to set a limit of e.g. 1000 to avoid when starting up the server that there are thousands of files.
1096     * Set a value of 0 or negative to disabled it.
1097     * Notice: If this option is in use then the File and FTP components will limit before any sorting.
1098     * For example if you have 100000 files and use maxMessagesPerPoll=500, then only the first 500 files will be picked up, and then sorted.
1099     * You can use the eagerMaxMessagesPerPoll option and set this to false to allow to scan all files first and then sort afterwards.
1100     */
1101    public void setMaxMessagesPerPoll(int maxMessagesPerPoll) {
1102        this.maxMessagesPerPoll = maxMessagesPerPoll;
1103    }
1104
1105    public boolean isEagerMaxMessagesPerPoll() {
1106        return eagerMaxMessagesPerPoll;
1107    }
1108
1109    /**
1110     * Allows for controlling whether the limit from maxMessagesPerPoll is eager or not.
1111     * If eager then the limit is during the scanning of files. Where as false would scan all files, and then perform sorting.
1112     * Setting this option to false allows for sorting all files first, and then limit the poll. Mind that this requires a
1113     * higher memory usage as all file details are in memory to perform the sorting.
1114     */
1115    public void setEagerMaxMessagesPerPoll(boolean eagerMaxMessagesPerPoll) {
1116        this.eagerMaxMessagesPerPoll = eagerMaxMessagesPerPoll;
1117    }
1118
1119    public int getMaxDepth() {
1120        return maxDepth;
1121    }
1122
1123    /**
1124     * The maximum depth to traverse when recursively processing a directory.
1125     */
1126    public void setMaxDepth(int maxDepth) {
1127        this.maxDepth = maxDepth;
1128    }
1129
1130    public int getMinDepth() {
1131        return minDepth;
1132    }
1133
1134    /**
1135     * The minimum depth to start processing when recursively processing a directory.
1136     * Using minDepth=1 means the base directory. Using minDepth=2 means the first sub directory.
1137     */
1138    public void setMinDepth(int minDepth) {
1139        this.minDepth = minDepth;
1140    }
1141
1142    public IdempotentRepository<String> getInProgressRepository() {
1143        return inProgressRepository;
1144    }
1145
1146    /**
1147     * A pluggable in-progress repository org.apache.camel.spi.IdempotentRepository.
1148     * The in-progress repository is used to account the current in progress files being consumed. By default a memory based repository is used.
1149     */
1150    public void setInProgressRepository(IdempotentRepository<String> inProgressRepository) {
1151        this.inProgressRepository = inProgressRepository;
1152    }
1153
1154    public boolean isKeepLastModified() {
1155        return keepLastModified;
1156    }
1157
1158    /**
1159     * Will keep the last modified timestamp from the source file (if any).
1160     * Will use the Exchange.FILE_LAST_MODIFIED header to located the timestamp.
1161     * This header can contain either a java.util.Date or long with the timestamp.
1162     * If the timestamp exists and the option is enabled it will set this timestamp on the written file.
1163     * Note: This option only applies to the file producer. You cannot use this option with any of the ftp producers.
1164     */
1165    public void setKeepLastModified(boolean keepLastModified) {
1166        this.keepLastModified = keepLastModified;
1167    }
1168
1169    public boolean isAllowNullBody() {
1170        return allowNullBody;
1171    }
1172
1173    /**
1174     * Used to specify if a null body is allowed during file writing.
1175     * If set to true then an empty file will be created, when set to false, and attempting to send a null body to the file component,
1176     * a GenericFileWriteException of 'Cannot write null body to file.' will be thrown.
1177     * If the `fileExist` option is set to 'Override', then the file will be truncated, and if set to `append` the file will remain unchanged.
1178     */
1179    public void setAllowNullBody(boolean allowNullBody) {
1180        this.allowNullBody = allowNullBody;
1181    }
1182
1183    public boolean isJailStartingDirectory() {
1184        return jailStartingDirectory;
1185    }
1186
1187    /**
1188     * Used for jailing (restricting) writing files to the starting directory (and sub) only.
1189     * This is enabled by default to not allow Camel to write files to outside directories (to be more secured out of the box).
1190     * You can turn this off to allow writing files to directories outside the starting directory, such as parent or root folders.
1191     */
1192    public void setJailStartingDirectory(boolean jailStartingDirectory) {
1193        this.jailStartingDirectory = jailStartingDirectory;
1194    }
1195
1196    public ExceptionHandler getOnCompletionExceptionHandler() {
1197        return onCompletionExceptionHandler;
1198    }
1199
1200    /**
1201     * To use a custom {@link org.apache.camel.spi.ExceptionHandler} to handle any thrown exceptions that happens
1202     * during the file on completion process where the consumer does either a commit or rollback. The default
1203     * implementation will log any exception at WARN level and ignore.
1204     */
1205    public void setOnCompletionExceptionHandler(ExceptionHandler onCompletionExceptionHandler) {
1206        this.onCompletionExceptionHandler = onCompletionExceptionHandler;
1207    }
1208
1209    /**
1210     * Configures the given message with the file which sets the body to the
1211     * file object.
1212     */
1213    public void configureMessage(GenericFile<T> file, Message message) {
1214        message.setBody(file);
1215
1216        if (flatten) {
1217            // when flatten the file name should not contain any paths
1218            message.setHeader(Exchange.FILE_NAME, file.getFileNameOnly());
1219        } else {
1220            // compute name to set on header that should be relative to starting directory
1221            String name = file.isAbsolute() ? file.getAbsoluteFilePath() : file.getRelativeFilePath();
1222
1223            // skip leading endpoint configured directory
1224            String endpointPath = getConfiguration().getDirectory() + getFileSeparator();
1225
1226            // need to normalize paths to ensure we can match using startsWith
1227            endpointPath = FileUtil.normalizePath(endpointPath);
1228            String copyOfName = FileUtil.normalizePath(name);
1229            if (ObjectHelper.isNotEmpty(endpointPath) && copyOfName.startsWith(endpointPath)) {
1230                name = name.substring(endpointPath.length());
1231            }
1232
1233            // adjust filename
1234            message.setHeader(Exchange.FILE_NAME, name);
1235        }
1236    }
1237
1238    /**
1239     * Set up the exchange properties with the options of the file endpoint
1240     */
1241    public void configureExchange(Exchange exchange) {
1242        // Now we just set the charset property here
1243        if (getCharset() != null) {
1244            exchange.setProperty(Exchange.CHARSET_NAME, getCharset());
1245        }
1246    }
1247
1248    /**
1249     * Strategy to configure the move, preMove, or moveExisting option based on a String input.
1250     *
1251     * @param expression the original string input
1252     * @return configured string or the original if no modifications is needed
1253     */
1254    protected String configureMoveOrPreMoveExpression(String expression) {
1255        // if the expression already have ${ } placeholders then pass it unmodified
1256        if (StringHelper.hasStartToken(expression, "simple")) {
1257            return expression;
1258        }
1259
1260        // remove trailing slash
1261        expression = FileUtil.stripTrailingSeparator(expression);
1262
1263        StringBuilder sb = new StringBuilder();
1264
1265        // if relative then insert start with the parent folder
1266        if (!isAbsolute(expression)) {
1267            sb.append("${file:parent}");
1268            sb.append(getFileSeparator());
1269        }
1270        // insert the directory the end user provided
1271        sb.append(expression);
1272        // append only the filename (file:name can contain a relative path, so we must use onlyname)
1273        sb.append(getFileSeparator());
1274        sb.append("${file:onlyname}");
1275
1276        return sb.toString();
1277    }
1278
1279    protected Map<String, Object> getParamsAsMap() {
1280        Map<String, Object> params = new HashMap<String, Object>();
1281
1282        if (isNoop()) {
1283            params.put("noop", Boolean.toString(true));
1284        }
1285        if (isDelete()) {
1286            params.put("delete", Boolean.toString(true));
1287        }
1288        if (move != null) {
1289            params.put("move", move);
1290        }
1291        if (moveFailed != null) {
1292            params.put("moveFailed", moveFailed);
1293        }
1294        if (preMove != null) {
1295            params.put("preMove", preMove);
1296        }
1297        if (exclusiveReadLockStrategy != null) {
1298            params.put("exclusiveReadLockStrategy", exclusiveReadLockStrategy);
1299        }
1300        if (readLock != null) {
1301            params.put("readLock", readLock);
1302        }
1303        if ("idempotent".equals(readLock) || "idempotent-changed".equals(readLock) || "idempotent-rename".equals(readLock)) {
1304            params.put("readLockIdempotentRepository", idempotentRepository);
1305        }
1306        if (readLockCheckInterval > 0) {
1307            params.put("readLockCheckInterval", readLockCheckInterval);
1308        }
1309        if (readLockTimeout > 0) {
1310            params.put("readLockTimeout", readLockTimeout);
1311        }
1312        params.put("readLockMarkerFile", readLockMarkerFile);
1313        params.put("readLockDeleteOrphanLockFiles", readLockDeleteOrphanLockFiles);
1314        params.put("readLockMinLength", readLockMinLength);
1315        params.put("readLockLoggingLevel", readLockLoggingLevel);
1316        params.put("readLockMinAge", readLockMinAge);
1317        params.put("readLockRemoveOnRollback", readLockRemoveOnRollback);
1318        params.put("readLockRemoveOnCommit", readLockRemoveOnCommit);
1319        return params;
1320    }
1321
1322    private Expression createFileLanguageExpression(String expression) {
1323        Language language;
1324        // only use file language if the name is complex (eg. using $)
1325        if (expression.contains("$")) {
1326            language = getCamelContext().resolveLanguage("file");
1327        } else {
1328            language = getCamelContext().resolveLanguage("constant");
1329        }
1330        return language.createExpression(expression);
1331    }
1332
1333    private Predicate createFileLanguagePredicate(String expression) {
1334        Language language = getCamelContext().resolveLanguage("file");
1335        return language.createPredicate(expression);
1336    }
1337
1338    /**
1339     * Creates the associated name of the done file based on the given file name.
1340     * <p/>
1341     * This method should only be invoked if a done filename property has been set on this endpoint.
1342     *
1343     * @param fileName the file name
1344     * @return name of the associated done file name
1345     */
1346    protected String createDoneFileName(String fileName) {
1347        String pattern = getDoneFileName();
1348        ObjectHelper.notEmpty(pattern, "doneFileName", pattern);
1349
1350        // we only support ${file:name} or ${file:name.noext} as dynamic placeholders for done files
1351        String path = FileUtil.onlyPath(fileName);
1352        String onlyName = Matcher.quoteReplacement(FileUtil.stripPath(fileName));
1353
1354        pattern = pattern.replaceFirst("\\$\\{file:name\\}", onlyName);
1355        pattern = pattern.replaceFirst("\\$simple\\{file:name\\}", onlyName);
1356        pattern = pattern.replaceFirst("\\$\\{file:name.noext\\}", FileUtil.stripExt(onlyName));
1357        pattern = pattern.replaceFirst("\\$simple\\{file:name.noext\\}", FileUtil.stripExt(onlyName));
1358
1359        // must be able to resolve all placeholders supported
1360        if (StringHelper.hasStartToken(pattern, "simple")) {
1361            throw new ExpressionIllegalSyntaxException(fileName + ". Cannot resolve reminder: " + pattern);
1362        }
1363
1364        String answer = pattern;
1365        if (ObjectHelper.isNotEmpty(path) && ObjectHelper.isNotEmpty(pattern)) {
1366            // done file must always be in same directory as the real file name
1367            answer = path + getFileSeparator() + pattern;
1368        }
1369
1370        if (getConfiguration().needToNormalize()) {
1371            // must normalize path to cater for Windows and other OS
1372            answer = FileUtil.normalizePath(answer);
1373        }
1374
1375        return answer;
1376    }
1377
1378    /**
1379     * Is the given file a done file?
1380     * <p/>
1381     * This method should only be invoked if a done filename property has been set on this endpoint.
1382     *
1383     * @param fileName the file name
1384     * @return <tt>true</tt> if its a done file, <tt>false</tt> otherwise
1385     */
1386    protected boolean isDoneFile(String fileName) {
1387        String pattern = getDoneFileName();
1388        ObjectHelper.notEmpty(pattern, "doneFileName", pattern);
1389
1390        if (!StringHelper.hasStartToken(pattern, "simple")) {
1391            // no tokens, so just match names directly
1392            return pattern.equals(fileName);
1393        }
1394
1395        // the static part of the pattern, is that a prefix or suffix?
1396        // its a prefix if ${ start token is not at the start of the pattern
1397        boolean prefix = pattern.indexOf("${") > 0;
1398
1399        // remove dynamic parts of the pattern so we only got the static part left
1400        pattern = pattern.replaceFirst("\\$\\{file:name\\}", "");
1401        pattern = pattern.replaceFirst("\\$simple\\{file:name\\}", "");
1402        pattern = pattern.replaceFirst("\\$\\{file:name.noext\\}", "");
1403        pattern = pattern.replaceFirst("\\$simple\\{file:name.noext\\}", "");
1404
1405        // must be able to resolve all placeholders supported
1406        if (StringHelper.hasStartToken(pattern, "simple")) {
1407            throw new ExpressionIllegalSyntaxException(fileName + ". Cannot resolve reminder: " + pattern);
1408        }
1409
1410        if (prefix) {
1411            return fileName.startsWith(pattern);
1412        } else {
1413            return fileName.endsWith(pattern);
1414        }
1415    }
1416
1417    @Override
1418    protected void doStart() throws Exception {
1419        // validate that the read lock options is valid for the process strategy
1420        if (!"none".equals(readLock) && !"off".equals(readLock)) {
1421            if (readLockTimeout > 0 && readLockTimeout <= readLockCheckInterval) {
1422                throw new IllegalArgumentException("The option readLockTimeout must be higher than readLockCheckInterval"
1423                        + ", was readLockTimeout=" + readLockTimeout + ", readLockCheckInterval=" + readLockCheckInterval
1424                        + ". A good practice is to let the readLockTimeout be at least 3 times higher than the readLockCheckInterval"
1425                        + " to ensure that the read lock procedure has enough time to acquire the lock.");
1426            }
1427        }
1428        if ("idempotent".equals(readLock) && idempotentRepository == null) {
1429            throw new IllegalArgumentException("IdempotentRepository must be configured when using readLock=idempotent");
1430        }
1431
1432        if (antInclude != null) {
1433            if (antFilter == null) {
1434                antFilter = new AntPathMatcherGenericFileFilter<>();
1435            }
1436            antFilter.setIncludes(antInclude);
1437        }
1438        if (antExclude != null) {
1439            if (antFilter == null) {
1440                antFilter = new AntPathMatcherGenericFileFilter<>();
1441            }
1442            antFilter.setExcludes(antExclude);
1443        }
1444        if (antFilter != null) {
1445            antFilter.setCaseSensitive(antFilterCaseSensitive);
1446        }
1447
1448        // idempotent repository may be used by others, so add it as a service so its stopped when CamelContext stops
1449        if (idempotentRepository != null) {
1450            getCamelContext().addService(idempotentRepository, true);
1451        }
1452        ServiceHelper.startServices(inProgressRepository);
1453        super.doStart();
1454    }
1455
1456    @Override
1457    protected void doStop() throws Exception {
1458        super.doStop();
1459        ServiceHelper.stopServices(inProgressRepository);
1460    }
1461}