DailyFileAppender.java a Solution to a log4j Daily Appender with a Retention!

I use the Log4j DailyRollingFileAppender over many years and many projects but always end up controlling the retention via a Cron jobs. That's work well, but what if we can control the retention in the log4j configuration file? Wouldn't that be great? Well, that came up and I wanted to share my solution with you:

package org.cnci.util;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;

import org.apache.log4j.FileAppender;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;

/**
 * Provide AS IS, no warranty!
 * 
 * Special log appender for log4j. Adds the current date (ie. year-mon) to the
 * end of the file name, so that rolling on to the next log is simply a case of
 * starting a new one - no renaming of old logs.
 * 
 * ONLY support: yyyy (yearly) | yyyy-MM (monthly) | yyyy-MM-dd (daily) |
 * yyyy-ww (weekly) yyyy-MM-dd-a (midnight, midday) | yyyy-MM-dd-HH (hourly) |
 * yyyy-MM-dd-HH-mm (every minute)
 * 
 * An example log4j.properties (one log per day, retains 30 days of logs)
 * 
 * log4j.rootCategory=INFO, DAILY
 * log4j.appender.DAILY=org.cnci.util.DailyFileAppender
 * log4j.appender.DAILY.File=log/MyApplication.log
 * log4j.appender.DAILY.DatePattern=yyyy-MM-dd log4j.appender.DAILY.MaxLogs=30
 * log4j.appender.DAILY.layout=org.apache.log4j.PatternLayout
 * log4j.appender.DAILY.layout.ConversionPattern=%d %-5p %c @ %m%n
 * 
 * @author T.A. Nguyen
 **/
public class DailyFileAppender extends FileAppender {
  private static final String DEFAULT_DATE_PATTERN = "'.'yyyy-MM-dd";
  private static final String[] SUPPORTED_DATE_PATTERNS = {
    "yyyy" /** (yearly) **/, 
    "yyyy-MM" /** (monthly) **/, 
    "yyyy-MM-dd" /** (daily) **/,
    "yyyy-ww" /** (weekly) **/,
    "yyyy-MM-dd-a" /** (midnight, midday) **/,
    "yyyy-MM-dd-HH" /** (hourly) **/,
    "yyyy-MM-dd-HH-mm" /** (every minute) **/
  };

  /**
   * Used internally and contains the name of the date derived from current
   * system date.
   **/
  private Date now = new Date();
  private String datePattern;
  private SimpleDateFormat formater;
  private int maxLogs = 0;
  private String scheduledFileName;

  /**
   * Default constructor. This is required as the appender class is dynamically
   * loaded.
   **/
  public DailyFileAppender() {
    super();
  }

  /**
   * computing for the scheduledFileName and set the now
   **/
  @Override
  public void activateOptions() {
    super.activateOptions();
    if (datePattern != null && fileName != null) {
      now.setTime(System.currentTimeMillis());
      formater = new SimpleDateFormat(datePattern);
      File file = new File(fileName);
      scheduledFileName = fileName + formater.format(new Date(file.lastModified()));
    } else {
      LogLog.error("Either File or DatePattern options are not set for appender [" + name + "].");
    }
  }

  public String getDatePattern() {
    return this.datePattern;
  }

  public void setDatePattern(String datePattern) {
    // make sure the the pattern conform to what we allow.
    this.datePattern = checkPattern(datePattern);
  }

  public int getMaxLogs() {
    return maxLogs;
  }

  public void setMaxLogs(int maxLogs) {
    this.maxLogs = maxLogs;
  }

  @Override
  protected void subAppend(LoggingEvent logEvent) {
    now.setTime(System.currentTimeMillis());

    try {
      rollOver();
    } catch (IOException IOEx) {
      LogLog.error("rollOver() failed!", IOEx);
    }

    super.subAppend(logEvent);
  }

  @Override
  public void setFile(String file) {
    try {
      // Fix the STUPID file path... Windows will use \ and *NIX will use
      // /,
      // so make sure we store the correct file path.
      File f = new File(file);
      file = f.getAbsolutePath();
    } catch (Exception e) {
      // ignore!
    }
    super.setFile(file);
  }

  void rollOver() throws IOException {
    /** Compute filename, but only if datePattern is specified **/
    if (datePattern == null) {
      // LogLog.error("Missing DatePattern option in rollOver().");
      errorHandler.error("Missing DatePattern option in rollOver().");
      return;
    }
    String datedFilename = fileName + formater.format(now);
    LogLog.debug(fileName + " ==> " + scheduledFileName + " ==> " + datedFilename);
    // It is too early to roll over because we are still within the
    // bounds of the current interval. Rollover will occur once the
    // next interval is reached.
    if (scheduledFileName.equals(datedFilename)) {
      return;
    }

    // close current file, and rename it to datedFilename
    this.closeFile();

    File file = new File(fileName);
    File target = new File(fileName + formater.format(new Date(file.lastModified())));
    if (target.exists()) {
      target.delete();
    }

    boolean result = file.renameTo(target);
    if (result) {
      // LogLog.debug(fileName + " -> " + scheduledFileName);
      scheduledFileName = datedFilename;
    } else {
      LogLog.error("Failed to rename [" + fileName + "] to [" + scheduledFileName + "].");
    }

    try {
      // This will also close the file. This is OK since multiple
      // close operations are safe.
      this.setFile(fileName, true, this.bufferedIO, this.bufferSize);
    } catch (IOException e) {
      // LogLog.error("setFile(" + fileName + ", true) call failed.");
      errorHandler.error("setFile(" + fileName + ", true) call failed.");
    }
    cleanupOldFiles();
  }

  /**
   * The helper function to validate the DatePattern.
   * 
   * @param pattern
   *          The DatePattern to be validated.
   * @return The validated date pattern or default DEFAULT_DATE_PATTERN
   **/
  private String checkPattern(String pattern) {
    if (isSupportedPattern(pattern)) {
      try {
        this.formater = new SimpleDateFormat(pattern);
        return pattern;
      } catch (Exception e) {
        LogLog.error("Invalid DatePattern " + pattern, e);
      }
    }
    return DEFAULT_DATE_PATTERN;
  }

  private boolean isSupportedPattern(String pattern) {
    if (pattern != null) {
      // give 'em that last chance to conform to our supported pattern.
      // Convert " " and ":" into "-" so that file operation will work
      // correctly!
      pattern = pattern.replaceAll(" |:", "-");

      for (String p : SUPPORTED_DATE_PATTERNS) {
        if (pattern.contains(p))
          return true;
      }
    }
    return false;
  }

  /**
   * Now delete those old files if needed.
   * 
   * @param pstrName
   *          The name of the new folder based on current system date.
   * @throws IOException
   **/
  static private boolean deletingFiles = false;

  private static synchronized void setDeletingFiles(boolean isDelete) {
    deletingFiles = isDelete;
  }

  private static synchronized boolean isDeletingFiles() {
    return deletingFiles;
  }

  boolean isExtendedLogFile(String name) {
    if (name == null)
      return false;
    File logMaster = new File(fileName);
    String master = logMaster.getName();
    return name.contains(master);
  }

  void cleanupOldFiles() {
    // If we need to delete log files
    if (maxLogs > 0 && !isDeletingFiles()) {
      // tell other thread, that we will do the clean up
      setDeletingFiles(true);

      try {
        // Array to hold the logs we are going to keep
        File[] logsToKeep = new File[maxLogs];

        // Get a 'master' file handle, and the parent directory from it
        File logMaster = new File(fileName);
        File logDir = logMaster.getParentFile();
        if (logDir.isDirectory()) {
          File[] logFiles = getLogFiles(logDir);
          // more files than expected, time to clean up!
          if (logFiles.length > maxLogs)
            for (File curLog : logFiles) {
              removeOldFile(logsToKeep, curLog);
            }
        }
      } catch (Exception e) {
        // ignore
        // LogLog.error("While deleting old files. " + e.getMessage(), e);
      } finally {
        // we are done with clean up.
        setDeletingFiles(false);
      }
    }
  }

  private void removeOldFile(File[] logsToKeep, File curLog) {
    String name = curLog.getAbsolutePath();

    // Check that the file is indeed one we want
    // (contains the master file name)
    if (name.indexOf(fileName) >= 0) {
      // Iterate through the array of logs we are
      // keeping
      for (int i = 0; curLog != null && i < logsToKeep.length; i++) {
        // Have we exhausted the 'to keep' array?
        if (logsToKeep[i] == null) {
          // Empty space, retain this log file
          logsToKeep[i] = curLog;
          curLog = null;
        }
        // If the 'kept' file is older than the
        // current one
        else if (logsToKeep[i].getName().compareTo(curLog.getName()) < 0) {
          // Replace tested entry with current
          // file
          File temp = logsToKeep[i];
          logsToKeep[i] = curLog;
          curLog = temp;
        }
      }

      // If we have a 'current' entry at this point,
      // it's a log we don't want
      if (curLog != null) {
        curLog.delete();
      }
    }
  }

  private File[] getLogFiles(File logDir) {
    // Iterate all the files in that directory
    FilenameFilter filter = new FilenameFilter() {
      @Override
      public boolean accept(File dir, String name) {
        return isExtendedLogFile(name);
      }
    };
    File[] logFiles = logDir.listFiles(filter); // only get the
    // extended
    // files
    return logFiles;
  }
}

And here is the sample log4j.properties

#------------------------------------------------------------------------------
#  Provide AS IS No Warranty!
#  Author: T.A. Nguyen
#
#  Default log4j.properties file.  This should be in the class path
#
#    Possible Log Levels:
#      FATAL, ERROR, WARN, INFO, DEBUG
#
# Production Setting:
#log4j.rootCategory=ERROR, DAILY
# Development and Build Setting:
log4j.rootCategory=DEBUG, DAILY, CONSOLE

#------------------------------------------------------------------------------
#
#  The following properties configure the console (stdout) appender.
#
#------------------------------------------------------------------------------
log4j.appender.CONSOLE = org.apache.log4j.ConsoleAppender
#log4j.appender.CONSOLE.Target=System.out
log4j.appender.CONSOLE.layout = org.apache.log4j.PatternLayout
log4j.appender.CONSOLE.layout.ConversionPattern = %d{ABSOLUTE} %5p %c{1}:%L - %m%n

#------------------------------------------------------------------------------
#
#  The following properties configure the Daily File appender.
#
#------------------------------------------------------------------------------
log4j.appender.DAILY = org.cnci.util.DailyFileAppender
log4j.appender.DAILY.File = log/MyApplication.log
log4j.appender.DAILY.MaxLogs = 30
log4j.appender.DAILY.Append = true
log4j.appender.DAILY.DatePattern = '.'yyy-MM-dd
log4j.appender.DAILY.layout = org.apache.log4j.PatternLayout
log4j.appender.DAILY.layout.ConversionPattern = %d{ABSOLUTE} %5p %c{1}:%L - %m%n

# Extracting all MyApplication logging into it own file
#log4j.logger.org.cnci=DEBUG, DAILY
#log4j.additivity.org.cnci=false

Double dereference for Ant Property

Just random note on solving the double dereference that ANT property will not allow.

  
 
    
    
    
    
     
    
       myApp     : ${org.cnci.firstapp}
       myCompany : ${firstCompany}
       myPackage : ${firstPackage}
       myClass   : ${firstClass}
    
 
 
 
    
    
    
       
    
 
 
 
    
    
    
    
     
    
       myApp     : ${org.cnci.secondapp}
       myCompany : ${secondCompany}
       myPackage : ${secondPackage}
       myClass   : ${secondClass}
    
 
 
 

And the output looks like this:

NOT-WORKING-double-dereferences:
     [echo] 
     [echo]         myApp     : SimpleIRC
     [echo]         myCompany : cnci
     [echo]         myPackage : org.cnci.firstapp
     [echo]         myClass   : ${org.${firstCompany}.firstapp}
     [echo]         
WORKING-double-dereferences:
     [echo] 
     [echo]         myApp     : AdvanceIRC
     [echo]         myCompany : cnci
     [echo]         myPackage : org.cnci.secondapp
     [echo]         myClass   : AdvanceIRC
     [echo]         

Simple trick to convert Date format with timezone in Java!

I am sure most of you got frustrated from the fact that SimpleDateFormat can not handle ISO8601 format. Here is my little trick to solve this nuisance.

Create a list of Know format you know that you will use for your application and apply SimpleDateFormat to the list. Now, in your formatDate() method, simply try all your known format and trap the Exception, then if still did not have a date, just use javax.xml.bind.DatatypeConverter.parseDateTime(date) to try it and see if that work.

Here is my DateUtils.java

package org.cnci.util;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.Locale;
public abstract class DateUtils {
    // XML formater: this only for converting date into XML string
    private static final SimpleDateFormat xmlFormater = 
                           new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
    // define all known date formats
    private static final SimpleDateFormat[] allFormats = new SimpleDateFormat[] {
        //new SimpleDateFormat("MM/dd/yyyy hh:mm:ss.SSS a"),
        new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"), // XSD 
        new SimpleDateFormat("MM/dd/yyyy HH:mm:ssZ"), // Oracle
        new SimpleDateFormat("EEE MMM d HH:mm:ss z yyyy"), 
        new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"), // rfc822
        new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z", Locale.US), 
        new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"), // XSD
        new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"), 
        new SimpleDateFormat("MM/dd/yyyy HH:mm:ss.SSS"),
        new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"), // Oracle
        new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"),
        new SimpleDateFormat("yyyy.MM.dd G 'at' HH:mm:ss z"),
        new SimpleDateFormat("yyyy.MM.dd GGG hh:mm aaa"),
        new SimpleDateFormat("yy-MM-dd HH:mm:ss.SSS"),
        new SimpleDateFormat("M/d/yyyy hh:mm:ss"), 
        new SimpleDateFormat("M/d/yyyy hh:mm a"), 
        new SimpleDateFormat("M/d/yy hh:mm:ss"), 
        new SimpleDateFormat("M/d/yy hh:mm a"),
        new SimpleDateFormat("M/d/yy HH:mm"),
        new SimpleDateFormat("M/d/yyyy HH:mm"),
        new SimpleDateFormat("M-d-yy HH:mm"), 
        new SimpleDateFormat("M-d-yyyy HH:mm"), 
        new SimpleDateFormat("M/d/yy"), 
        new SimpleDateFormat("M/d/yyyy"), 
        new SimpleDateFormat("M-d-yy"),
        new SimpleDateFormat("M-d-yyyy"), 
        new SimpleDateFormat("MMMM d, yyyyy"), 
        new SimpleDateFormat("MMM d, yyyyy") 
    };
    //-----------------------------------------------------------------------
    public static Date parseDate(final String date) throws ParseException {
        if (isEmptyString(date))
            throw new ParseException("Error: date is null or empty!", 0);
        // iterate over the array and parse
        Date myDate = null;
        if (date.endsWith("Z")) {
            myDate = parseXMLDate(date);
        }
        if (myDate == null) {
            for (DateFormat df : allFormats) {
                try {
                    myDate = df.parse(date);
                    return myDate;
                }
                catch (Exception e) {
                    // do nothing because we want to try the next
                    // format if current one fails
                }
            }
            if (myDate == null) {
                myDate = parseXMLDate(date);
            }
            // nothing returned so couldn't parse
            if (myDate == null) throw new ParseException("Error: unable to parse date: " + date, 0);
        }
        return myDate;
    }
    //Use the JAXB data type conversion, since it already know the xml valid date format
    private static java.util.Date parseXMLDate(final String date) {
        Calendar cal = javax.xml.bind.DatatypeConverter.parseDateTime(date);
        return cal.getTime();
    }
     
    //convert to the string representation of the datetime format
    //NOTES: this should be the same as formatDate() but keep 
    // and preferred to use this one instead.  Just in case the JAXB
    // document DatetypeConverter change in the future.
    public static String formatXMLDate(final java.util.Date date) {
        Calendar cal = Calendar.getInstance();
        cal.setTime(date);
        return javax.xml.bind.DatatypeConverter.printDateTime(cal);
    }
     
    // should yield the same results as formatXMLDate() method above.
    public static String formatDate(final java.util.Date date) {
        return xmlFormater.format(date);
    }
     
    public static boolean isEmptyString(String str)
    {
        if (str == null) return true;
        return "".equals(str.trim());
    }
}