LuceneIndexAppender

log4lucene.indexlocation=./index
log4j.appender.LUCENE=com.manheim.log4lucene.appender.LuceneIndexAppender 

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Properties;

import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;

/**
 * Simple Log4J Appender that writes LoggingEvent items to a Lucene
 * index (index must be specified in log4j.properties as "log4lucene.indexlocation").
 * 
 * 
 * @author ccollins
 *
 */
public class LuceneIndexAppender extends AppenderSkeleton
{

    private static final Logger LOG = Logger.getLogger(LuceneIndexAppender.class);
    private static final int BUFFER_THRESHOLD = 100;  // tweak this
    private static final long OPTIMIZE_THRESHOLD = 600000;

    private static ArrayList buffer = new ArrayList();
    private static String indexLocation = null;
    private static long optimizeTimestamp = System.currentTimeMillis(); 
    
    static
    {
        LuceneIndexAppender.establishIndexLocation();
    }
  

    /**
     * Main as test/example driver.
     * 
     * @param args
     */
    public static void main(String[] args)
    {
        System.out.println("here we go, check the logs and the index . . . ");
        for (int i = LuceneIndexAppender.BUFFER_THRESHOLD; i > 0; i--)
        {
            LOG.debug("log test iteration - " + i);
        }
    }

    /**
     * Return the value of log4j.properties element "log4lucene.indexlocation".  
     * 
     * TODO - somehow enhance this to get property in a standard Log4J manner such
     * that if user is using different named property file, or XML file, etc, still
     * retrieves the setting. 
     * 
     * @return
     */
    private static void establishIndexLocation()
    {
        LOG.debug("establishIndexLocation invoked");

        // INDEX_LOCATION
        Properties props = new Properties();
        try
        {
            URL configURL = LuceneIndexAppender.class.getClassLoader().getResource("log4j.properties");
            FileInputStream fis = new FileInputStream(configURL.getFile());
            props.load(fis);
            fis.close();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        String indexLoc = props.getProperty("log4lucene.indexlocation");
                
        if ((indexLoc == null) || (indexLoc.length() == 0))
        {
            System.err
                    .println("log4lucene.indexlocation NOT SET in log4j.properties (or cannot locate log4j.properties)");
        }
        
        LuceneIndexAppender.indexLocation = indexLoc;        
    }

    /**
     * Return File representing the index directory specified in the log4j.propertes 
     * log4lucene.indexlocation value.  If the directory does not exist then attempt
     * to create it and attempt to Lucene initialize it.  
     * 
     * @return
     */
    private static File getIndexDir()
    {
        File indexDir = new File(LuceneIndexAppender.indexLocation);
        if (!indexDir.exists())
        {
            boolean make = indexDir.mkdirs();
            if (!make)
            {
                System.err.println("log4lucene attempted to create indexlocation and failed - "
                        + LuceneIndexAppender.indexLocation);
                indexDir = null;
            }
            else
            {
                try
                {
                    // after directory created then setup Lucene index itself (with "true" create call to IndexWriter)
                    IndexWriter writer = new IndexWriter(indexDir, new StandardAnalyzer(), true);
                    writer.close();
                }
                catch (IOException e)
                {
                    e.printStackTrace();
                }
            }
        }
        else if ((!indexDir.isDirectory()) || (!indexDir.canRead() || (!indexDir.canWrite())))
        {
            System.err.println("log4lucene cannot access indexlocation (must be directory and have rw perms) - "
                    + LuceneIndexAppender.indexLocation);
            indexDir = null;
        }
        return indexDir;
    }

    /**
     * Flush BUFFER log elements to Lucene index (invoked when BUFFER_MAX is met). 
     *
     */
    private static void flushBuffer()
    {
        IndexWriter writer = null;
        try
        {
            if ((LuceneIndexAppender.buffer != null) && (LuceneIndexAppender.buffer.size() > 0))
            {
                File indexDir = LuceneIndexAppender.getIndexDir();
                if ((indexDir.isDirectory()) && (indexDir.canWrite()))
                {
                    int size = LuceneIndexAppender.buffer.size();

                    // write to index
                    writer = new IndexWriter(indexDir, new StandardAnalyzer(), false);
                    writer.setMaxFieldLength(1000);
                    for (int i = 0; i < size; i++)
                    {
                        Document logEventDoc = (Document) LuceneIndexAppender.buffer.get(i);
                        writer.addDocument(logEventDoc);
                    }
                    
                    // optimize every X seconds 
                    long optimizeCheck = optimizeTimestamp + OPTIMIZE_THRESHOLD;
                    if (System.currentTimeMillis() >= optimizeCheck)
                    {
                        optimizeTimestamp = System.currentTimeMillis();
                        writer.optimize();                        
                    }                    
                    
                    writer.close();

                    // clean out buffer
                    LuceneIndexAppender.buffer.clear();
                }
                else
                {
                    System.err.println("log4lucene cannot read/write to specified index directory - "
                            + LuceneIndexAppender.indexLocation);
                }
            }
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
    }

    /**
     * Append for Log4J AppenderSkeleton interface.
     * 
     */
    protected void append(LoggingEvent event)
    {
        Document logEventDoc = new Document();

        // date as a lucene construct via DateTools            
        Date logDate = new Date(event.timeStamp);
        String logDateString = DateTools.dateToString(logDate, DateTools.Resolution.SECOND);

        logEventDoc.add(new Field("category", event.getLoggerName(), Field.Store.YES, Field.Index.UN_TOKENIZED));
        logEventDoc.add(new Field("date", logDateString, Field.Store.YES, Field.Index.UN_TOKENIZED));
        logEventDoc.add(new Field("priority", event.getLevel().toString(), Field.Store.YES, Field.Index.UN_TOKENIZED));
        logEventDoc.add(new Field("message", event.getRenderedMessage(), Field.Store.YES, Field.Index.TOKENIZED));
        logEventDoc.add(new Field("thread", event.getThreadName(), Field.Store.YES, Field.Index.UN_TOKENIZED));
        
        // TODO NDC and MDC
        
        LuceneIndexAppender.buffer.add(logEventDoc);

        if (LuceneIndexAppender.buffer.size() >= LuceneIndexAppender.BUFFER_THRESHOLD)
        {
            LuceneIndexAppender.flushBuffer();
        }
    }

    /**
     * RequiresLayout for Log4J AppenderSkeleton interface.
     * 
     */
    public boolean requiresLayout()
    {
        return false;
    }

    /**
     * Close for Log4J AppenderSkeleton interface.
     * 
     */
    public void close()
    {
    }

}