/**************************************************************************
 OmegaT - Computer Assisted Translation (CAT) tool 
          with fuzzy matching, translation memory, keyword search, 
          glossaries, and translation leveraging into updated projects.
 
 Copyright (C) 2013 Aaron Madlon-Kay
               Home page: http://www.omegat.org/
               Support center: http://groups.yahoo.com/group/OmegaT/

 This file is part of OmegaT.

 OmegaT is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 OmegaT is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 **************************************************************************/
package org.omegat.tokenizer;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.en.EnglishAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.omegat.core.Core;

/**
 * @author Aaron Madlon-Kay
 */
@Tokenizer(languages = { "en" }, isDefault = true)
public class LuceneEnglishTokenizer extends BaseTokenizer {
    public static final Set<String> STOP_WORDS = new HashSet<String>();

    static {
        // Load stopwords
        try {
            InputStream in = LuceneEnglishTokenizer.class
                    .getResourceAsStream("StopList_en.txt");
            try {
                BufferedReader rd = new BufferedReader(new InputStreamReader(
                        in, "UTF-8"));
                String s;
                while ((s = rd.readLine()) != null) {
                    s = s.trim();
                    if (s.isEmpty() || s.startsWith("#")) {
                        continue;
                    }
                    STOP_WORDS.add(s);
                }
            } finally {
                in.close();
            }
        } catch (Exception ex) {
            throw new ExceptionInInitializerError(
                    "Error load stopwords in SnowballEnglishTokenizer: "
                            + ex.getMessage());
        }
    }

    /**
     * Register plugin into OmegaT.
     */
    public static void loadPlugins() {
        Core.registerTokenizerClass(LuceneEnglishTokenizer.class);
    }

    public static void unloadPlugins() {
    }

    @SuppressWarnings("resource")
    @Override
    protected TokenStream getTokenStream(final String strOrig,
            final boolean stemsAllowed, final boolean stopWordsAllowed) {
        if (stemsAllowed) {
            return new EnglishAnalyzer(getBehavior(),
                    stopWordsAllowed ? STOP_WORDS : Collections.emptySet()).tokenStream("",
                    new StringReader(strOrig));
        } else {
            return new StandardTokenizer(getBehavior(),
                    new StringReader(strOrig));
        }
    }
 
     public org.apache.lucene.analysis.Analyzer getGlossaryAnalyser() {
        return new EnglishAnalyzer(getBehavior(), STOP_WORDS);
    }
 	
}
