问文档处理库

EN

        StringToWordVector filter = new StringToWordVector(); //create new filter for vector transformation

        NGramTokenizer tokenizer = new NGramTokenizer(); //tokenizer for filter
        tokenizer.setNGramMinSize(1); 
        tokenizer.setNGramMaxSize(1);
        tokenizer.setDelimiters("\\W");


        SnowballStemmer stemmer = new SnowballStemmer();  //Initializes stemmer 
        stemmer.setStemmer("english"); //set english stemmer


        filter.setLowerCaseTokens(true); 
        filter.setOutputWordCounts(true); 

        filter.setTFTransform(true); //   //Tf–idf can be successfully used for stop-words filtering in various subject fields including text summarization and classification.
        filter.setIDFTransform(true);      
        filter.setStopwords(new File("C:\\Users\\hp\\Desktop\\SVM implementation\\StopWordsR4.txt")); // stop word removal 
        filter.setTokenizer(tokenizer); 
        filter.setStemmer(stemmer);