11個Java開源中文分詞器使用方法

1、11個Java開源中文分詞器使用方法動力節(jié)點java11大Java開源中文分詞器，不同的分詞器有不同的用法，定義的接口也不一樣，我們先定 義一個統(tǒng)一的接口：/*獲取文本的所有分詞結果，對比不同分詞器結果*/public interface WordSegmenter /*獲取文本的所有分詞結果*param text 文本*return所有的分詞結果，去除重復*/default public Set seg(String text) return segMore(text).values().stream().collect(Collectors.toSet();/*獲取文本的所有分詞結果*pa

2、ram text 文本*return所有的分詞結果，KEY為分詞器模式，VALUE為分詞器結果*/public Map segMore(String text);從上面的定義我們知道，在Java中，同樣的方法名稱和參數，但是返回值不同，這種情況不可以使用重載。這兩個方法的區(qū)別在于返回值，每一個分詞器都可能有多種分詞模式，每種模式的分詞結果 都可能不相同，第一個方法忽略分詞器模式，返回所有模式的所有不重復分詞結果，第二個 方法返回每一種分詞器模式及其對應的分詞結果。在這里，需要注意的是我們使用了 Java8中的新特性默認方法，并使用stream把一個map的valu e轉換為不重復的集合。下面我

3、們利用這11大分詞器來實現這個接口：1、word分詞器Overridepublic Map segMore(String text) Map map = new HashMap();for(SegmentationAlgorithm segmentationAlgorithm : SegmentationAlgorithm.va lues()map.put(segmentationAlgorithm.getDes(), seg(text, segmentationAlgorith m);return map;private static String seg(String text, Segm

4、entationAlgorithm segmentationAlgo rithm) StringBuilder result =new StringBuilder();for(Word word : WordSegmenter.segWithStopWords(text, segmentationAlgorith m)result.append(word.getText().append );,2、Ansj分詞器Overridepublic Map segMore(String text) Map map = new HashMap();StringBuilder result = new S

5、tringBuilder();for(Term term : BaseAnalysis.parse(text)result.append(term.getName().append ); map.put(BaseAnalysis, result.toString();result.setLength 0);for(Term term : ToAnalysis.parse(text)result.append(term.getName().append ); map.put(ToAnalysis, result.toString();result.setLength 0);for (Term t

6、erm : NlpAnalysis .parse(text)result.append(term.getName().append );map.put(NlpAnalysis, result.toString();result.setLength 0);for(Term term :IndexAnalysis.parse(text)map.put(IndexAnalysis, result.toString();return map;3、Stanford 分詞器private static final StanfordCoreNLP CTB = new StanfordCoreNLP(Stan

7、fordCoreNL P-chinese-ctb);private static final StanfordCoreNLP PKU = new StanfordCoreNLP(StanfordCoreNL P-chinese-pku);private static final PrintStream NULL_PRINT_STREAM = new PrintStream(new NullO utputStream(), false);public Map segMore(String text) Map map = new HashMap();map.put Stanford Beijing

8、 University segmentation, seg(PKU, text);map.put Stanford Chinese Treebank segmentation, seg(CTB, text);return map;private static String seg(StanfordCoreNLP stanfordCoreNLP, String text)PrintStream err = System.err;System.setErr(NULL_PRINT_STREAM);Annotationdocument = new Annotation(text);stanfordCo

9、reNLP.annotate document);List sentences =document.get(CoreAnnotations.SentencesAnnotatio n.class);StringBuilder result =new StringBuilder();for(CoreMap sentence: sentences) for (CoreLabel token: sentence.get(CoreAnnotations.TokensAnnotation.cl ass) String word = token.get(CoreAnnotations.TextAnnotat

10、ion.class);result.append(word).appen();System.setErr(err);return result.toString();4、FudanNLP 分詞器private static CWSTagger tagger = null;statictrytagger =iew CWSTagger(lib/fudannlpseg.m);taggersetEnFilterf) tagger.setEnFtertrue);Catch(Exception e)_;一public Map segMore(String text) Map map = new HashM

11、ap();map.put FudanNLP, tagger.tag(text); i;5、Jieba分詞器private static final JiebaSegmenter JIEBA_SEGMENTER = new JiebaSegmenter(); Overridepublic Map segMore(String text) Map map = new HashMap();map.putINDEX, seg(text, SegMode.INDEX);map.putSEARCH, seg(text, SegMode.SEARCH);return map; private static

12、String seg(String text, SegMode segMode) StringBuilder result =new StringBuilder();for(SegToken token : JIEBA_SEGMENTER.process(text, segMode) result.append(token.word.getToken().append );return result.toString(); 6、Jcseg分詞器private static final JcsegTaskConfig CONFIG = new JcsegTaskConfig();private

13、static final ADictionary DIC = DictionaryFactory.createDefaultDiction ary(CONFIG);static CONFIG.setLoadCJKSynfalse);CONFIG.setLoadCJKPinyinfalse);Overridepublic Map segMore (String text) Map map =new HashMap();map.put 復雜模式，segText(text, JcsegTaskConfig.COMPLEX_MODE);map.put 簡易模式，segText(text, JcsegT

14、askConfig.SIMPLE_MODE);return map;private String segText(String text, int segMode) StringBuilder result =new StringBuilder();try ISegment seg = SegmentFactory.createJcseg(segModenew Objectnew Str ingReader(text), CONFIG, DIC);IWord word null;while(word=seg.next()!=null) result.append(word.getValue()

15、.appen();catch (Exception ex) ption(ex) 廠_.7、MMSeg4j分詞器private static final Dictionary DIC = Dictionary.getInstance();private static final SimpleSeg SIMPLE_SEG = new SimpleSeg(DIC);private static final ComplexSeg COMPLEX_SEG = new ComplexSeg(DIC);private static final MaxWordSeg MAX_WORD_SEG = new Ma

16、xWordSeg(DIC);Overridepublic Map segMore (String text) Map map =new HashMap();map.put(SIMPLE_SEG.getClass().getSimpleName(), segText(text, SIMPLE_SE G);map.put(COMPLEX_SEG.getClass().getSimpleName(), segText(text, COMPLEX_SE G);map.put(MAX_WORD_SEG.getClass().getSimpleName(), segText(text, MAX_WORD_

17、S EG);return map; private String segText(String text, Seg seg) StringBuilder result =new StringBuilder();MMSeg mmSeg =new MMSeg(new StringReader(text), seg);try Word word null;while(word=mmSeg.next()!=null) result.append(word.getString().appen();throw new RuntimeException(ex););8、IKAnalyzer 分詞器Overr

18、idepublic Map segMore(String text) Map map = new HashMap();map.put 智能切分, segText(text, true);map.put 細粒度切分,segText(text, false);return map;private String segText(String text, boolean useSmart) StringBuilder result =new StringBuilder();IKSegmenter ik =new IKSegmenter(new StringReader(text), useSmart)

19、;try Lexeme word null;while(word=ik.next()!=null) result.append(word.getLexemeText().appen();_throw new RuntimeException(ex);9、Paoding分詞器private static final PaodingAnalyzer ANALYZER = new PaodingAnalyzer();Overridepublic Map segMore(String text) Map map = new HashMap();map.put MOST_WORDS_MODE”, seg

20、(text, PaodingAnalyzer.MOST_WORDS_MODE);map.put MAX_WORD_LENGTH_MODE”, seg(text, PaodingAnalyzer.MAX_WORD_LENGTH _MODE);return map;private static String seg(String text, int mode)ANALYZER.setMode(mode);StringBuilder result =new StringBuilder();try Token reusableToken new Token();TokenStream stream =

21、 ANALYZER.tokenStrea, new StringReader(text);Token token null;while(token = stream.next(reusableToken) ! = null)result.append(token.term().appen();catch (Exception ex) throw new RuntimeException(ex);return result.toString();10、smarten 分詞器private static final SmartChineseAnalyzer SMART_CHINESE_ANALYZ

22、ER = new SmartCh ineseAnalyzer();Overridepublic Map segMore(String text) Map map = new HashMap();map.putsmartcn, segText(text);return map;private static String segText(String text) StringBuilder result =new StringBuilder();try TokenStream tokenStream = SMART_CHINESE_ANALYZER.tokenStreaitext, n ew St

23、ringReader(text);tokenStream.reset();while (tokenStream.incrementToken()CharTermAttribute charTermAttribute = tokenStream.getAttribute(Char TermAttribute.class);result.append(charTermAttribute.toString().appen();tokenStream.close();Catch (Exception e)e.printStackTrace();return result.toString();11、H

24、anLP分詞器private static final Segment N_SHORT_SEGMENT = new NShortSegment().enableCusto mDictionary(false).enablePlaceRecognize(true).enableOrganizationRecognize(tr ue);private static final Segment DIJKSTRA_SEGMENT = new DijkstraSegment().enableCu stomDictionary(false).enablePlaceRecognize(true).enabl

25、eOrganizationRecognize (true);Overridepublic Map segMore(String text) Map map =new HashMap();map.put 標準分詞,standard(text);map.put NLP 分詞，nlp(text);map.put 索弓I分詞，index(text);map.put N-最短路徑分詞,nShort(text);map.put 最短路徑分詞,shortest(text);map.put 極速詞典分詞，speed(text);return map;private static String standard(String text) StringBuilder result =new StringBuilder();StandardTokenizer.segment(text) forEach(term-result.append(term.word).ap pend();return result.toString();private static String nlp(String text) StringBuilder result =