中文分詞處理源代碼C++

1、精選優(yōu)質(zhì)文檔-傾情為你奉上#include #include #include #include using namespace std;const int START1 = 0XB0,START2 = 0XA1, END1 = 0XF8,END2 = 0XFF;const int MAXWORDLEN = 48;ifstream fin(segdict.txt);ofstream out(out1.txt);/- 建樹部分-struct Node3 string S; bool IsWord; Node3 *L,*R; Node3(string s = ,bool isWord = 0, N

2、ode3 *l = 0, Node3 *r = 0): S(s),IsWord(isWord),L(l),R(r);struct Node2 string S; bool IsWord; Node3 *Child; Node2(string s =,bool isWord = 0, Node3* child =0): S(s),IsWord(isWord),Child(child);struct Node string S; vectorv;vectorDic;int HASHEND1 - START1END2 - START2;void Begin() /初始化 for (int i = 0

3、; i END1- START1; i+) for (int j = 0;j L != 0) LAST = LAST-L; if (LAST-S != t) LAST-L = new Node3(t,(len = 2),0, 0); LAST = LAST-L; if (len 2) BuildTree(s.substr(2,MAXWORDLEN),LAST-R);void Dictionary() /構(gòu)造整個(gè)結(jié)構(gòu) Begin(); string s; int N,k = 0; while(fin s) Node n; n.S = s.substr(0,2); int m1 = (unsign

4、ed char)s0 - START1; int m2 = (unsigned char)s1 - START2; HASHm1m2 = k+; out s HASHm1m2 N; out N endl; for (int i = 0; i s; out s 0 & n.vSIZE-1.S != t) n.v.push_back(Node2(t, (Len = 4),0); SIZE = n.v.size(); if (Len 4) BuildTree(s.substr(4,MAXWORDLEN),n.vSIZE-1.Child); Dic.push_back(n); out END HASH

5、 endl endl;/-查詢部分-vectorDest;int BinarySearch(int x, string Sec)/二分查找第二個(gè)字 int L = 0,R = Dicx.v.size() - 1; while (L 1; if (Dicx.vmid.S = Sec) return mid; else if (Dicx.vmid.S S = cc) return p; else p = p-L; return 0;unsigned CharToInt(char c) return unsigned(unsigned char)c) ;bool IsCC(char c) unsig

6、ned val= CharToInt(c); return val = START1 & val END1;bool IsEC(char c) unsigned val= CharToInt(c); return val 0x80;void FindNum(string src, vector&dest, int &StarPos,int &EndPos) int Strlen = src.length(); while (EndPos StarPos) dest.push_back(src.substr(StarPos,EndPos-StarPos); StarPos = EndPos; v

7、oid Segment(string src, vector&dest) int StrLen = src.length(); int StartPos = 0, EndPos; while (StartPos = StrLen) return ; unsigned SegLen = 2; string HeadCC = src.substr(StartPos, 2); cout HeadCC endl = 0); string SecCC = src.substr(StartPos + 2,2); if (SecCC.length() 0 & IsCC(SecCC0) int B2 = Bi

8、narySearch(HeadIndex,SecCC); if (B2=0) if (DicHeadIndex.vB2.IsWord) SegLen += 2; EndPos = StartPos + 4; Node3 *p = DicHeadIndex.vB2.Child; while(EndPos IsWord) SegLen = EndPos - StartPos; p = p-R; dest.push_back(src.substr(StartPos,SegLen); StartPos += SegLen; int main() Dictionary(); ofstream out2(

9、out2.txt); / string SS =有時(shí)，我會(huì)抬頭，看一看這喧囂的人群，有沒有我想見得身影，若是有那身影，或許我會(huì)看著她，看她慢慢的融入人群，直到不見。然后我會(huì)低下頭，走著我的道。; / string SS=中華人民萬歲;string SS=程序編碼基本正確，實(shí)現(xiàn)了程序設(shè)計(jì)中提到的兩種分詞策略，分詞結(jié)果就在預(yù)料之中。; / string SS= 在詞典中對(duì)于特定的首字，前兩字相同的詞條很少，前三字相同的詞條更少。當(dāng)我們以這種形式組織詞典后，除子表的第一層外，各個(gè)節(jié)點(diǎn)的兄弟數(shù)目都很小，對(duì)它們的查找采用順序查找方法較為適宜。 ; /string SS = 主要分為兩大模塊：一個(gè)建立一棵樹，一個(gè)是查詢。建樹有三個(gè)層次，第一層是HASH表，第二層是數(shù)組，用于二分查找使用，第三層是二叉樹。查詢分為直接查詢第一層的HASH表，第二層用二分查找（第二層漢子相同的平均概率是26，一般第二字成詞切相同