網(wǎng)絡(luò)爬蟲實(shí)驗(yàn)

1、湖北文理學(xué)院 網(wǎng)絡(luò)數(shù)據(jù)挖掘項(xiàng)目解決方案 實(shí)驗(yàn)報(bào)告專業(yè)班級(jí)： 軟件工程1111班 姓 名： 韓江玲 學(xué) 號(hào)： 2011193158 任課教師： 吳中博 2014 年 9 月 20 日實(shí)驗(yàn)5 網(wǎng)絡(luò)數(shù)據(jù)挖掘解決方案（X學(xué)時(shí)）一、實(shí)驗(yàn)?zāi)康呐c要求1、掌握小型項(xiàng)目的架構(gòu)與設(shè)計(jì)；2、能夠獨(dú)自實(shí)現(xiàn)小型項(xiàng)目的開發(fā)；3、通過實(shí)現(xiàn)網(wǎng)絡(luò)數(shù)據(jù)挖掘項(xiàng)目，鍛煉綜合應(yīng)用能力。二、預(yù)習(xí)與準(zhǔn)備1、Swing編程；2、正則表達(dá)式的使用；3、網(wǎng)絡(luò)數(shù)據(jù)的提??；4、分層架構(gòu)。三、實(shí)驗(yàn)內(nèi)容1、實(shí)現(xiàn)網(wǎng)絡(luò)數(shù)據(jù)挖掘項(xiàng)目；2、撰寫實(shí)驗(yàn)報(bào)告。四、實(shí)驗(yàn)過程1.首先明確項(xiàng)目解決思路：首先建立數(shù)據(jù)庫用swing編寫用戶界面從數(shù)據(jù)庫獲取StockId通過S

2、tockId獲取UR利用URL可以得到URL里面的內(nèi)容對(duì)該內(nèi)容進(jìn)行解析，獲取需要的數(shù)據(jù)存入數(shù)據(jù)庫。2.了解該項(xiàng)目的系統(tǒng)架構(gòu)：顯示層:構(gòu)建用戶界面，提供用戶中介業(yè)務(wù)邏輯層:1)獲取StockId； 2) 通過StockId獲取URL；3)通過URL提取網(wǎng)頁內(nèi)容； 4)解析網(wǎng)頁內(nèi)容獲取需要的數(shù)據(jù)； 5)存入相應(yīng)的數(shù)據(jù)庫。數(shù)據(jù)訪問層:實(shí)現(xiàn)對(duì)于數(shù)據(jù)庫的訪問，連接，增加，修改，刪除，查詢。3、開始項(xiàng)目（1）建數(shù)據(jù)庫（2）建立數(shù)據(jù)庫的連接并定義增刪改查操作接口public class Conn private static String driver=com.mysql.jdbc.Driver; priv

3、ate static String url=jdbc:mysql:/127.0.0.1:3306/st?useUnicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull;private static String user=root;private static String password=root;public static Connection getConn()Connection conn=null;tryClass.forName(driver);conn=DriverManager.getConn

4、ection(url, user, password);catch (Exception e) e.printStackTrace();return conn;/獲得數(shù)據(jù)庫的連接public static Connection getConnection(String className,String url,String user,String password)Connection connection = null;try Class.forName(className);connection = DriverManager.getConnection(url, user, passwo

5、rd); catch (Exception e) e.printStackTrace();return connection;public static void main(String args) int i = updateStockName(華夏,0600031);/更新股票名稱public static int updateStockName(String stockName,String stockId)Connection connection = getConnection(driver, url, user, password);int result = 0;try State

6、ment statement = connection.createStatement();result = statement.executeUpdate(update stockbase set stockname = +stockName+ where stockid = +stockId+);return result; catch (Exception e) / TODO Auto-generated catch blockresult = 0;return result;/獲得股票的idpublic static List getStockIds() List list=null;

7、Connection connection = getConnection(driver, url, user, password);try Statement statement = connection.createStatement();ResultSet resultSet = statement.executeQuery(select stockid from stockbase);list = new ArrayList();while(resultSet.next()list.add(resultSet.getString(1); catch (Exception e) / TO

8、DO Auto-generated catch blocke.printStackTrace();return list;/向stockmoney中插入數(shù)據(jù)public static int insertStockFH(List list) Connection connection = getConnection(driver, url, user, password);int result = 0 ;try for(int i=0;ilist.size();i+)List stocklist = list.get(i);for(int j=0;j(stocklist.size()/5);j

9、+)Statement statement = connection.createStatement();String id=stocklist.get(5*j+0).toString()+stocklist.get(5*j+1).toString();result = statement.executeUpdate(insert stockmoney(id,stockid,year,qian,songgu,zhuangu) values ( +id+,+stocklist.get(5*j+0)+,+stocklist.get(5*j+1)+,+stocklist.get(5*j+4)+,+s

10、tocklist.get(5*j+2)+,+stocklist.get(5*j+3)+); catch (Exception e) / TODO Auto-generated catch blockresult=0;System.out.println(result);return result;/刪除所有的錢public static Boolean deleteAllmoney() Boolean result = null;Connection connection = getConnection(driver, url, user, password);try Statement st

11、atement = connection.createStatement();result = statement.execute(delete from stockmoney); catch (SQLException e) / TODO Auto-generated catch blocke.printStackTrace();return result;（3）定義URL連接接口public class WebConnUtil public String getWebContent(URLConnection urlConnection) InputStream is;StringBuff

12、er sb = new StringBuffer();try is = urlConnection.getInputStream();System.out.println(is);InputStreamReader isr=new InputStreamReader(is,utf-8); BufferedReader br=new BufferedReader(isr);String strLine;while(strLine=br.readLine()!=null)sb.append(strLine); catch (IOException e) System.out.println(連接失

13、敗);return new String(sb);（4）對(duì)獲取的內(nèi)容進(jìn)行分析public class Search extends Thread/通過id獲得名字public List getStockNames(Stock stock, List stockId) List list = new ArrayList();StringBuffer sb=new StringBuffer();sb=sb.append(tt股票代碼t股票名稱n);for(int i=0;istockId.size();i+)String nameurl = stock.jTextField1.setText(正在

14、加載id為+stockId.get(i)+的股票名稱);try URL url = new URL(nameurl);URLConnection urlConnection = url.openConnection();String webcontent =new WebConnUtil().getWebContent(urlConnection);String regExp = (.1,2)s*(.1,8);StringBuffer sbSec = new StringBuffer();Pattern pat = Ppile( regExp );Matcher mat = pat.match

15、er(webcontent);if( mat.find() )list.add(mat.group(3);stock.jProgressBar1.setValue(100);Thread.sleep(100);sb=sb.append(tt+stockId.get(i)+t+mat.group(3).toString()+n);stock.jTextArea1.setText(sb.toString();stock.jProgressBar1.setValue(0); catch (Exception e) /e.printStackTrace();list.add(無);System.out

16、.println(不匹配);stock.jTextField1.setText(加載完成,已下載到數(shù)據(jù)庫！);return list;public List getStockFenhong(Stock stock, List stockId) List fhlist = new ArrayList();StringBuffer sb = new StringBuffer(); sb=sb.append(t股票名稱t股票代碼t分紅年度t送股t轉(zhuǎn)股t分紅n); stock.jTextArea1.setText(new String(sb);for(int i=0;istockId.size();i

17、+)String nameurl = stock.jTextField1.setText(正在加載id為+stockId.get(i)+的股票的分紅信息);try URL url = new URL(nameurl);URLConnection urlConnection = url.openConnection();String webcontent =new WebConnUtil().getWebContent(urlConnection);List list = null;String regExp = (.1,12)(.1,4)(.1,10)(.1,10)(.1,10)(.1,20)

18、(.1,20)(.1,20);StringBuffer sbSec = new StringBuffer();Pattern pat = Ppile( regExp );Document doc = Jsoup.parse(webcontent); Elements trs = doc.getElementsByClass(inner_box) .select(table).get(0).select(tr).select(td).after().before( ); Elements stockname = doc.getElementsByClass(name).select(h1).select(a); list = new ArrayList(); stock.jProgressBar1.setMaximum(trs.size()/8); for(int j=1;j=trs.size();j+=8) stock.jProgressBar1.setValue(j); list.add(stockId.get(i).toString().substring(1); list.add(trs.get