python实现简易搜索引擎（含代码）

def crawl(pages,depth=2):
    for i in range(depth):
        newpages = set()
        for page in pages:
            try:
                c = urllib.request.urlopen(page)
            except:
                print('Invaild page:',page)
                continue
            soup = bs4.BeautifulSoup(c.read())

            links = soup('a')
            for link in links:
                if('href' in dict(link.attrs)):
                    url = urllib.urljoin(page,link['href'])
                    if url.find("'")!=-1:continue
                    url = url.split('#')[0]
                    if url[0:3]=='http':
                        newpages.add(url)
        pages = newpages

output = sys.stdout
outputfile = open('lujing.txt', 'w')
sys.stdout = outputfile
list = GetFileList(lujing, [])

# 将生成的路径文件lujing.txt读取，并按照路径文件对文本处理，去标签
for line in open("lujing.txt"):
    print(line)
    # line=line[0:-2]
    line1 = line[0:12]
    line2 = line[13:16]
    line3 = line[17:-1]
    line4 = line[17:-6]
    line = line1 + '\\' + line2 + '\\' + line3
    print(line4)
    path = line
    fb = open(path, "rb")
    data = fb.read()
    bianma = chardet.detect(data)['encoding']  # 获取当前文件的编码方式，并按照此编码类型处理文档
    page = open(line, 'r', encoding=bianma, errors='ignore').read()
    dr = re.compile(r'<[^>]+>', re.S)  # 去HTML标签
    dd = dr.sub('', page)
    print(dd)
    fname = 'TXT' + "\\" + line4 + ".txt"
    # print(fname)
    f = open(fname, "w+", encoding=bianma)  # 将去标签的文件写到文件夹内，并按照原命名以txt文档方式保存
    # fo=open(fname,"w+")
    f.write(dd)

import jieba
import chardet
import pymysql
import importlib, sys
importlib.reload(sys)

# 如果使用MongoDB
# from pymongo import MongoClient
# #data processing
# client = MongoClient('localhost',27017)
# apiDB = client['urlDB']    #serverDB_name:test_nodedata
# questionnaires = apiDB['weburl']
# data = list(questionnaires.find())
conn = pymysql .connect(host="localhost",user="root",
                       password="123456",db="suoyin",port=3307)
conn.text_factory = str
c = conn.cursor()
c.execute('drop table doc')
c.execute('create table doc (id int primary key,link text)')
c.execute('drop table word')
c.execute('create table word (term varchar(25) primary key,list text)')
conn.commit()
conn.close()


def Fenci():
    num = 0
    for line in open("url.txt"):
        lujing = line
        print(lujing)
        num += 1
        print(line)
        line = line[17:-5]
        print(line)
        line = 'TXT' + '\\' + line + 'Txt'  # line为文件位置
        print(line)  # 文件名称
        path = line
        fb = open(path, "rb")
        data = fb.read()
        bianma = chardet.detect(data)['encoding']  # 获取文件编码        print(bianma)
        # page = open(line, 'r', encoding=bianma, errors='ignore').read()
        # page1=page.decode('UTF-8')
        if bianma == 'UTF-16':
            data = data.decode('UTF-16')
            data = data.encode('utf-8')
        word = jieba.cut_for_search(data)
        seglist = list(word)
        print(seglist)

        # 创建数据库
        c = conn.cursor()  # 创建游标
        c.execute('insert into doc values(?,?)', (num, lujing))
        # 对每个分出的词语建立词表
        for word in seglist:
            # print(word)
            # 检验看看这个词语是否已存在于数据库
            c.execute('select list from word where term=?', (word,))
            result = c.fetchall()
            # 如果不存在
            if len(result) == 0:
                docliststr = str(num)
                c.execute('insert into word values(?,?)', (word, docliststr))
            # 如果已存在
            else:
                docliststr = result[0][0]  # 得到字符串
                docliststr += ' ' + str(num)
                c.execute('update word set list=? where term=?', (docliststr, word))
        conn.commit()
        conn.close()


Fenci()

import pymsql
import jieba
import math

conn = pymysql .connect(host="localhost",user="root",
                       password="123456",db="suoyin",port=3307)
c = conn.cursor()
c.execute('select count(*) from doc')
N = 1 + c.fetchall()[0][0]  # 文档总数
target = input('请输入搜索词：')
seggen = jieba.cut_for_search(target)
score = {}  # 文档号：匹配度
for word in seggen:
    print('得到查询词：', word)
    # 计算score
    tf = {}  # 文档号：文档数
    c.execute('select list from word where term=?', (word,))
    result = c.fetchall()
    if len(result) > 0:
        doclist = result[0][0]
        doclist = doclist.split(' ')
        # 把字符串转换为元素为int的list
        doclist = [int(x) for x in doclist]
        # 当前word对应的df数
        df = len(set(doclist))
        idf = math.log(N / df)
        print('idf：', idf)
        for num in doclist:
            if num in tf:
                tf[num] = tf[num] + 1
            else:
                tf[num] = 1
        # tf统计结束，现在开始计算score
        for num in tf:
            if num in score:
                # 如果该num文档已经有分数了，则累加
                score[num] = score[num] + tf[num] * idf
            else:
                score[num] = tf[num] * idf
sortedlist = sorted(score.items(), key=lambda d: d[1], reverse=True)

cnt = 0
for num, docscore in sortedlist:
    cnt = cnt + 1
    c.execute('select link from doc where id=?', (num,))
    url = c.fetchall()[0][0]
    print("Result Ranking：", cnt)
    print('url：', url, 'match degree：', docscore)

    if cnt > 20:
        break
if cnt == 0:
    print('No result')