Python列表的理解速度太慢

内容来源于 Stack Overflow,并遵循CC BY-SA 3.0许可协议进行翻译与使用

  • 回答 (2)
  • 关注 (0)
  • 查看 (217)

我有231个PDF文件,并希望将它们转换为字符串格式。随后,我会将每个这些字符串保存到一个txt文件中。

我能够为此创建一个代码(当我运行少量元素的代码时,我检查它是否有效),但是即使在10小时后,python也没有完成执行程序。

我使用“for循环”尝试了相同的代码,但它也太慢了。我怎么能使这个代码更快?

这是我的代码:

from pdfminer.pdfinterp import PDFResourceManager, 
PDFPageInterpreter#process_pdf
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams

from io import StringIO

def pdf_to_text(pdfname):

    # PDFMiner boilerplate
    rsrcmgr = PDFResourceManager()
    sio = StringIO()
    codec = 'utf-8'
    laparams = LAParams()
    device = TextConverter(rsrcmgr, sio, codec=codec, laparams=laparams)
    interpreter = PDFPageInterpreter(rsrcmgr, device)

    # Extract text
    fp = open(pdfname, 'rb')
    for page in PDFPage.get_pages(fp):
        interpreter.process_page(page)
    fp.close()

    # Get text from StringIO
    text = sio.getvalue()

    # Cleanup
    device.close()
    sio.close()

    return text

lista2 = [pdf_to_text(k) for k in lista1]
提问于
用户回答回答于
#!/usr/bin/env python

import os

from multiprocessing import Pool, cpu_count
from io import BytesIO

from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams

PDF_DIRECTORY = "pdfs"


def pdf_to_text(pdf_filename):
    # PDFMiner boilerplate
    pdf_resource_manager = PDFResourceManager()
    bytes_io = BytesIO()
    device = TextConverter(
        pdf_resource_manager, bytes_io, codec="utf-8", laparams=LAParams())
    interpreter = PDFPageInterpreter(pdf_resource_manager, device)

    # Extract text
    with open(pdf_filename, "rb") as fp:
        for page in PDFPage.get_pages(fp):
            interpreter.process_page(page)

    text = str(bytes_io.getvalue())

    # Cleanup
    bytes_io.close()
    device.close()

    # Print current filename with some of the parsed text
    print("{} - {}".format(pdf_filename, text[:15].replace("\n", "")))

    return text


def process_pdf_file(pdf_filename):
    text_filename = "{}.txt".format(os.path.splitext(pdf_filename)[0])

    # Write the text file to disk to avoid having to keep
    # it in memory
    with open(text_filename, "w") as text_file:
        text_file.write(pdf_to_text(pdf_filename))


def main():
    pdf_filename_list = []
    for filename in os.listdir(PDF_DIRECTORY):
        if not filename.endswith(".pdf"):
            continue
        pdf_filename_list.append(os.path.join(PDF_DIRECTORY, filename))

    MULTITHREAD = True
    if MULTITHREAD:
        # Use a thread pool to process multiple PDF files at the same time
        pool = Pool(cpu_count())
        pool.map(process_pdf_file, pdf_filename_list)
    else:
        # Version without thread pool
        for pdf_filename in pdf_filename_list:
            process_pdf_file(pdf_filename)


main()
用户回答回答于

例如:

for f in files:
   text = pdf_to_text(f)
   output.write(text)

扫码关注云+社区

领取腾讯云代金券