问使用EC2的多线程Python

EN

from multiprocessing import Process
import datetime
import Prod_Modeling_Pipeline as PMP
import boto3
import pandas
import time
import numpy
import os

#Define locations
bucketName = 'bucketgoeshere'
output_location = '/home/ec2-user/'

#Pull ATM Setter Over
client = boto3.client('s3')
transfer = boto3.s3.transfer.S3Transfer(client=client)
transfer.download_file(bucketName,'Root_Folder/Control_Files/'+'execution_file.csv', output_location+'execution_file.csv')

#Read-in id list
execution_data = pandas.read_csv(output_location+'execution_file.csv')
ids = execution_data['id']

ni = 90
id_row = [['AAA']*ni for _ in xrange(int(numpy.ceil(len(tids)/float(ni))))]

for i in xrange(len(ids)):
    id_row[i/ni][i%ni] = ids[i]

Date = datetime.date.today().strftime('%Y-%m-%d')

totalstart = time.time()
for q in xrange(len(tid_row)):
    processes = []
    for m in xrange(len(tid_row[q])):
        temp = tid_row[q]
        try:
            p = Process(target=PMP.Model_Function, args=(temp[m],Date,'VALIDATION'))
            p.start()
            processes.append(p)
            time.sleep(1)
            print("Started "+temp[m]+" as "+str(os.getpid()))
        except:
            print("Invalid Run")

    for p in processes:
        p.join()

    print(processes)

print (time.time() - totalstart)