问Boto3与多处理

EN

import boto3
import multiprocessing

_bucket=None
def init_read_file_from_s3(bucket_name):
    # Helper to create a S3 resource object for this worker
    global _bucket
    s3 = boto3.resource('s3')
    _bucket = s3.Bucket(bucket_name)

def read_file_from_s3(values):
    # Pull out the data
    i, s3_key = values

    # Perform work on a given S3 Key, use the single S3 bucket object the
    # initializer created before this worker started
    obj = _bucket.Object(s3_key)
    response = obj.get()
    # TODO: Operate on the object here, "i" is a unique ID for this item

    # Inform the caller that we're done with this key
    return s3_key

def main():
    bucket_name = 'example-bucket'
    prefix = 'example/prefix/'

    # Get a s3 resource object to list the objects with
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(bucket_name)

    # Initialize all of the workers with the bucket name so they can create a S3 resource object
    with multiprocessing.Pool(64, initializer=init_read_file_from_s3, initargs=(bucket_name,)) as a_pool:
        # List the objects, and feed each key in turn to a worker
        # Note that the object returned from filter() or all() isn't picklable, so we
        # pass just the string of the S3 key
        for result in a_pool.imap(
            read_file_from_s3, 
            [(i,x.key) for i,x in enumerate(bucket.objects.filter(Prefix=prefix))]):
            
            # Just show the key that was operated on
            print("Done with: " + result)

if __name__ == "__main__":
    main()