blocks|key|231725|text|您可以像这样下载存储桶中的所有文件(未测试)：|type|unstyled|depth|inlineStyleRanges|entityRanges|data|231726|from+boto.s3.connection+import+S3Connection

conn+=+S3Connection('your-access-key','your-secret-key')
bucket+=+conn.get_bucket('bucket')
for+key+in+bucket.list():
++++try:
++++++++res+=+key.get_contents_to_filename(key.name)
++++except:
++++++++logging.info(key.name%2B":"%2B"FAILED")|code-block|syntax|javascript|231727|请记住，S3中的文件夹只是另一种写键名的方式，只有客户端才会将其显示为文件夹。|231728|entityMap^0|0|0|0^^$0|@$1|2|3|4|5|6|7|K|8|@]|9|@]|A|$]]|$1|B|3|C|5|D|7|L|8|@]|9|@]|A|$E|F]]|$1|G|3|H|5|6|7|M|8|@]|9|@]|A|$]]|$1|I|3|-4|5|6|7|N|8|@]|9|@]|A|$]]]|J|$]]

You can download all files in a bucket like this (untested):

<pre><code>from boto.s3.connection import S3Connection

conn = S3Connection('your-access-key','your-secret-key')
bucket = conn.get_bucket('bucket')
for key in bucket.list():
 try:
 res = key.get_contents_to_filename(key.name)
 except:
 logging.info(key.name+":"+"FAILED")
</code></pre>

Keep in mind that folders in S3 are simply another way of writing the key name and only clients will show this as folders.

blocks|key|231735|text|import+boto,+os

LOCAL_PATH+=+'tmp/'

AWS_ACCESS_KEY_ID+=+'YOUUR_AWS_ACCESS_KEY_ID'
AWS_SECRET_ACCESS_KEY+=+'YOUR_AWS_SECRET_ACCESS_KEY'
bucket_name+=+'your_bucket_name'

#+connect+to+the+bucket
conn+=+boto.connect_s3(AWS_ACCESS_KEY_ID,+AWS_SECRET_ACCESS_KEY)
bucket+=+conn.get_bucket(bucket_name)

#+go+through+the+list+of+files
bucket_list+=+bucket.list()
for+l+in+bucket_list:
++keyString+=+str(l.key)
++d+=+LOCAL_PATH+%2B+keyString
++try:
++++l.get_contents_to_filename(d)
++except+OSError:
++++#+check+if+dir+exists
++++if+not+os.path.exists(d):
++++++os.makedirs(d)++#+Creates+dirs+recurcivly+|type|code-block|depth|inlineStyleRanges|entityRanges|data|syntax|javascript|231736|unstyled|entityMap^0|0^^$0|@$1|2|3|4|5|6|7|G|8|@]|9|@]|A|$B|C]]|$1|D|3|-4|5|E|7|H|8|@]|9|@]|A|$]]]|F|$]]

<pre><code>import boto, os

LOCAL_PATH = 'tmp/'

AWS_ACCESS_KEY_ID = 'YOUUR_AWS_ACCESS_KEY_ID'
AWS_SECRET_ACCESS_KEY = 'YOUR_AWS_SECRET_ACCESS_KEY'
bucket_name = 'your_bucket_name'

# connect to the bucket
conn = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
bucket = conn.get_bucket(bucket_name)

# go through the list of files
bucket_list = bucket.list()
for l in bucket_list:
 keyString = str(l.key)
 d = LOCAL_PATH + keyString
 try:
 l.get_contents_to_filename(d)
 except OSError:
 # check if dir exists
 if not os.path.exists(d):
 os.makedirs(d) # Creates dirs recurcivly 
</code></pre>

blocks|key|449640|text|#!/usr/bin/env+python

import+boto
import+sys,+os
from+boto.s3.key+import+Key
from+boto.exception+import+S3ResponseError


DOWNLOAD_LOCATION_PATH+=+os.path.expanduser("~")+%2B+"/s3-backup/"
if+not+os.path.exists(DOWNLOAD_LOCATION_PATH):
++++print+("Making+download+directory")
++++os.mkdir(DOWNLOAD_LOCATION_PATH)


def+backup_s3_folder():
++++BUCKET_NAME+=+"your-bucket-name"
++++AWS_ACCESS_KEY_ID=+os.getenv("AWS_KEY_ID")+#+set+your+AWS_KEY_ID++on+your+environment+path
++++AWS_ACCESS_SECRET_KEY+=+os.getenv("AWS_ACCESS_KEY")+#+set+your+AWS_ACCESS_KEY++on+your+environment+path
++++conn++=+boto.connect_s3(AWS_ACCESS_KEY_ID,+AWS_ACCESS_SECRET_KEY)
++++bucket+=+conn.get_bucket(BUCKET_NAME)

++++#goto+through+the+list+of+files
++++bucket_list+=+bucket.list()

++++for+l+in+bucket_list:
++++++++key_string+=+str(l.key)
++++++++s3_path+=+DOWNLOAD_LOCATION_PATH+%2B+key_string
++++++++try:
++++++++++++print+("Current+File+is+",+s3_path)
++++++++++++l.get_contents_to_filename(s3_path)
++++++++except+(OSError,S3ResponseError)+as+e:
++++++++++++pass
++++++++++++#+check+if+the+file+has+been+downloaded+locally++
++++++++++++if+not+os.path.exists(s3_path):
++++++++++++++++try:
++++++++++++++++++++os.makedirs(s3_path)
++++++++++++++++except+OSError+as+exc:
++++++++++++++++++++#+let+guard+againts+race+conditions
++++++++++++++++++++import+errno
++++++++++++++++++++if+exc.errno+!=+errno.EEXIST:
++++++++++++++++++++++++raise




if+__name__+==+'__main__':
++++backup_s3_folder()|type|code-block|depth|inlineStyleRanges|entityRanges|data|syntax|javascript|449641|unstyled|entityMap^0|0^^$0|@$1|2|3|4|5|6|7|G|8|@]|9|@]|A|$B|C]]|$1|D|3|-4|5|E|7|H|8|@]|9|@]|A|$]]]|F|$]]

<pre><code>#!/usr/bin/env python

import boto
import sys, os
from boto.s3.key import Key
from boto.exception import S3ResponseError


DOWNLOAD_LOCATION_PATH = os.path.expanduser("~") + "/s3-backup/"
if not os.path.exists(DOWNLOAD_LOCATION_PATH):
 print ("Making download directory")
 os.mkdir(DOWNLOAD_LOCATION_PATH)


def backup_s3_folder():
 BUCKET_NAME = "your-bucket-name"
 AWS_ACCESS_KEY_ID= os.getenv("AWS_KEY_ID") # set your AWS_KEY_ID on your environment path
 AWS_ACCESS_SECRET_KEY = os.getenv("AWS_ACCESS_KEY") # set your AWS_ACCESS_KEY on your environment path
 conn = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_ACCESS_SECRET_KEY)
 bucket = conn.get_bucket(BUCKET_NAME)

 #goto through the list of files
 bucket_list = bucket.list()

 for l in bucket_list:
 key_string = str(l.key)
 s3_path = DOWNLOAD_LOCATION_PATH + key_string
 try:
 print ("Current File is ", s3_path)
 l.get_contents_to_filename(s3_path)
 except (OSError,S3ResponseError) as e:
 pass
 # check if the file has been downloaded locally 
 if not os.path.exists(s3_path):
 try:
 os.makedirs(s3_path)
 except OSError as exc:
 # let guard againts race conditions
 import errno
 if exc.errno != errno.EEXIST:
 raise




if __name__ == '__main__':
 backup_s3_folder()
</code></pre>

blocks|key|449659|text|刚在@j0nes评论中添加了目录创建部分|type|unstyled|depth|inlineStyleRanges|entityRanges|data|449660|from+boto.s3.connection+import+S3Connection
import+os

conn+=+S3Connection('your-access-key','your-secret-key')
bucket+=+conn.get_bucket('bucket')

for+key+in+bucket.list():
++++print+key.name
++++if+key.name.endswith('/'):
++++++++if+not+os.path.exists('./'%2Bkey.name):
++++++++++++os.makedirs('./'%2Bkey.name)
++++else:
++++++++res+=+key.get_contents_to_filename('./'%2Bkey.name)|code-block|syntax|javascript|449661|这会将文件下载到当前目录，并在需要时创建目录。|449662|entityMap^0|0|0|0^^$0|@$1|2|3|4|5|6|7|K|8|@]|9|@]|A|$]]|$1|B|3|C|5|D|7|L|8|@]|9|@]|A|$E|F]]|$1|G|3|H|5|6|7|M|8|@]|9|@]|A|$]]|$1|I|3|-4|5|6|7|N|8|@]|9|@]|A|$]]]|J|$]]

Just added directory creation part to @j0nes comment

<pre><code>from boto.s3.connection import S3Connection
import os

conn = S3Connection('your-access-key','your-secret-key')
bucket = conn.get_bucket('bucket')

for key in bucket.list():
 print key.name
 if key.name.endswith('/'):
 if not os.path.exists('./'+key.name):
 os.makedirs('./'+key.name)
 else:
 res = key.get_contents_to_filename('./'+key.name)
</code></pre>

This will download files to current directory and will create directories when needed.

blocks|key|449676|text|import+boto
from+boto.s3.key+import+Key

keyId+=+'YOUR_AWS_ACCESS_KEY_ID'
sKeyId='YOUR_AWS_ACCESS_KEY_ID'
bucketName='your_bucket_name'

conn+=+boto.connect_s3(keyId,sKeyId)
bucket+=+conn.get_bucket(bucketName)
for+key+in+bucket.list():
++++print+">>>>>"%2Bkey.name
++++pathV+=+key.name.split('/')
++++if(pathV[0]+==+"data"):
++++++++if(pathV[1]+!=+""):
++++++++++++srcFileName+=+key.name
++++++++++++filename+=+key.name
++++++++++++filename+=+filename.split('/')[1]
++++++++++++destFileName+=+"model/data/"%2Bfilename
++++++++++++k+=+Key(bucket,srcFileName)
++++++++++++k.get_contents_to_filename(destFileName)
++++elif(pathV[0]+==+"nlu_data"):
++++++++if(pathV[1]+!=+""):
++++++++++++srcFileName+=+key.name
++++++++++++filename+=+key.name
++++++++++++filename+=+filename.split('/')[1]
++++++++++++destFileName+=+"model/nlu_data/"%2Bfilename
++++++++++++k+=+Key(bucket,srcFileName)
++++++++++++k.get_contents_to_filename(destFileName`|type|code-block|depth|inlineStyleRanges|entityRanges|data|syntax|javascript|449677|unstyled|entityMap^0|0^^$0|@$1|2|3|4|5|6|7|G|8|@]|9|@]|A|$B|C]]|$1|D|3|-4|5|E|7|H|8|@]|9|@]|A|$]]]|F|$]]

<pre><code>import boto
from boto.s3.key import Key

keyId = 'YOUR_AWS_ACCESS_KEY_ID'
sKeyId='YOUR_AWS_ACCESS_KEY_ID'
bucketName='your_bucket_name'

conn = boto.connect_s3(keyId,sKeyId)
bucket = conn.get_bucket(bucketName)
for key in bucket.list():
 print "&gt;&gt;&gt;&gt;&gt;"+key.name
 pathV = key.name.split('/')
 if(pathV[0] == "data"):
 if(pathV[1] != ""):
 srcFileName = key.name
 filename = key.name
 filename = filename.split('/')[1]
 destFileName = "model/data/"+filename
 k = Key(bucket,srcFileName)
 k.get_contents_to_filename(destFileName)
 elif(pathV[0] == "nlu_data"):
 if(pathV[1] != ""):
 srcFileName = key.name
 filename = key.name
 filename = filename.split('/')[1]
 destFileName = "model/nlu_data/"+filename
 k = Key(bucket,srcFileName)
 k.get_contents_to_filename(destFileName`
</code></pre>

blocks|key|231805|text|如果文件夹中有1000个以上的文件，则需要使用分页器遍历这些文件|type|unstyled|depth|inlineStyleRanges|entityRanges|data|231806|import+boto3
import+os
#+create+the+client+object
client+=+boto3.client(
's3',
aws_access_key_id=+S3_ACCESS_KEY,
aws_secret_access_key=++S3_SECRET_KEY
)
#+bucket+and+folder+urls
bucket=+'bucket-name'
data_key+=+'key/to/data/'

paginator+=+client.get_paginator("list_objects_v2")
for+page+in+paginator.paginate(Bucket=bucket,+Prefix=data_dir):
++++for+obj+in+page['Contents']:
++++++++key+=+obj['Key']
++++++++tmp_dir+=++'/'.join(key.split('/')[0:-1])
++++++++if+not+os.path.exists('/'.join(key.split('/')[0:-1])):
++++++++++++os.makedirs(tmp_dir)
++++++++else:
++++++++++++client.download_file(bucket,+key,+tmp_dir+%2B+key.split('/')[-1])|code-block|syntax|javascript|231807|entityMap^0|0|0^^$0|@$1|2|3|4|5|6|7|I|8|@]|9|@]|A|$]]|$1|B|3|C|5|D|7|J|8|@]|9|@]|A|$E|F]]|$1|G|3|-4|5|6|7|K|8|@]|9|@]|A|$]]]|H|$]]

if you have more than 1000 files in the folder you need to use a paginator
to iterate through them

<pre><code>import boto3
import os
# create the client object
client = boto3.client(
's3',
aws_access_key_id= S3_ACCESS_KEY,
aws_secret_access_key= S3_SECRET_KEY
)
# bucket and folder urls
bucket= 'bucket-name'
data_key = 'key/to/data/'

paginator = client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=bucket, Prefix=data_dir):
 for obj in page['Contents']:
 key = obj['Key']
 tmp_dir = '/'.join(key.split('/')[0:-1])
 if not os.path.exists('/'.join(key.split('/')[0:-1])):
 os.makedirs(tmp_dir)
 else:
 client.download_file(bucket, key, tmp_dir + key.split('/')[-1])
</code></pre>

I have a bucket in s3, which has deep directory structure. I wish I could download them all at once. My files look like this :

<pre><code>foo/bar/1. . 
foo/bar/100 . . 
</code></pre>

Are there any ways to download these files recursively from the s3 bucket using boto lib in python? 

Thanks in advance.

Downloading the files from s3 recursively using boto python.

我在s3中有一个存储桶，它有很深的目录结构。我希望我能一次把它们都下载下来。我的文件如下所示：foo/bar/1. . foo/bar/100 . . 有没有什么方法可以使用boto lib从s3存储桶中递归地下载这些文件呢？提前谢谢。

问使用boto python递归地从s3下载文件。
EN

社区

活动

资源

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问使用boto python递归地从s3下载文件。EN

社区

活动

资源

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问使用boto python递归地从s3下载文件。
EN