使用boto3将文件在两个AWS S3存储桶之间迁移

Question

使用boto3将文件在两个AWS S3存储桶之间迁移

pythonamazon-s3boto

55

我需要使用Python Boto API将文件从一个存储桶移动到另一个存储桶。(我需要将文件从第一个存储桶中“剪切”并“粘贴”到第二个存储桶中)。

最佳的方法是什么？

**注意：如果我有两个不同的ACCESS KEYS和SECRET KEYS，这是否重要？

- Gal

13个回答

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- Shneor Elmaleh · Answer 1

import boto3

def move_s3_object(bucket: str, old_key: str, new_key: str) -> None:
    boto3.resource('s3').Object(bucket,  new_key).copy_from(CopySource=f'{bucket}/{old_key}')
    boto3.client('s3').delete_object(Bucket=bucket, Key=old_key)


# example:
move_s3_object('my_bucket', old_key='tmp/test.txt', new_key='tmp/tmp2/test.txt')

这可能甚至可以使用两个不同的桶，但我还没有测试过。

- Kumar Ashutosh · Answer 2

使用s3fs库可以轻松完成。

import s3fs

src = 'source_bucket'
dst = 'destination_bucket'

s3 = s3fs.S3FileSystem(anon=False,key='aws_s3_key',secret='aws_s3_secret_key')

for i in s3.ls(src,refresh=True): # loading the file names
    if 'file_name' in i:          # checking the file name
        s3.mv(i,dst)              # moving file to destination

这是文档。 https://s3fs.readthedocs.io/zh/latest/

- Ganesh Kharad · Answer 3

这是我用来在S3存储桶的子目录中移动文件的代码。

# =============================================================================
# CODE TO MOVE FILES within subfolders in S3 BUCKET
# =============================================================================

from boto3.session import Session

ACCESS_KEY = 'a_key'
SECRET_KEY = 's_key'
session = Session(aws_access_key_id=ACCESS_KEY,
            aws_secret_access_key=SECRET_KEY)
s3 = session.resource('s3')#creating session of S3 as resource


s3client = session.client('s3')

resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub_folder/', Delimiter="/")

forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]]#here we got all files list (max limit is 1000 at a time)
reload_no = 0
while len(forms2_dw) != 0 :

    #resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub_folder/', Delimiter="/")
    #with open('dw_bucket.json','w') as f:
    #    resp_dws =str(resp_dw)
       # f.write(json.dumps(resp_dws))
    #forms_dw = [x['Prefix'] for x in resp_dw['CommonPrefixes']] 
    #forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]]
    #forms2_dw[-1]
    total_files = len(forms2_dw)
    #i=0
    for i in range(total_files):
    #zip_filename='1819.zip'
        foldername = resp_dw['Contents'][1:][i]['LastModified'].strftime('%Y%m%d')#Put your logic here for folder name
        my_bcket   =  'main_bucket'

        my_file_old = resp_dw['Contents'][1:][i]['Key'] #file to be copied path
        zip_filename =my_file_old.split('/')[-1]
        subpath_nw='new_sub_folder/'+foldername+"/"+zip_filename #destination path
        my_file_new = subpath_nw
        # 
        print str(reload_no)+ ':::  copying from====:'+my_file_old+' to :====='+s3_archive_subpath_nw
        #print my_bcket+'/'+my_file_old 

        if zip_filename[-4:] == '.zip':
            s3.Object(my_bcket,my_file_new).copy_from(CopySource=my_bcket+'/'+my_file_old)
            s3.Object(my_bcket,my_file_old).delete()

            print str(i)+' files moved of '+str(total_files)

    resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub-folder/', Delimiter="/")

    forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]] 
    reload_no +=1