我需要使用Python Boto API将文件从一个存储桶移动到另一个存储桶。(我需要将文件从第一个存储桶中“剪切”并“粘贴”到第二个存储桶中)。
最佳的方法是什么?
**注意:如果我有两个不同的ACCESS KEYS和SECRET KEYS,这是否重要?
将一个对象从一个目录移动到另一个目录:
import boto3
def move_s3_object(bucket: str, old_key: str, new_key: str) -> None:
boto3.resource('s3').Object(bucket, new_key).copy_from(CopySource=f'{bucket}/{old_key}')
boto3.client('s3').delete_object(Bucket=bucket, Key=old_key)
# example:
move_s3_object('my_bucket', old_key='tmp/test.txt', new_key='tmp/tmp2/test.txt')
这可能甚至可以使用两个不同的桶,但我还没有测试过。
使用s3fs库可以轻松完成。
import s3fs
src = 'source_bucket'
dst = 'destination_bucket'
s3 = s3fs.S3FileSystem(anon=False,key='aws_s3_key',secret='aws_s3_secret_key')
for i in s3.ls(src,refresh=True): # loading the file names
if 'file_name' in i: # checking the file name
s3.mv(i,dst) # moving file to destination
# =============================================================================
# CODE TO MOVE FILES within subfolders in S3 BUCKET
# =============================================================================
from boto3.session import Session
ACCESS_KEY = 'a_key'
SECRET_KEY = 's_key'
session = Session(aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY)
s3 = session.resource('s3')#creating session of S3 as resource
s3client = session.client('s3')
resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub_folder/', Delimiter="/")
forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]]#here we got all files list (max limit is 1000 at a time)
reload_no = 0
while len(forms2_dw) != 0 :
#resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub_folder/', Delimiter="/")
#with open('dw_bucket.json','w') as f:
# resp_dws =str(resp_dw)
# f.write(json.dumps(resp_dws))
#forms_dw = [x['Prefix'] for x in resp_dw['CommonPrefixes']]
#forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]]
#forms2_dw[-1]
total_files = len(forms2_dw)
#i=0
for i in range(total_files):
#zip_filename='1819.zip'
foldername = resp_dw['Contents'][1:][i]['LastModified'].strftime('%Y%m%d')#Put your logic here for folder name
my_bcket = 'main_bucket'
my_file_old = resp_dw['Contents'][1:][i]['Key'] #file to be copied path
zip_filename =my_file_old.split('/')[-1]
subpath_nw='new_sub_folder/'+foldername+"/"+zip_filename #destination path
my_file_new = subpath_nw
#
print str(reload_no)+ '::: copying from====:'+my_file_old+' to :====='+s3_archive_subpath_nw
#print my_bcket+'/'+my_file_old
if zip_filename[-4:] == '.zip':
s3.Object(my_bcket,my_file_new).copy_from(CopySource=my_bcket+'/'+my_file_old)
s3.Object(my_bcket,my_file_old).delete()
print str(i)+' files moved of '+str(total_files)
resp_dw = s3client.list_objects(Bucket='main_bucket', Prefix='sub-folder/', Delimiter="/")
forms2_dw = [x['Key'] for x in resp_dw['Contents'][1:]]
reload_no +=1