我通过执行以下操作读取S3存储桶中的文件名：
objs = boto3.client.list_objects(Bucket='my_bucket')
while 'Contents' in objs.keys():
objs_contents = objs['Contents']
for i in range(len(objs_contents)):
filename = objs_contents[i]['Key']
现在，我需要获取文件的实际内容，类似于open(filename).readlines()的方式。最好的方法是什么？
StreamingBody 不支持 readline 或 readlines。s3 = boto3.resource('s3')
bucket = s3.Bucket('test-bucket')
# Iterates through all the objects, doing the pagination for you. Each obj
# is an ObjectSummary, so it doesn't contain the body. You'll need to call
# get to get the whole body.
for obj in bucket.objects.all():
key = obj.key
body = obj.get()['Body'].read()
使用客户端而不是资源：
s3 = boto3.client('s3')
bucket='bucket_name'
result = s3.list_objects(Bucket = bucket, Prefix='/something/')
for o in result.get('Contents'):
data = s3.get_object(Bucket=bucket, Key=o.get('Key'))
contents = data['Body'].read()
print(contents.decode("utf-8"))
您可以考虑使用支持迭代器的smart_open模块：
from smart_open import smart_open
# stream lines from an S3 object
for line in smart_open('s3://mybucket/mykey.txt', 'rb'):
print(line.decode('utf8'))
以及上下文管理器：
with smart_open('s3://mybucket/mykey.txt', 'rb') as s3_source:
for line in s3_source:
print(line.decode('utf8'))
s3_source.seek(0) # seek to the beginning
b1000 = s3_source.read(1000) # read 1000 bytes
可以在https://pypi.org/project/smart_open/找到smart_open。
当您想要使用不同于默认配置的配置读取文件时，可以直接使用mpu.aws.s3_read(s3path)，或者复制粘贴以下代码：
def s3_read(source, profile_name=None):
"""
Read a file from an S3 source.
Parameters
----------
source : str
Path starting with s3://, e.g. 's3://bucket-name/key/foo.bar'
profile_name : str, optional
AWS profile
Returns
-------
content : bytes
botocore.exceptions.NoCredentialsError
Botocore is not able to find your credentials. Either specify
profile_name or add the environment variables AWS_ACCESS_KEY_ID,
AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN.
See https://boto3.readthedocs.io/en/latest/guide/configuration.html
"""
session = boto3.Session(profile_name=profile_name)
s3 = session.client('s3')
bucket_name, key = mpu.aws._s3_path_split(source)
s3_object = s3.get_object(Bucket=bucket_name, Key=key)
body = s3_object['Body']
return body.read()
如果您已经知道filename，可以使用boto3内置的download_fileobj方法进行下载。
import boto3
from io import BytesIO
session = boto3.Session()
s3_client = session.client("s3")
f = BytesIO()
s3_client.download_fileobj("bucket_name", "filename", f)
print(f.getvalue())
f.seek(0)是不必要的。read从当前位置开始读取，但getvalue始终从位置0开始读取。 - Adam Hoelscherread来满足他们的用例。我只是为了演示目的使用了getvalue。 - reubanoimport boto3
print("started")
s3 = boto3.resource('s3',region_name='region_name', aws_access_key_id='your_access_id', aws_secret_access_key='your access key')
obj = s3.Object('bucket_name','file_name')
data=obj.get()['Body'].read()
print(data)
以下是使用boto3从s3存储桶访问文件内容的正确且经过测试的代码。截至发布日期，该代码对我有效。
def get_file_contents(bucket, prefix):
s3 = boto3.resource('s3')
s3.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
bucket = s3.Bucket(bucket)
for obj in bucket.objects.filter(Prefix=prefix):
key = obj.key
body = obj.get()['Body'].read()
print(body)
return body
get_file_contents('coderbytechallengesandbox', '__cb__')
from s3fs import S3FileSystem
s3 = S3FileSystem()
bucket = 's3://your-bucket'
def read_file(key):
with s3.open(f'{s3_path}/{key}', 'r') as file: # s3://bucket/file.txt
return file.readlines()
for obj in bucket.objects.all():
key = obj.key
lines = read_file(key)
...
对我来说，最好的方法是这样的：
result = s3.list_objects(Bucket = s3_bucket, Prefix=s3_key)
for file in result.get('Contents'):
data = s3.get_object(Bucket=s3_bucket, Key=file.get('Key'))
contents = data['Body'].read()
#if Float types are not supported with dynamodb; use Decimal types instead
j = json.loads(contents, parse_float=Decimal)
for item in j:
timestamp = item['timestamp']
table.put_item(
Item={
'timestamp': timestamp
}
)
一旦您有了内容，您可以通过另一个循环运行它，以便将其写入DynamoDB表中。
Resources，建议的方法是返回使用Client。
.filter()减少列表请求。或者如果您知道所需的键，可以直接使用bucket.Object('mykey')获取它。 - Jordon Phillips