我想直接将multipart/form-data(大型)文件上传到AWS S3,并尽可能地减少内存和文件磁盘占用。 我该如何实现? 在线资源仅解释如何上传并在服务器上本地存储文件。
我想直接将multipart/form-data(大型)文件上传到AWS S3,并尽可能地减少内存和文件磁盘占用。 我该如何实现? 在线资源仅解释如何上传并在服务器上本地存储文件。
package main
import (
"fmt"
"os"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
var filename = "file_name.zip"
var myBucket = "myBucket"
var myKey = "file_name.zip"
var accessKey = ""
var accessSecret = ""
func main() {
var awsConfig *aws.Config
if accessKey == "" || accessSecret == "" {
//load default credentials
awsConfig = &aws.Config{
Region: aws.String("us-west-2"),
}
} else {
awsConfig = &aws.Config{
Region: aws.String("us-west-2"),
Credentials: credentials.NewStaticCredentials(accessKey, accessSecret, ""),
}
}
// The session the S3 Uploader will use
sess := session.Must(session.NewSession(awsConfig))
// Create an uploader with the session and default options
//uploader := s3manager.NewUploader(sess)
// Create an uploader with the session and custom options
uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
u.PartSize = 5 * 1024 * 1024 // The minimum/default allowed part size is 5MB
u.Concurrency = 2 // default is 5
})
//open the file
f, err := os.Open(filename)
if err != nil {
fmt.Printf("failed to open file %q, %v", filename, err)
return
}
//defer f.Close()
// Upload the file to S3.
result, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(myBucket),
Key: aws.String(myKey),
Body: f,
})
//in case it fails to upload
if err != nil {
fmt.Printf("failed to upload file, %v", err)
return
}
fmt.Printf("file uploaded to, %s\n", result.Location)
}
我试图使用aws-sdk v2包来完成这个任务,所以必须稍微修改@maaz的代码。我把它留在这里供其他人参考 -
type TokenMeta struct {
AccessToken string
SecretToken string
SessionToken string
BucketName string
}
// Create S3Client struct with the token meta and use it as a receiver for this method
func (s3Client S3Client) StreamUpload(fileToUpload string, fileKey string) error {
accessKey := s3Client.TokenMeta.AccessToken
secretKey := s3Client.TokenMeta.SecretToken
awsConfig, err := config.LoadDefaultConfig(context.TODO(),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKey, secretKey, s3Client.TokenMeta.SessionToken)),
)
if err != nil {
return fmt.Errorf("error creating aws config: %v", err)
}
client := s3.NewFromConfig(awsConfig)
uploader := manager.NewUploader(client, func(u *manager.Uploader) {
u.PartSize = 5 * 1024 * 1024
u.BufferProvider = manager.NewBufferedReadSeekerWriteToPool(10 * 1024 * 1024)
})
f, err := os.Open(fileToUpload)
if err != nil {
return fmt.Errorf("failed to open fileToUpload %q, %v", fileToUpload, err)
}
defer func(f *os.File) {
err := f.Close()
if err != nil {
fmt.Errorf("error closing fileToUpload: %v", err)
}
}(f)
inputObj := &s3.PutObjectInput{
Bucket: aws.String(s3Client.TokenMeta.BucketName),
Key: aws.String(fileKey),
Body: f,
}
uploadResult, err := uploader.Upload(context.TODO(), inputObj)
if err != nil {
return fmt.Errorf("failed to uploadResult fileToUpload, %v", err)
}
fmt.Printf("%s uploaded to, %s\n", fileToUpload, uploadResult.Location)
return nil
}