从URL下载文件并将其上传到AWS S3而无需保存 - 使用Node.js

Question

从URL下载文件并将其上传到AWS S3而无需保存 - 使用Node.js

javascriptnode.jsamazon-web-servicesamazon-s3fs

43

我正在编写一个应用程序，它从URL下载图像，然后使用aws-sdk将其上传到S3存储桶。

之前，我只是像这样下载图像并将它们保存到磁盘中。

request.head(url, function(err, res, body){

    request(url).pipe(fs.createWriteStream(image_path));

});

然后像这样将图片上传到 AWS S3

fs.readFile(image_path, function(err, data){
    s3.client.putObject({
        Bucket: 'myBucket',
        Key: image_path,
        Body: data
        ACL:'public-read'
    }, function(err, resp) {
        if(err){
            console.log("error in s3 put object cb");
        } else { 
            console.log(resp);
            console.log("successfully added image to s3");
        }
    });
});

不过我想跳过把图片保存到磁盘的步骤。有没有办法我可以使用pipe函数将request(url)的响应传递到一个变量，然后上传它？

- Loourr

在iOS上能否做到同样的事情？ - CodeCracker

7个回答

16

这是我所做的，而且效果很好:

const request = require('request-promise')
const AWS = require('aws-sdk')
const s3 = new AWS.S3()

const options = {
    uri: uri,
    encoding: null
};

async load() {

  const body = await request(options)
  
  const uploadResult = await s3.upload({
    Bucket: 'bucket_name',
    Key   : path,
    Body  : body,   
  }).promise()
  
}

- Sietze Keuning

请问您能否指定路径参数？ - insivika

5

那么这个问题会是什么样子呢：

const stream = require('stream');
const request = require('request');
const s3 = new AWS.S3()

const pass = new stream.PassThrough();

request(url).pipe(pass);

s3.upload({
    Bucket: 'bucket_name',
    Key: path,
    Body: pass,
});

- user108828

3

import axios from "axios";
import aws from 'aws-sdk'
import crypto from 'crypto'

const s3 = new aws.S3();

export const urlToS3 = async ({ url, bucket = "rememoio-users", key = Date.now() + crypto.randomBytes(8).toString('hex') + ".png" }) => {
  try {
    const { data } = await axios.get(url, { responseType: "stream" });

    const upload = await s3.upload({
      Bucket: bucket,
      ACL: 'public-read',
      Key: key,
      Body: data,
    }).promise();

    return upload.Location;
  } catch (error) {
    console.error(error);
    throw new Error;
  }
};

- Daniel

2

使用 fetch：

//fetch image from url
const imageResp = await fetch(
    '<image url>'
)
// transform to arrayBuffer
const imageBinaryBuffer = Buffer.from(await imageResp.arrayBuffer())
//get image type
const imageType = imageName.toLowerCase().includes(".png")
    ? "image/png"
    : "image/jpg";

//get presigned url and data [this can be different on your end]
const presignedResponse = await getPresignedURL(imageBinaryBuffer, imageName, imageType)

const s3Result = presignedResponse.data
// build the formData 
let formData = new FormData()
Object.keys(s3Result.fields).forEach(key => {
    formData.append(key, s3Result.fields[key]);
});
formData.append("file", imageBinaryBuffer);

const s3resp = await fetch(s3Result.url, {
    method: "POST",
    body: formData,
});

return s3resp.headers.location

- Moises Macias

2

您可以像这样使用Axios进行实现。有关更多信息，请参阅this。

const axios = require("axios");
const AWS = require("aws-sdk");
const { PassThrough } = require("stream");

const s3 = new AWS.S3({
  accessKeyId: "accessKeyId",
  secretAccessKey: "accessKey",
  region: "region",
});

const bucket = "BucketName";
const key = "key";

const uploadToS3 = async (bucket, key) => {
  try {
    const stream = await axios.get(url, { responseType: "stream" });

    const passThrough = new PassThrough();

    const response = s3.upload({ Bucket: bucket, Key: key, Body: passThrough });

    stream.data.pipe(passThrough);

    return response.then((data) => data.Location).catch((e) => console.error(e));
  } catch (error) {
    console.error(error);
  }
};

uploadToS3(bucket, key);

- bkmalan

0

使用axios和responseType arrayBuffer，这个可以正常工作。

axios
   .get(url, { responseType: 'arraybuffer' })
   .then((resp) => {
        S3.putObject(
            {
                Bucket: s3.bucket,
                Key: imageName,
                ACL: 'public-read',
                Body: resp.data,
            },
            async (err) => {
               if (err) {
                    log.error(`Could not save image from URL: ${url}, Error =>${err}`);
               } else {log.info(`successfully added image to s3 from url: ${url}`);
               }
            }
        );
    })

- Yogini Bende

网页内容由stack overflow 提供, 点击上面的

可以查看英文原文，
原文链接

- Loourr · Accepted Answer

这里有一些很好地实现了这个功能的JavaScript代码：

    var options = {
        uri: uri,
        encoding: null
    };
    request(options, function(error, response, body) {
        if (error || response.statusCode !== 200) { 
            console.log("failed to get image");
            console.log(error);
        } else {
            s3.putObject({
                Body: body,
                Key: path,
                Bucket: 'bucket_name'
            }, function(error, data) { 
                if (error) {
                    console.log("error downloading image to s3");
                } else {
                    console.log("success uploading to s3");
                }
            }); 
        }   
    });