如何在文件下载时打印字节？- Golang

Question

我想知道在文件下载时是否有可能计算和打印下载的字节数。

out, err := os.Create("file.txt")
defer out.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}
resp, err := http.Get("http://example.com/zip")
defer resp.Body.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}

n, er := io.Copy(out, resp.Body)
if er != nil {
    fmt.Println(fmt.Sprint(err))
}
fmt.Println(n, "bytes ")

- The user with no hat

---

你能否详细说明你的问题，而不是用填充语言来夸大它？你尝试过什么？有什么不起作用的地方吗？ - Graham Savage

我对着电脑大喊大叫，但它还是不工作 :) - The user with no hat

“打印字节”是什么意思？是指正在下载的文件的字节吗？还是一些任意的数据？或者是迄今已下载的字节数？具体指的是什么？ - kostix

是的，目前已下载的字节数。我认为这很明显。 - The user with no hat

Answer 1

如果我理解正确的话，您希望在数据传输时显示读取的字节数，可能是为了维护某种进度条或其他东西。在这种情况下，您可以使用Go的组合数据结构来将读取器或写入器包装在自定义的io.Reader或io.Writer实现中。

它简单地将相应的Read或Write调用转发到底层流，同时对它们返回的(int, error)值进行一些额外的处理。以下是一个您可以在Go playground上运行的示例。

package main

import (
    "bytes"
    "fmt"
    "io"
    "os"
    "strings"
)

// PassThru wraps an existing io.Reader.
//
// It simply forwards the Read() call, while displaying
// the results from individual calls to it.
type PassThru struct {
    io.Reader
    total int64 // Total # of bytes transferred
}

// Read 'overrides' the underlying io.Reader's Read method.
// This is the one that will be called by io.Copy(). We simply
// use it to keep track of byte counts and then forward the call.
func (pt *PassThru) Read(p []byte) (int, error) {
    n, err := pt.Reader.Read(p)
    pt.total += int64(n)

    if err == nil {
        fmt.Println("Read", n, "bytes for a total of", pt.total)
    }

    return n, err
}

func main() {
    var src io.Reader    // Source file/url/etc
    var dst bytes.Buffer // Destination file/buffer/etc

    // Create some random input data.
    src = bytes.NewBufferString(strings.Repeat("Some random input data", 1000))

    // Wrap it with our custom io.Reader.
    src = &PassThru{Reader: src}

    count, err := io.Copy(&dst, src)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    fmt.Println("Transferred", count, "bytes")
}

它所生成的输出是这样的：

Read 512 bytes for a total of 512
Read 1024 bytes for a total of 1536
Read 2048 bytes for a total of 3584
Read 4096 bytes for a total of 7680
Read 8192 bytes for a total of 15872
Read 6128 bytes for a total of 22000
Transferred 22000 bytes

- jimt

---

你假设如果 err != nil 那么 n == 0，但这并非一定正确。io 包中的 Reader 类型：Read 函数会读取最多 len(p) 字节到 p 中，并返回读取的字节数（0 <= n <= len(p)）和任何遇到的错误。 - peterSO

pt.Reader.Read 真的从网络中读取数据吗？在 http.Get 返回响应后，下载是否已完成？OP 说：“当文件正在下载时”。 - 425nesp

有没有办法让src = "/tmp/some-file.pdf"？有人可以帮忙吗？ - Jacky Supit

能否对TCP连接（net.Dial）进行仪器化呢？当我尝试对conn reader对象进行仪器化时，出现“未实现net.Conn（缺少Close方法）”的错误。 - Zyber

Answer 2

现在标准库提供了类似jimt的PassThru功能：io.TeeReader。它可以帮助简化一些事情：

// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
    Total int64 // Total # of bytes transferred
}

// Write implements the io.Writer interface.  
// 
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
    n := len(p)
    wc.Total += int64(n)
    fmt.Printf("Read %d bytes for a total of %d\n", n, wc.Total)
    return n, nil
}

func main() {

    // ...    

    // Wrap it with our custom io.Reader.
    src = io.TeeReader(src, &WriteCounter{})

    // ...
}

playground

- Dave Jack

---

io.TeeReader 是当今最好的选择；这应该是被接受的答案。 - Eric Dand

Answer 3

基础 @Dave Jack

我添加了进度值并从NC（直接TCP数据传输）接收文件数据。

// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
 Total      int64 // Total # of bytes transferred
 Last       int64
 LastUpdate time.Time
}

// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
  n := len(p)
  wc.Total += int64(n)
  now := time.Now()
  duration := now.Sub(wc.LastUpdate).Seconds()
  if duration > 1 {
    wc.LastUpdate = now

    rate := float64(wc.Total-wc.Last) / (duration) / 1024.0
    wc.Last = wc.Total

    fmt.Printf("Read %d bytes for a total of %d , Rate  %.1fKb/s \n", n, wc.Total, rate)
  }

 return n, nil
}

func Server(dest string) {
  outputFile, err := os.Create(dest)
  if err != nil {
    fmt.Println(err)
  }
  defer outputFile.Close()
  fileWriter := bufio.NewWriter(outputFile)

  serverListener, err := net.Listen("tcp", "0.0.0.0:"+PORT)
  if err != nil {
    fmt.Println(err)
  }
  defer serverListener.Close()

  serverConn, err := serverListener.Accept()
  if err != nil {
    fmt.Println(err)
  }
  defer serverConn.Close()

  wc := &WriteCounter{}
  reader := io.TeeReader(serverConn, wc)
  serverConnReader := bufio.NewReaderSize(reader, 32*1024*1024)

  io.Copy(fileWriter, serverConnReader)
  fileWriter.Flush()
  outputFile.Sync()
  fmt.Println("Done: Writer")
 }

- temple

Answer 4

grab Go包用于实现文件下载的进度更新（以及许多其他功能）。

以下步骤演示如何在下载过程中打印进度更新：http://cavaliercoder.com/blog/downloading-large-files-in-go.html

您可以基本上调用grab.GetAsync，这将在新的Go协程中进行下载，然后从调用线程监视返回的grab.Response的BytesTransferred或Progress。

- Ryan Armstrong

Answer 5

其他答案已经解释了PassThru。根据Dave Jack的回答，提供一个带有回调函数的完整示例。

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "strconv"
)

// writeCounter counts the number of bytes written to it.
type writeCounter struct {
    total      int64 // total size
    downloaded int64 // downloaded # of bytes transferred
    onProgress func(downloaded int64, total int64)
}

// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *writeCounter) Write(p []byte) (n int, e error) {
    n = len(p)
    wc.downloaded += int64(n)
    wc.onProgress(wc.downloaded, wc.total)
    return
}

func newWriter(size int64, onProgress func(downloaded, total int64)) io.Writer {
    return &writeCounter{total: size, onProgress: onProgress}
}

func main() {
    client := http.DefaultClient
    url := "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
    saveTo := "/Users/tin/Desktop/ForBiggerFun.mp4"

    download(client, url, saveTo, func(downloaded, total int64) {
        fmt.Printf("Downloaded %d bytes for a total of %d\n", downloaded, total)
    })
}

func download(client *http.Client, url, filePath string, onProgress func(downloaded, total int64)) (err error) {
    // Create file writer
    file, err := os.Create(filePath)
    if err != nil {
        return
    }
    defer file.Close()

    // Determinate the file size
    resp, err := client.Head(url)
    if err != nil {
        return
    }
    contentLength := resp.Header.Get("content-length")
    length, err := strconv.Atoi(contentLength)
    if err != nil {
        return
    }

    // Make request
    resp, err = client.Get(url)
    if err != nil {
        return
    }
    defer resp.Body.Close()

    // pipe stream
    body := io.TeeReader(resp.Body, newWriter(int64(length), onProgress))
    _, err = io.Copy(file, body)
    return err
}

- Ninh Pham