Requests是一个非常好用的库，我想用它来下载大文件（>1GB）。 问题是无法将整个文件保存在内存中，我需要分块读取。下面的代码存在这个问题:
import requests
def DownloadFile(url)
local_filename = url.split('/')[-1]
r = requests.get(url)
f = open(local_filename, 'wb')
for chunk in r.iter_content(chunk_size=512 * 1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.close()
return
以某种原因，它不能按照这种方式工作；它在保存到文件之前仍然会将响应加载到内存中。
使用以下流式代码，即使下载文件的大小不同，也可以限制Python的内存使用情况：
def download_file(url):
local_filename = url.split('/')[-1]
# NOTE the stream=True parameter below
with requests.get(url, stream=True) as r:
r.raise_for_status()
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=8192):
# If you have chunk encoded response uncomment if
# and set chunk_size parameter to None.
#if chunk:
f.write(chunk)
return local_filename
请注意，使用iter_content返回的字节数并不完全等于chunk_size；它预计是一个通常要大得多的随机数，并且预计在每次迭代中都会有所不同。
有关更多信息，请参见body-content-workflow和Response.iter_content。
如果您使用Response.raw和shutil.copyfileobj()，那么会更容易些：
import requests
import shutil
def download_file(url):
local_filename = url.split('/')[-1]
with requests.get(url, stream=True) as r:
with open(local_filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)
return local_filename
这种方法可以将文件流写入磁盘，而不会使用过多的内存，代码也很简单。
注意：根据文档，Response.raw 不会解码 gzip 和 deflate 的传输编码方式，因此您需要手动进行解码。
.raw需要注意一个小细节，它不会处理解码。文档在这里提到：http://docs.python-requests.org/en/master/user/quickstart/#raw-response-content - Eric Cousineauraw 和 iter_content 的下载速度进行匹配之后，我将 chunk_size 从 1024 增加到了 10*1024 ，这样就可以使它们的下载速度相同了（针对 Debian ISO 文件和普通连接）。 - Jan Benesread方法：response.raw.read = functools.partial(response.raw.read，decode_content=True)]（https://github.com/requests/requests/issues/2155#issuecomment-50771010） - Nuno André虽然不完全是原始问题所问的，但使用urllib做到这一点非常容易：
from urllib.request import urlretrieve
url = 'http://mirror.pnl.gov/releases/16.04.2/ubuntu-16.04.2-desktop-amd64.iso'
dst = 'ubuntu-16.04.2-desktop-amd64.iso'
urlretrieve(url, dst)
或者采用这种方式，如果你想将它保存到临时文件中：
from urllib.request import urlopen
from shutil import copyfileobj
from tempfile import NamedTemporaryFile
url = 'http://mirror.pnl.gov/releases/16.04.2/ubuntu-16.04.2-desktop-amd64.iso'
with urlopen(url) as fsrc, NamedTemporaryFile(delete=False) as fdst:
copyfileobj(fsrc, fdst)
我观察了这个过程：
watch 'ps -p 18647 -o pid,ppid,pmem,rsz,vsz,comm,args; ls -al *.iso'
我发现文件大小在增加，但内存使用量保持在17兆字节。我是否遗漏了什么？
from urllib import urlretrieve。 - Vadim Kotov你的块大小可能太大了，尝试降低它-也许每次降低1024字节？（另外，你可以使用with来简化语法）
def DownloadFile(url):
local_filename = url.split('/')[-1]
r = requests.get(url)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return
顺便问一下，您是如何推断响应已加载到内存中的？
听起来像是Python没有将数据刷新到文件中。从其他SO问题中可以尝试使用f.flush()和os.fsync()强制进行文件写入并释放内存;
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
os.fsync(f.fileno())
f.flush(); os.fsync() 可以强制写入并释放内存。 - danodonovanos.fsync(f.fileno())。 - sebdelsoldef DownloadFile(url) 后面缺少冒号（“：”）。 - Aubrey使用 Python 的 wget 模块代替。以下是代码片段：
import wget
wget.download(url)
参考Roman上面最受赞的评论，这是我的实现， 包括“下载为”和“重试”机制：
def download(url: str, file_path='', attempts=2):
"""Downloads a URL content into a file (with large file support by streaming)
:param url: URL to download
:param file_path: Local file name to contain the data downloaded
:param attempts: Number of attempts
:return: New file path. Empty string if the download failed
"""
if not file_path:
file_path = os.path.realpath(os.path.basename(url))
logger.info(f'Downloading {url} content to {file_path}')
url_sections = urlparse(url)
if not url_sections.scheme:
logger.debug('The given url is missing a scheme. Adding http scheme')
url = f'http://{url}'
logger.debug(f'New url: {url}')
for attempt in range(1, attempts+1):
try:
if attempt > 1:
time.sleep(10) # 10 seconds wait time between downloads
with requests.get(url, stream=True) as response:
response.raise_for_status()
with open(file_path, 'wb') as out_file:
for chunk in response.iter_content(chunk_size=1024*1024): # 1MB chunks
out_file.write(chunk)
logger.info('Download finished successfully')
return file_path
except Exception as ex:
logger.error(f'Attempt #{attempt} failed with error: {ex}')
return ''
asyncio库(aiohttp读取URL和aiofiles写文件)。Python 3.7及以上版本。SRC_URL和DEST_FILE变量即可。import aiofiles
import aiohttp
import asyncio
async def async_http_download(src_url, dest_file, chunk_size=65536):
async with aiofiles.open(dest_file, 'wb') as fd:
async with aiohttp.ClientSession() as session:
async with session.get(src_url) as resp:
async for chunk in resp.content.iter_chunked(chunk_size):
await fd.write(chunk)
SRC_URL = "/path/to/url"
DEST_FILE = "/path/to/file/on/local/machine"
asyncio.run(async_http_download(SRC_URL, DEST_FILE))
requests很好，但socket解决方案怎么样？def stream_(host):
import socket
import ssl
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
context = ssl.create_default_context(Purpose.CLIENT_AUTH)
with context.wrap_socket(sock, server_hostname=host) as wrapped_socket:
wrapped_socket.connect((socket.gethostbyname(host), 443))
wrapped_socket.send(
"GET / HTTP/1.1\r\nHost:thiscatdoesnotexist.com\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9\r\n\r\n".encode())
resp = b""
while resp[-4:-1] != b"\r\n\r":
resp += wrapped_socket.recv(1)
else:
resp = resp.decode()
content_length = int("".join([tag.split(" ")[1] for tag in resp.split("\r\n") if "content-length" in tag.lower()]))
image = b""
while content_length > 0:
data = wrapped_socket.recv(2048)
if not data:
print("EOF")
break
image += data
content_length -= len(data)
with open("image.jpeg", "wb") as file:
file.write(image)
另一种下载大文件的选择。这将允许您停止并稍后继续（按Enter键停止），如果您的连接断开，可以从上次离开的地方继续。
import datetime
import os
import requests
import threading as th
keep_going = True
def key_capture_thread():
global keep_going
input()
keep_going = False
pkey_capture = th.Thread(target=key_capture_thread, args=(), name='key_capture_process', daemon=True).start()
def download_file(url, local_filepath):
#assumptions:
# headers contain Content-Length:
# headers contain Accept-Ranges: bytes
# stream is not encoded (otherwise start bytes are not known, unless this is stored seperately)
chunk_size = 1048576 #1MB
# chunk_size = 8096 #8KB
# chunk_size = 1024 #1KB
decoded_bytes_downloaded_this_session = 0
start_time = datetime.datetime.now()
if os.path.exists(local_filepath):
decoded_bytes_downloaded = os.path.getsize(local_filepath)
else:
decoded_bytes_downloaded = 0
with requests.Session() as s:
with s.get(url, stream=True) as r:
#check for required headers:
if 'Content-Length' not in r.headers:
print('STOP: request headers do not contain Content-Length')
return
if ('Accept-Ranges','bytes') not in r.headers.items():
print('STOP: request headers do not contain Accept-Ranges: bytes')
with s.get(url) as r:
print(str(r.content, encoding='iso-8859-1'))
return
content_length = int(r.headers['Content-Length'])
if decoded_bytes_downloaded>=content_length:
print('STOP: file already downloaded. decoded_bytes_downloaded>=r.headers[Content-Length]; {}>={}'.format(decoded_bytes_downloaded,r.headers['Content-Length']))
return
if decoded_bytes_downloaded>0:
s.headers['Range'] = 'bytes={}-{}'.format(decoded_bytes_downloaded, content_length-1) #range is inclusive
print('Retrieving byte range (inclusive) {}-{}'.format(decoded_bytes_downloaded, content_length-1))
with s.get(url, stream=True) as r:
r.raise_for_status()
with open(local_filepath, mode='ab') as fwrite:
for chunk in r.iter_content(chunk_size=chunk_size):
decoded_bytes_downloaded+=len(chunk)
decoded_bytes_downloaded_this_session+=len(chunk)
time_taken:datetime.timedelta = (datetime.datetime.now() - start_time)
seconds_per_byte = time_taken.total_seconds()/decoded_bytes_downloaded_this_session
remaining_bytes = content_length-decoded_bytes_downloaded
remaining_seconds = seconds_per_byte * remaining_bytes
remaining_time = datetime.timedelta(seconds=remaining_seconds)
#print updated statistics here
fwrite.write(chunk)
if not keep_going:
break
output_folder = '/mnt/HDD1TB/DownloadsBIG'
# url = 'https://file-examples.com/storage/fea508993d645be1b98bfcf/2017/10/file_example_JPG_100kB.jpg'
# url = 'https://file-examples.com/storage/fe563fce08645a90397f28d/2017/10/file_example_JPG_2500kB.jpg'
url = 'https://ftp.ncbi.nlm.nih.gov/blast/db/nr.00.tar.gz'
local_filepath = os.path.join(output_folder, os.path.split(url)[-1])
download_file(url, local_filepath)