我需要在大约 600k 个 Blob 上运行批处理作业，发现了两个真正有用的方法：
并行运行操作。.NET v4 中的任务并行库（TPL）使这变得非常容易。以下是设置容器中每个 Blob 的缓存控制标头的并行代码：
// get the info for every blob in the container
var blobInfos = cloudBlobContainer.ListBlobs(
new BlobRequestOptions() { UseFlatBlobListing = true });
Parallel.ForEach(blobInfos, (blobInfo) =>
{
// get the blob properties
CloudBlob blob = container.GetBlobReference(blobInfo.Uri.ToString());
blob.FetchAttributes();
// set cache-control header if necessary
if (blob.Properties.CacheControl != YOUR_CACHE_CONTROL_HEADER)
{
blob.Properties.CacheControl = YOUR_CACHE_CONTROL_HEADER;
blob.SetProperties();
}
});
5. 删除临时网站，或将其更改为免费版(在“扩展”下)。using Azure;
using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
namespace AzureHeaders
{
class Program
{
private static string connectionString = "DefaultEndpointsProtocol=https;AccountName=REPLACE_WITH_YOUR_CONNECTION_STRING";
private static string newCacheControl = "public, max-age=7776001"; // 3 months
private static string[] containersToProcess = { "container1", "container2" };
static async Task Main(string[] args)
{
BlobServiceClient blobServiceClient = new BlobServiceClient(connectionString);
var tasks = new List<Task>();
foreach (var container in containersToProcess)
{
BlobContainerClient containerClient = blobServiceClient.GetBlobContainerClient(container);
tasks.Add(Task.Run(() => UpdateHeaders(containerClient, 1000))); // I have no idea what segmentSize should be!
}
Task.WaitAll(tasks.ToArray());
}
private static async Task UpdateHeaders(BlobContainerClient blobContainerClient, int? segmentSize)
{
int processed = 0;
int failed = 0;
try
{
// Call the listing operation and return pages of the specified size.
var resultSegment = blobContainerClient.GetBlobsAsync()
.AsPages(default, segmentSize);
// Enumerate the blobs returned for each page.
await foreach (Azure.Page<BlobItem> blobPage in resultSegment)
{
var tasks = new List<Task>();
foreach (BlobItem blobItem in blobPage.Values)
{
BlobClient blobClient = blobContainerClient.GetBlobClient(blobItem.Name);
tasks.Add(UpdateOneBlob(blobClient));
processed++;
}
Task.WaitAll(tasks.ToArray());
Console.WriteLine($"Container {blobContainerClient.Name} processed: {processed}");
}
}
catch (RequestFailedException e)
{
Console.WriteLine(e.Message);
failed++;
}
Console.WriteLine($"Container {blobContainerClient.Name} processed: {processed}, failed: {failed}");
}
private static async Task UpdateOneBlob(BlobClient blobClient) {
Response<BlobProperties> propertiesResponse = await blobClient.GetPropertiesAsync();
BlobHttpHeaders httpHeaders = new BlobHttpHeaders
{
// copy any existing headers you wish to preserve
ContentType = propertiesResponse.Value.ContentType,
ContentHash = propertiesResponse.Value.ContentHash,
ContentEncoding = propertiesResponse.Value.ContentEncoding,
ContentDisposition = propertiesResponse.Value.ContentDisposition,
// update CacheControl
CacheControl = newCacheControl
};
await blobClient.SetHttpHeadersAsync(httpHeaders);
}
}
}
Cache-Control属性设置值（例如public，max-age=2592000）。 public async Task BackfillCacheControlAsync()
{
var container = await GetCloudBlobContainerAsync();
BlobContinuationToken continuationToken = null;
do
{
var blobInfos = await container.ListBlobsSegmentedAsync(string.Empty, true, BlobListingDetails.None, null, continuationToken, null, null);
continuationToken = blobInfos.ContinuationToken;
foreach (var blobInfo in blobInfos.Results)
{
var blockBlob = (CloudBlockBlob)blobInfo;
var blob = await container.GetBlobReferenceFromServerAsync(blockBlob.Name);
if (blob.Properties.CacheControl != "public, max-age=31536000")
{
blob.Properties.CacheControl = "public, max-age=31536000";
await blob.SetPropertiesAsync();
}
}
}
while (continuationToken != null);
}
private async Task<CloudBlobContainer> GetCloudBlobContainerAsync()
{
var storageAccount = CloudStorageAccount.Parse(_appSettings.AzureStorageConnectionString);
var blobClient = storageAccount.CreateCloudBlobClient();
var container = blobClient.GetContainerReference("uploads");
return container;
}
# Update Azure Blob Storage blob's cache-control headers
# /content-cache properties
#
# Quite slow, since there is no `az storage blob update-batch`
#
# Created by Jon Tingvold, March 2021
#
#
# If you want progress, you need to install pv:
# >>> brew install pv # Mac
# >>> sudo apt install pv # Ubuntu
#
set -e # exit when any command fails
AZURE_BLOB_CONNECTION_STRING='DefaultEndpointsProtocol=https;EndpointSuffix=core.windows.net;AccountName=XXXXXXXXXXXX;AccountKey=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX=='
CONTAINER_NAME=main
BLOB_PREFIX='admin/'
CONTENT_CACHE='max-age=3600'
NUM_RESULTS=10000000 # Defaults to 5000
BLOB_NAMES=$(az storage blob list --connection-string $AZURE_BLOB_CONNECTION_STRING --container-name $CONTAINER_NAME --query '[].name' --output tsv --num-results $NUM_RESULTS --prefix $BLOB_PREFIX)
NUMBER_OF_BLOBS=$(echo $BLOB_NAMES | wc -w)
echo "Ask Azure for files in Blob Storage ..."
echo "Set content-cache on $NUMBER_OF_BLOBS blobs ..."
for BLOB_NAME in $BLOB_NAMES
do
az storage blob update --connection-string $AZURE_BLOB_CONNECTION_STRING --container-name $CONTAINER_NAME --name $BLOB_NAME --content-cache $CONTENT_CACHE > /dev/null;
echo "$BLOB_NAME"
# If you don't have pv install, uncomment everything after done
done | cat | pv -pte --line-mode --size $NUMBER_OF_BLOBS > /dev/null
Add-Type -Path "C:\Program Files\Microsoft SDKs\Windows Azure\.NET SDK\v2.3\ref\Microsoft.WindowsAzure.StorageClient.dll"
$accountName = "[azureaccountname]"
$accountKey = "[azureaccountkey]"
$blobContainerName = "images"
$storageCredentials = New-Object Microsoft.WindowsAzure.StorageCredentialsAccountAndKey -ArgumentList $accountName,$accountKey
$storageAccount = New-Object Microsoft.WindowsAzure.CloudStorageAccount -ArgumentList $storageCredentials,$true
#$blobClient = $storageAccount.CreateCloudBlobClient()
$blobClient = [Microsoft.WindowsAzure.StorageClient.CloudStorageAccountStorageClientExtensions]::CreateCloudBlobClient($storageAccount)
$cacheControlValue = "public, max-age=604800"
echo "Setting cache control: $cacheControlValue"
Get-Content "imagelist.txt" | foreach {
$blobName = "$blobContainerName/$_".Trim()
echo $blobName
$blob = $blobClient.GetBlobReference($blobName)
$blob.Properties.CacheControl = $cacheControlValue
$blob.SetProperties()
}
使用 PowerShell 脚本设置存储 Blob 缓存控制属性
https://gallery.technet.microsoft.com/How-to-set-storage-blob-4774aca5
#creat CloudBlobClient
Add-Type -Path "C:\Program Files\Microsoft SDKs\Windows Azure\.NET SDK\v2.3\ref\Microsoft.WindowsAzure.StorageClient.dll"
$storageCredentials = New-Object Microsoft.WindowsAzure.StorageCredentialsAccountAndKey -ArgumentList $StorageName,$StorageKey
$blobClient = New-Object Microsoft.WindowsAzure.StorageClient.CloudBlobClient($BlobUri,$storageCredentials)
#set Properties and Metadata
$cacheControlValue = "public, max-age=60480"
foreach ($blob in $blobs)
{
#set Metadata
$blobRef = $blobClient.GetBlobReference($blob.Name)
$blobRef.Metadata.Add("abcd","abcd")
$blobRef.SetMetadata()
#set Properties
$blobRef.Properties.CacheControl = $cacheControlValue
$blobRef.SetProperties()
}