最近我在我的电子商务应用程序中存储产品时不得不实现这一点。由于我每4个小时必须插入10000个项目,所以我的数据库经常超时。对我来说,一种选择是在连接到数据库时在mongoose中设置socketTimeoutMS和connectTimeoutMS,但这种方法感觉有些hacky,而且我不想操纵数据库的连接超时默认值。我还看到@neil lunn的解决方案采用了一个简单的同步方法,在for循环内部进行取模运算。这里是我的异步版本,我相信它可以更好地完成工作。
let BATCH_SIZE = 500
Array.prototype.chunk = function (groupsize) {
var sets = [];
var chunks = this.length / groupsize;
for (var i = 0, j = 0; i < chunks; i++ , j += groupsize) {
sets[i] = this.slice(j, j + groupsize);
}
return sets;
}
function upsertDiscountedProducts(products) {
let chunks = products.chunk(BATCH_SIZE), current = 0
console.log('Number of chunks ', chunks.length)
let bulk = models.Product.collection.initializeUnorderedBulkOp();
let timestamp = new Date(),
pendingCount = 0,
inserted = 0,
upserted = 0,
matched = 0,
modified = 0,
removed = 0,
upsertHappened = false;
load()
function load() {
if (current < chunks.length) {
console.log('Current value ', current)
for (let i = 0; i < chunks[current].length; i++) {
let item = chunks[current][i]
item.updatedAt = timestamp;
bulk.find({ _id: item._id })
.upsert()
.updateOne({
"$set": item,
"$setOnInsert": {
"createdAt": timestamp
}
})
}
bulk.execute((error, result) => {
if (error) {
console.error('Error while inserting products' + JSON.stringify(error))
next()
}
else {
upsertHappened = true;
inserted += result.nInserted
upserted += result.nUpserted
matched += result.nMatched
modified += result.nModified
removed += result.nRemoved
next()
}
})
}
else {
console.log("Calling finish")
finish()
}
}
function next() {
current++;
bulk = models.Product.collection.initializeUnorderedBulkOp();
setTimeout(load, 0)
}
function finish() {
console.log('Inserted ', inserted + ' Upserted ', upserted, ' Matched ', matched, ' Modified ', modified, ' Removed ', removed)
if (upsertHappened) {
console.log("Calling remove")
remove()
}
}
function remove() {
models.Product.remove(
{
"$or":
[{
"updatedAt": { "$lt": timestamp }
},
{
"discount": { "$eq": 0 }
}]
}, (error, obj) => {
if (error) {
console.log('Error while removing', JSON.stringify(error))
}
else {
if (obj.result.n === 0) {
console.log('Nothing was removed')
} else {
console.log('Removed ' + obj.result.n + ' documents')
}
}
}
)
}
}