从数百万/数十亿条记录中删除MongoDB 4中的重复项

db.collection.aggregate([
  { $group: { _id: "$field", count: { $sum: 1 } } },
  { $match: { count: { $gt: 1 } } }
])

db.collection.aggregate([
  { $group: { _id: "$field", dups: { $addToSet: "$_id" }, count: { $sum: 1 } } },
  { $match: { count: { $gt: 1 } } }
]).forEach(function(doc) {
  doc.dups.shift(); // 保留第一个重复项
  db.collection.remove({ _id: { $in: doc.dups } });
});

function map() {
  emit(this.field, this._id);
}

function reduce(key, values) {
  var ids = [];
  values.forEach(function(id) {
    if (ids.indexOf(id) < 0) {
      ids.push(id);
    }
  });
  return ids;
}

var mapReduceResult = db.collection.mapReduce(map, reduce, { out: { inline: 1 } });
mapReduceResult.results.forEach(function(doc) {
  if (doc.value.length > 1) {
    db.collection.remove({ _id: { $nin: doc.value } });
  }
});