问用于在OSX上数千个文件目录中搜索二进制文件匹配的高性能搜索工具。

EN

#given a folder path, makes a hash index of every file, recursively
import sys, os, hashlib, io

hash_md5 = hashlib.md5()

#some files need to be hashed incrementally as they may be too big to fit in memory
#http://stackoverflow.com/a/40961519/2518451
def md5sum(src, length=io.DEFAULT_BUFFER_SIZE):
    md5 = hashlib.md5()
    with io.open(src, mode="rb") as fd:
        for chunk in iter(lambda: fd.read(length), b''):
            md5.update(chunk)
    return md5

#this project done on macOS. There may be other files that are appropriate to hide on other platforms.
ignore_files = [".DS_Store"]

def index(source, index_output):

    index_output_f = open(index_output, "wt")
    index_count = 0

    for root, dirs, filenames in os.walk(source):

        for f in filenames:
            if f in ignore_files:
                continue

            #print f
            fullpath = os.path.join(root, f)
            #print fullpath

            md5 = md5sum(fullpath)
            md5string = md5.hexdigest()
            line = md5string + ":" + fullpath
            index_output_f.write(line + "\n")
            print line
            index_count += 1

    index_output_f.close()
    print("Index Count: " + str(index_count))


if __name__ == "__main__":
    index_output = "index_output.txt"

    if len(sys.argv) < 2:
        print("Usage: md5index [path]")
    else:
        index_path = sys.argv[1]
        print("Indexing... " + index_path)
        index(index_path, index_output)

#given an index_output.txt in the same directory and an input path,
#remove all files that already have a hash in index_output.txt

import sys, os
from md5index import md5sum
from send2trash import send2trash
SENDING_TO_TRASH = True

def load_index():
    index_output = "index_output.txt"
    index = []
    with open(index_output, "rt") as index_output_f:
        for line in index_output_f:
            line_split = line.split(':')
            md5 = line_split[0]
            index.append(md5)
    return index

#traverse file, compare against index
def traverse_merge_path(merge_path, index):
    found = 0
    not_found = 0

    for root, dirs, filenames in os.walk(merge_path):
        for f in filenames:
            #print f
            fullpath = os.path.join(root, f)
            #print fullpath

            md5 = md5sum(fullpath)
            md5string = md5.hexdigest()

            if md5string in index:
                if SENDING_TO_TRASH:
                    send2trash(fullpath)

                found += 1
            else:
                print "\t NON-DUPLICATE ORIGINAL: " + fullpath
                not_found += 1


    print "Found Duplicates: " + str(found) + " Originals: " + str(not_found)


if __name__ == "__main__":
    index = load_index()
    print "Loaded index with item count: " + str(len(index))

    print "SENDING_TO_TRASH: " + str(SENDING_TO_TRASH) 

    merge_path = sys.argv[1]
    print "Merging To: " + merge_path

    traverse_merge_path(merge_path, index)

python uniquemerge.py folderB
# deletes all files in folderB that already existed in folderA
# I can now manually merge folderB into folderA