技术总结|Bitcask存储

crc32(4byte)|timeStamp(4byte)|ksz(4byte)|valueSz(4byte)|key|value

timeStamp(4byte)|ksz(4byte)|valuesz(4byte)|valuePos(8byte)|key

import os
import shutil
import cPickle as pickle

class Bitcask(Object):
    def __init__(self, threshold = 10000, path = './'):
        self.table_ = {}
        self.threshold_ = threshold
        self.path_ = path
        self.curr_active_ = # 最新使用的文件

        if os.path.exists(path) == False:
            # 创建文件夹

    def get(self, key):
        record = self.__get_record(key)
        if record != None:
            return record[5] # 第六个值是value

        return None

    def put(self, key, value):
        key_size = len(key)
        value_size = 0 if value is None else len(value)
        ts = datetime.now()
        crc = crc32('{0}{1}{2}{3}{4}'.format(ts, key_size, value_size, key, size))
        info = self.__io_write(crc, ts, key_size, value_size, key, value)
        self.table_[key] = info
        return True

    def delete(self, key):
        ''' 删除数据，将对应的key添加为一条None '''
        return self.put(key, None)

    def update(self, key, value):
        ''' 更新数据，将对应的key添加为一条value '''
        return self.put(key, value)

    def load_data(self, path):
        ''' 加载目录下的数据 '''
        if path is not None:
            self.path_ = path
            if os.path.exists(path) == False:
                # 创建文件夹

        for filename in #所有的文件
            self.load_file(filename)

    def load_file(self, filename):
        f = open(filename)
        data = f.read()
        list_items = data.split("t.")

        start = 0
        length = 0
        for item in list_items[:-1]:
            s = item + "t."
            record = pickle.load(s) # 加载文件
            length = len(item) + 2
            if self.table_.has_key(record[4]) is True:
                ts1 = record[1] # 获取时间戳
                ts2 = self.__get_record(record[4])[1]
                if ts1 > ts2: # 对比时间戳
                    self.table_[record[4]] = (filename, start, length)
            else:
                self.table_[record[4]] = (filename, start, length)

            start += length
        f.close()

    def __get_record(self, key):
        ''' 根据key读取完整的记录 '''
        info = self.table_[key] # 先获取对应的记录信息
        if not info:
            return None;

        pickled_data = self.__io_read(*info)
        data = pickle.loads(pickled_data)

    def __io_read(self, filename, start, length):
        with open(filename, 'rb') as f:
            f.seek(start)
            return f.read(length)

    def __io_write(self, crc, ts, key_size, value_size, key, value):
        active_filename = '%s%s.sst'%(self.path_, self.curr_active_)
        try:
            if os.path.getsize(active_filename) >= self.threshold_:
                self.curr_active_ += 1
                active_filename = '%s%s.sst'%(self.path_, self.curr_active_)
        except:
            pass
        # 保存数据到文件
        with open(active_filename, 'a') as f:
            data = pickle.dumps(crc, ts, key_size, value_size, key, value)
            start = f.tell()
            length = len(data)
            f.write(data)
            return active_filename, start, length

    def merge(self, dst_path):
        ''' 
          合并多个文件，基本操作就是读取self.table_中所有的key，
          然后将key的数据写入到对应的合并文件中，代码可以自行添加 
        '''
        pass

    def load_file_with_hint(self, hint_filename):
        ''' 
          使用hint文件加快key的load，将ts,key_size,value_size,value_pos,key
          这些数据写入到磁盘文件，然后通过load_file_with_hint加载出来，代码可以自行添加 
        '''
        pass

if __name__ == '__main__':
    s = Bitcask()
    # 加载数据，然后进行一系列操作
    ...