Lucene系列(七)索引格式之fdx文件

  FieldsIndexWriter(Directory dir, String name, String suffix, String extension,
                    String codecName, byte[] id, int blockShift, IOContext ioContext) throws IOException {
    this.dir = dir;
    this.name = name;
    this.suffix = suffix;
    this.extension = extension;
    this.codecName = codecName;
    this.id = id;
    this.blockShift = blockShift;
    this.ioContext = ioContext;
    // docNum 的 tmp 文件
    this.docsOut = dir.createTempOutput(name, codecName + "-doc_ids", ioContext);
    boolean success = false;
    try {
      CodecUtil.writeHeader(docsOut, codecName + "Docs", VERSION_CURRENT);

      // StartPoint 的 tmp 文件
      filePointersOut = dir.createTempOutput(name, codecName + "file_pointers", ioContext);
      CodecUtil.writeHeader(filePointersOut, codecName + "FilePointers", VERSION_CURRENT);
      success = true;
    } finally {
      if (success == false) {
        close();
      }
    }
  }

  void writeIndex(int numDocs, long startPointer) throws IOException {
    assert startPointer >= previousFP;
    // doc num
    docsOut.writeVInt(numDocs);
    // filepoint
    filePointersOut.writeVLong(startPointer - previousFP);
    previousFP = startPointer;
    totalDocs += numDocs;
    totalChunks++;
  }

  /**
   * 在这里生成的 fdx 文件，从两个 tmp 文件里面找到每个 chunk 的 doc 数量，fdt 文件中存储的字节数，
   * 这两个内容，写到 meta 文件和 fdx 文件中，配合起来存储的
   * <p>
   * 这个类本身就是为了 fdx 文件搞的，就是为了写 fdt 的索引，写得少很正常
   */
  void finish(int numDocs, long maxPointer, IndexOutput metaOut) throws IOException {
    if (numDocs != totalDocs) {
      throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs);
    }
    CodecUtil.writeFooter(docsOut);
    CodecUtil.writeFooter(filePointersOut);
    IOUtils.close(docsOut, filePointersOut);

    // dataOut　是 fdx 文件，是用来对 fdt 文件做索引的文件，所以 fdt 文件写入内容，我这里记录每个 chunk 的 doc 数量，占用字节数即可
    // 所以这里只能调用一次么，无论是多少个多大的 field，都只能调用一次这里么
    // 写 fdx 文件
    try (IndexOutput dataOut = dir.createOutput(IndexFileNames.segmentFileName(name, suffix, extension), ioContext)) {
      // 这个 header，48 个字节。
      CodecUtil.writeIndexHeader(dataOut, codecName + "Idx", VERSION_CURRENT, id, suffix);

      metaOut.writeInt(numDocs);
      metaOut.writeInt(blockShift);
      metaOut.writeInt(totalChunks + 1);
      // 这个 filePointer, 此时只写了一个 header 的长度，48
      long filePointer = dataOut.getFilePointer();
      metaOut.writeLong(filePointer);

      try (ChecksumIndexInput docsIn = dir.openChecksumInput(docsOut.getName(), IOContext.READONCE)) {
        CodecUtil.checkHeader(docsIn, codecName + "Docs", VERSION_CURRENT, VERSION_CURRENT);
        Throwable priorE = null;
        try {
          // 这里做的配合是，　meta 里面存了 min/斜率等，真实的数组偏移量在 dataOut 里面存储
          final DirectMonotonicWriter docs = DirectMonotonicWriter.getInstance(metaOut, dataOut, totalChunks + 1, blockShift);
          long doc = 0;
          docs.add(doc);
          // 注意，这里是每一 chunk, 而不是 per document
          for (int i = 0; i < totalChunks; ++i) {
            // 每个 chunk 的 doc 数量
            doc += docsIn.readVInt();
            docs.add(doc);
          }
          docs.finish();
          if (doc != totalDocs) {
            throw new CorruptIndexException("Docs don't add up", docsIn);
          }
        } catch (Throwable e) {
          priorE = e;
        } finally {
          CodecUtil.checkFooter(docsIn, priorE);
        }
      }
      dir.deleteFile(docsOut.getName());
      docsOut = null;

      long filePointer1 = dataOut.getFilePointer();
      metaOut.writeLong(filePointer1);
      try (ChecksumIndexInput filePointersIn = dir.openChecksumInput(filePointersOut.getName(), IOContext.READONCE)) {
        CodecUtil.checkHeader(filePointersIn, codecName + "FilePointers", VERSION_CURRENT, VERSION_CURRENT);
        Throwable priorE = null;
        try {
          // 其实由于我测试的时候只有一两个 doc，肯定在一个 chunk, 所以 dataOut 里面都没写入啥东西
          final DirectMonotonicWriter filePointers = DirectMonotonicWriter.getInstance(metaOut, dataOut, totalChunks + 1, blockShift);
          long fp = 0;
          // 这里存储的是每一个 chunk 的实际数据的字节长度
          for (int i = 0; i < totalChunks; ++i) {
            fp += filePointersIn.readVLong();
            filePointers.add(fp);
          }
          if (maxPointer < fp) {
            throw new CorruptIndexException("File pointers don't add up", filePointersIn);
          }
          filePointers.add(maxPointer);
          filePointers.finish();
        } catch (Throwable e) {
          priorE = e;
        } finally {
          CodecUtil.checkFooter(filePointersIn, priorE);
        }
      }
      dir.deleteFile(filePointersOut.getName());
      filePointersOut = null;

      // meta 里面再搞个索引
      long filePointer2 = dataOut.getFilePointer();
      metaOut.writeLong(filePointer2);
      metaOut.writeLong(maxPointer);

      CodecUtil.writeFooter(dataOut);
    }
  }