hadoop源码解析之hdfs内部结构分析

/***************************************************
 * FSNamesystem does the actual bookkeeping work for the
 * DataNode.
 *
 * It tracks several important tables.
 *
        文件名 -> 数据块(存放在磁盘，也就是FSImage和日志中)
 * 1)  valid fsname --> blocklist  (kept on disk, logged)
        合法的数据块列表(上面关系的逆关系)
 * 2)  Set of all valid blocks (inverted #1)
        数据块 -> DataNode(只保存在内存中，根据DataNode发过来的信息动态建立)
 * 3)  block --> machinelist (kept in memory, rebuilt dynamically from reports)
       DataNode上保存的数据块(上面关系的逆关系)
 * 4)  machine --> blocklist (inverted #2)
      通过LRU算法保存最近发送过心跳信息的DataNode
 * 5)  LRU cache of updated-heartbeat machines
 ***************************************************/

  /** The namespace tree. */
  FSDirectory dir;
  private final BlockManager blockManager;
  

  /** The inode id. */
  final private long id;
  /**
   *  The inode name is in java UTF8 encoding; 
   *  The name in HdfsFileStatus should keep the same encoding as this.
   *  if this encoding is changed, implicitly getFileInfo and listStatus in
   *  clientProtocol are changed; The decoding at the client
   *  side should change accordingly.
   */
  private byte[] name = null;
  /** 
   * Permission encoded using {@link PermissionStatusFormat}.
   * Codes other than {@link #clonePermissionStatus(INodeWithAdditionalFields)}
   * and {@link #updatePermissionStatus(PermissionStatusFormat, long)}
   * should not modify it.
   */
  private long permission = 0L;
  /** The last modification time*/
  private long modificationTime = 0L;
  /** The last access time*/
  private long accessTime = 0L;

private BlockInfoContiguous[] blocks;

private List<INode> children = null;

private final BlockManager blockManager;

private final Namesystem namesystem;

private final DatanodeManager datanodeManager;
  

 /**
   * Mapping: Block -> { BlockCollection, datanodes, self ref }
   * Updated only in response to client-sent information.
 */
final BlocksMap blocksMap;

  /**
   * This array contains triplets of references. For each i-th storage, the
   * block belongs to triplets[3*i] is the reference to the
   * {@link DatanodeStorageInfo} and triplets[3*i+1] and triplets[3*i+2] are
   * references to the previous and the next blocks, respectively, in the list
   * of blocks belonging to this storage.
   * 
   * Using previous and next in Object triplets is done instead of a
   * {@link LinkedList} list to efficiently use memory. With LinkedList the cost
   * per replica is 42 bytes (LinkedList#Entry object per replica) versus 16
   * bytes using the triplets.
   */
  private Object[] triplets;

  /**
   * Construct an entry for blocksmap
   * @param replication the block's replication factor
   */
  public BlockInfoContiguous(short replication) {
    this.triplets = new Object[3*replication];
    this.bc = null;
  }

private GSet<Block, BlockInfoContiguous> blocks;

/**
 * A {@link GSet} is set,
 * which supports the {@link #get(Object)} operation.
 * The {@link #get(Object)} operation uses a key to lookup an element.
 * 
 * Null element is not supported.
 * 
 * @param <K> The type of the keys.
 * @param <E> The type of the elements, which must be a subclass of the keys.
 */
@InterfaceAudience.Private
public interface GSet<K, E extends K> extends Iterable<E>

/**
 
  一个低内存的GSet的实现类，使用数据存储数据，使用链表解决冲突
 * A low memory footprint {@link GSet} implementation,
 * which uses an array for storing the elements
 * and linked lists for collision resolution.
 *
 没有重新哈希操作
 * No rehash will be performed.
 * Therefore, the internal array will never be resized.
 *
 不支持空元素
 * This class does not support null element.
 *
 非线程安全
 * This class is not thread safe.
 *
 * @param <K> Key type for looking up the elements
 * @param <E> Element type, which must be
 *       (1) a subclass of K, and  K的子类
 *       (2) implementing {@link LinkedElement} interface. 实现LinkedElement接口
 */
@InterfaceAudience.Private
public class LightWeightGSet<K, E extends K> implements GSet<K, E> {

  //最大长度，2的30次方
  static final int MAX_ARRAY_LENGTH = 1 << 30; //prevent int overflow problem
  static final int MIN_ARRAY_LENGTH = 1;

  /**
   * An internal array of entries, which are the rows of the hash table.
   * The size must be a power of two.
   */
  private final LinkedElement[] entries;//用来存数据的数组
  /** A mask for computing the array index from the hash value of an element. */
  private final int hash_mask;//hash掩码
  /** The size of the set (not the entry array). */
  private int size = 0;//长度
  /** Modification version for fail-fast.
   * @see ConcurrentModificationException
   */
  private int modification = 0;

  @Override
  public E put(final E element) {
    //validate element
    if (element == null) {
      throw new NullPointerException("Null element is not supported.");
    }
    if (!(element instanceof LinkedElement)) {
      throw new HadoopIllegalArgumentException(
          "!(element instanceof LinkedElement), element.getClass()="
          + element.getClass());
    }
    final LinkedElement e = (LinkedElement)element;

     //查找索引，也就是找到数据要存储的数组的位置
    //find index
    final int index = getIndex(element);

    //remove if it already exists 存在则删除
    final E existing = remove(index, element);

    //insert the element to the head of the linked list
    modification++;
    size++;
    e.setNext(entries[index]);
    entries[index] = e;

    return existing;
  }
  

  private int getIndex(final K key) {
    return key.hashCode() & hash_mask;
  }

e.setNext(entries[index]);
entries[index] = e;

   @Override
  public E get(final K key) {
    //validate key
    if (key == null) {
      throw new NullPointerException("key == null");
    }

    //find element
    final int index = getIndex(key);
    for(LinkedElement e = entries[index]; e != null; e = e.getNext()) {
      if (e.equals(key)) {
        return convert(e);
      }
    }
    //element not found
    return null;
  }

/**
 * A Datanode has one or more storages. A storage in the Datanode is represented
 * by this class.
 */
public class DatanodeStorageInfo{
    ........................
    
  private final DatanodeDescriptor dn;
  private final String storageID;
  private StorageType storageType; //存储类型，disk，ssd等
  private State state;

  private long capacity; 
  private long dfsUsed;
  private volatile long remaining;
  private long blockPoolUsed;

  private volatile BlockInfoContiguous blockList = null;
  private int numBlocks = 0;

  // The ID of the last full block report which updated this storage.
  private long lastBlockReportId = 0;

  /** The number of block reports received */
  private int blockReportCount = 0;
  
  ..............................
    
}

  private String ipAddr;     // IP address
  private String hostName;   // hostname claimed by datanode
  private String peerHostName; // hostname from the actual connection
  private int xferPort;      // data streaming port
  private int infoPort;      // info server port
  private int infoSecurePort; // info server port
  private int ipcPort;       // IPC server port
  private String xferAddr;