聊聊lustre中的fid和fld

// mds获取到ost上的fid信息的结构
struct ost_id {
	union {
		struct {
			__u64	oi_id;
			__u64	oi_seq;
		} oi;
		struct lu_fid oi_fid;
	};
} __attribute__((packed));

#define lov_ost_data lov_ost_data_v1

// 定义mds上文件stripe的数据分片的信息
struct lov_ost_data_v1 {          /* per-stripe data structure (little-endian)*/
	struct ost_id l_ost_oi;	  /* OST object ID */
	__u32 l_ost_gen;          /* generation of this l_ost_idx */
	__u32 l_ost_idx;          /* OST index in LOV (lov_tgt_desc->tgts) */
};

// mdt 上的文件元数据对应的fid,fid对应的inode存储在inode上的扩展属性的trusted.link的值
struct lov_mds_md_v3 {            /* LOV EA mds/wire data (little-endian) */
	__u32 lmm_magic;          /* magic number = LOV_MAGIC_V3 */
	__u32 lmm_pattern;        /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */
	struct ost_id	lmm_oi;	  /* LOV object ID */
	__u32 lmm_stripe_size;    /* size of stripe in bytes */
	/* lmm_stripe_count used to be __u32 */
	__u16 lmm_stripe_count;   /* num stripes in use for this object */
	__u16 lmm_layout_gen;     /* layout generation number */
	char  lmm_pool_name[LOV_MAXPOOLNAME + 1]; /* must be 32bit aligned */
	struct lov_ost_data_v1 lmm_objects[0]; /* per-stripe data */
};

struct lu_fid {
	// 申请到的序列号
	__u64 f_seq;
	// 对象的ID
	__u32 f_oid;
	// 版本号
	__u32 f_ver;
} __attribute__((packed));

// 从fid中获取客户端查看的inode
static inline __u64 fid_flatten(const struct lu_fid *fid)
{
	__u64 ino;
	__u64 seq;

	if (fid_is_igif(fid)) {
		ino = lu_igif_ino(fid);
		return ino;
	}

	seq = fid_seq(fid);

	ino = (seq << 24) + ((seq >> 24) & 0xffffff0000ULL) + fid_oid(fid);

	return ino ?: fid_oid(fid);
}

/* seq client type */
// sequence 服务的客户端类型
enum lu_cli_type {
	// 负责申请mdt上的Sequence 客户端类型
	LUSTRE_SEQ_METADATA = 1,
	// 负责处理ost上数据对象的 sequence客户端类型
	LUSTRE_SEQ_DATA
};

// sequence 服务端类型
enum lu_mgr_type {
		// 
        LUSTRE_SEQ_SERVER,
        LUSTRE_SEQ_CONTROLLER
};

// 运行于mdt上的sequence 服务
static int mdt_seq_init(const struct lu_env *env, struct mdt_device *mdt)
{
	/* init sequence controller server(MDT0) */
	if (ss->ss_node_id == 0) {
		// 运行mdt0上的sequence controller，该controller负责分配seq的范围给sequence server
		rc = seq_server_init(env, ss->ss_control_seq, mdt->mdt_bottom,
				     mdt_obd_name(mdt), LUSTRE_SEQ_CONTROLLER,
				     ss);
		if (rc)
			GOTO(out_seq_fini, rc);
	}
	// 运行其他mdt(包括mdt0)上的sequence server服务，负责给fid客户端分配sequnce来构建fid
	rc = seq_server_init(env, ss->ss_server_seq, mdt->mdt_bottom,
			     mdt_obd_name(mdt), LUSTRE_SEQ_SERVER, ss);
	if (rc)
		GOTO(out_seq_fini, rc);

	// 初始化连接sequence controller的客户端，主要是和mdt0通信，获取sequence range
	rc = mdt_seq_init_cli(env, mdt);
	EXIT;
out_seq_fini:
	if (rc)
		mdt_seq_fini(env, mdt);

	return rc;
}

// 运行于ost上的sequnce服务
int ofd_fid_init(const struct lu_env *env, struct ofd_device *ofd)
{
	// OST上启动一个的Seqence server服务
	rc = seq_server_init(env, ss->ss_server_seq, ofd->ofd_osd, obd_name,
			     LUSTRE_SEQ_SERVER, ss);
	if (rc) {
		CERROR("%s: seq server init error: rc = %d\n", obd_name, rc);
		GOTO(out_server, rc);
	}

	// 客户端的初始化
	seq_client_init(ss->ss_client_seq, NULL, LUSTRE_SEQ_DATA,
			name, NULL);
	rc = seq_server_set_cli(env, ss->ss_server_seq, ss->ss_client_seq);
out_name:
	OBD_FREE(name, len);

	return rc;
}

// sequence rpc请求的op type
enum seq_rpc_opc {
        SEQ_QUERY                       = 700,
        SEQ_LAST_OPC,
        SEQ_FIRST_OPC                   = SEQ_QUERY
};

// sequence服务的类型类型
enum seq_op {
        SEQ_ALLOC_SUPER = 0,
        SEQ_ALLOC_META = 1
};


// 请求的查询sequence rpc 的op_type
struct req_msg_field RMF_SEQ_OPC =
        DEFINE_MSGF("seq_query_opc", 0,
                    sizeof(__u32), lustre_swab_generic_32s, NULL);


// 请求sequence range范围
struct req_msg_field RMF_SEQ_RANGE =
        DEFINE_MSGF("seq_query_range", 0,
                    sizeof(struct lu_seq_range),
                    lustre_swab_lu_seq_range, NULL);
EXPORT_SYMBOL(RMF_SEQ_RANGE);

// seq_handler 是sequence处理函数；sequence的处理函数，对于lustre客户端，mdt是自身申请文件数据的fid;于此mdt需要作为fid的客户端rpc连接到其他的ost的sequence server申请sequence,返回fid给mdt
static int seq_handler(struct tgt_session_info *tsi)
{
	// 获取fid客户端网络 rpc请求
	opc = req_capsule_client_get(tsi->tsi_pill, &RMF_SEQ_OPC);
	if (opc) {
		out = req_capsule_server_get(tsi->tsi_pill, &RMF_SEQ_RANGE);
		if (!out)
			RETURN(err_serious(-EPROTO));
		// 获取fid客户端网络 rpc请求
		tmp = req_capsule_client_get(tsi->tsi_pill, &RMF_SEQ_RANGE);

		out->lsr_index = tmp->lsr_index;
		out->lsr_flags = tmp->lsr_flags;
		// 在mdt0上的sequence controller或者非mdt0的和ost上的Sequence server处理sequence请求
		rc = seq_server_handle(site, tsi->tsi_env, *opc, out);
	} else {
		rc = err_serious(-EPROTO);
	}

	RETURN(rc);
}
// 运行在mdt和ost上申请sequence的函数，根据请求类型来决策是申请controller上 sequence范围还server上的sequence 
static int seq_server_handle(struct lu_site *site,
			     const struct lu_env *env,
			     __u32 opc, struct lu_seq_range *out)
{
	int rc;
	struct seq_server_site *ss_site;
	struct dt_device *dev;
	ENTRY;

	ss_site = lu_site2seq(site);

	switch (opc) {

	// 在sequence server上申请sequence 范围
	case SEQ_ALLOC_META:
		rc = seq_server_alloc_meta(ss_site->ss_server_seq, out, env);
		break;

	// 在controller上申请一个sequence 范围
	case SEQ_ALLOC_SUPER:
		rc = seq_server_alloc_super(ss_site->ss_control_seq, out, env);
		break;
	default:
		rc = -EINVAL;
		break;
	}
	RETURN(rc);
}

enum fld_op {
	FLD_CREATE = 0,
	FLD_DELETE = 1,
	FLD_LOOKUP = 2,
};


static int seq_server_handle(struct lu_site *site,
			     const struct lu_env *env,
			     __u32 opc, struct lu_seq_range *out)
{

	switch (opc) {
	// 在sequence server上申请sequence 范围
	case SEQ_ALLOC_META:
		rc = seq_server_alloc_meta(ss_site->ss_server_seq, out, env);
		break;

	// 在controller上申请一个sequence 范围
	case SEQ_ALLOC_SUPER:
		rc = seq_server_alloc_super(ss_site->ss_control_seq, out, env);
		break;
	default:
		rc = -EINVAL;
		break;
	}
}

// sequence server申请Sequence
int seq_server_alloc_meta(struct lu_server_seq *seq,
			  struct lu_seq_range *out,
			  const struct lu_env *env)
{
	int rc;
	// 实际的处理函数
	rc = __seq_server_alloc_meta(seq, out, env);
	RETURN(rc);
}

// sequence server申请Sequence范围
static int __seq_server_alloc_meta(struct lu_server_seq *seq,
				   struct lu_seq_range *out,
				   const struct lu_env *env)
{
	struct lu_seq_range *space = &seq->lss_space;
	int rc = 0;
	// 
	rc = seq_server_check_and_alloc_super(env, seq){
		// 向mdt0申请新的可用的sequence range
		rc = seq_client_alloc_super(seq->lss_cli, env);
		// 插入到本地的sequence server的fld
		rc = fld_insert_entry(env, fld, space);
	}

	// 设置sequence server中的sequence range,并且更新seqence server的fld
	rc = range_alloc_set(env, out, seq);

	RETURN(rc);
}


// mdt0上sequnece请求处理
int seq_server_alloc_super(struct lu_server_seq *seq,
			   struct lu_seq_range *out,
			   const struct lu_env *env)
{

	rc = __seq_server_alloc_super(seq, out, env);

	RETURN(rc);
}



// mdt0上同时运行sequence controller和sequence server,这时候只需要在本地申请sequence range即可，不需要网络请求
static int __seq_server_alloc_super(struct lu_server_seq *seq,
				    struct lu_seq_range *out,
				    const struct lu_env *env)
{
	struct lu_seq_range *space = &seq->lss_space;
	// 本mdt0内申请新的sequence 范围
	range_alloc(out, space, seq->lss_width);
	// 内存对应Sequence的范围，同时更新本地的fld
	rc = seq_store_update(env, seq, out, 1 /* sync */)
	{
		rc = fld_server_create(env, seq->lss_site->ss_server_fld, out,th)
		{
			rc = fld_index_create(env, fld, range, th);
		}
	}

	RETURN(rc);
}

// lustre sequence定义了2种类型的操作，一个是查询，一个是读取Sequence
struct tgt_handler fld_handlers[] = {
	TGT_FLD_HDL_VAR(0,	FLD_QUERY,	fld_handle_query),
	TGT_FLD_HDL_VAR(0,	FLD_READ,	fld_handle_read),
};

// sequence查询的请求处理函数
static int fld_handle_query(struct tgt_session_info *tsi)
{
	int	rc;

	ENTRY;

	req_capsule_set(tsi->tsi_pill, &RQF_FLD_QUERY);
	// 处理Sequence的查找工作
	rc = fld_handle_lookup(tsi)
	{
		rc = fld_server_lookup(tsi->tsi_env, fld, in->lsr_start, out)
		{
			// 优先查找fld cache是否存在
			rc = fld_local_lookup(env, fld, seq, range);

			// 如果不存在，则请求mdt0申请一个sequence range
			rc = fld_client_rpc(fld->lsf_control_exp,
				    range, FLD_QUERY, NULL);
			if (rc == 0)
				// 插入到本地fld Cache
				fld_cache_insert(fld->lsf_cache, range);
			}
	}

	RETURN(rc);
}


// sequence读取的请求处理函数
static int fld_handle_read(struct tgt_session_info *tsi)
{
	struct obd_export *exp = tsi->tsi_exp;
	struct lu_site *site = exp->exp_obd->obd_lu_dev->ld_site;
	struct lu_seq_range *in;
	void *data;
	int rc;

	ENTRY;

	// 初始化请求包
	req_capsule_set(tsi->tsi_pill, &RQF_FLD_READ);

	// 获取来自`fld client`的请求
	in = req_capsule_client_get(tsi->tsi_pill, &RMF_FLD_MDFLD);
	if (!in)
		RETURN(err_serious(-EPROTO));

	req_capsule_set_size(tsi->tsi_pill, &RMF_GENERIC_DATA, RCL_SERVER,
			     PAGE_SIZE);
	// 服务端需要解包
	rc = req_capsule_server_pack(tsi->tsi_pill);
	if (unlikely(rc != 0))
		RETURN(err_serious(rc));

	data = req_capsule_server_get(tsi->tsi_pill, &RMF_GENERIC_DATA);

	// 本地从本地fld数据库读取seqeunce
	rc = fld_server_read(tsi->tsi_env, lu_site2seq(site)->ss_server_fld,
			     in, data, PAGE_SIZE);
	RETURN(rc);
}