深入浅出Lustre扩展属性

// 目前使用的是lustre 2.15.0[root@CentOS-Lustre-Client ~]$ lfs --version
lfs 2.15.0

// 人为设置/bigfs/pg设置为2个分片，每个分片大小是1M[root@CentOS-Lustre-Client ~]$ lfs setstripe  -c 2  -S 1048576  /bigfs/pg/[root@CentOS-Lustre-Client ~]$ lfs getstripe  -v /bigfs/pg/
/bigfs/pg/
lmm_fid:           [0x200000bd1:0x1:0x0]stripe_count:  2 stripe_size:   1048576 pattern:       raid0 stripe_offset: -1

// 拷贝2M测试数据到/bigfs/pg目录中[root@CentOS-Lustre-Client ~]$ cp pg.log  /bigfs/pg/

[root@CentOS-Lustre-Client ~]$ getfattr -d -m - /bigfs/pg/pg.log
getfattr: Removing leading '/' from absolute path names# file: bigfs/pg/pg.log// 这里存储的是这个文件的getstripe信息
lustre.lov=0s0AvRCwEAAAACAAAAAAAAANELAAACAAAAAAAQAAIAAAAiAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAiAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAA=// trusted.link存储文件父目录的fid和自身文件名称
trusted.link=0s3/HqEQEAAAAwAAAAAAAAAAAAAAAAAAAAABgAAAACAAAL0QAAAAEAAAAAcGcubG9n

// trusted.lma 存储这个文件的fid和状态标记
trusted.lma=0sAAAAAAAAAADRCwAAAgAAAAIAAAAAAAAA

// trusted.lov 属性存储这个文件的layout，文件粗出在那些ost上
trusted.lov=0s0AvRCwEAAAACAAAAAAAAANELAAACAAAAAAAQAAIAAAAiAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAiAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAA=trusted.som=0sBAAAAAAAAAAAACAAAAAAAAAAAAAAAAAA

// trusted.lmv 是多MDS,子目录的的布局

// lustre源码中定义了lustre.lov的扩展属性#define XATTR_LUSTRE_PREFIX	"lustre."#define XATTR_LUSTRE_LOV	XATTR_LUSTRE_PREFIX"lov"

// 从解析结果来看，存储的是文件的stripe信息，和lfs getstripe得到的信息类似[root@CentOS-Lustre-Client ~/desc_extend_attribute]$ ./a.out  /bigfs/pg/pg.log 
******************lustre.lov on /bigfs/pg/pg.log**************
lmm_magic:0xBD10BD0
lmm_pattern:1
lmm_stripe_size:1048576
lmm_stripe_count:2
lmm_layout_gen:0    ost_idx=1,oi_id=34,oi_seq=0,fid.f_seq=34,fid.f_oid=0,fid.f_ver=0
    ost_idx=2,oi_id=34,oi_seq=0,fid.f_seq=34,fid.f_oid=0,fid.f_ver=0// 执行lustre的lfs getstripe命令可以看到输出内容和 上面的一致[root@CentOS-Lustre-Client ~]$ lfs getstripe  -v -L /bigfs/pg/pg.log 
/bigfs/pg/pg.log
lmm_magic:         0x0BD10BD0
lmm_seq:           0x200000bd1
lmm_object_id:     0x2
lmm_fid:           [0x200000bd1:0x2:0x0]lmm_stripe_count:  2lmm_stripe_size:   1048576lmm_pattern:       raid0
lmm_layout_gen:    0lmm_stripe_offset: 1
        obdidx           objid           objid           group             1              34           0x22                0
             2              34           0x22                0

// lustre源码中定义了这个属性#define XATTR_NAME_LINK         "trusted.link"

// link_ea属性中有个link_ea_header 和link_ea_entry两个数据结构[root@CentOS-Lustre-Client ~/desc_extend_attribute]$ ./a.out  /bigfs/pg/pg.log
******************trusted.link on /bigfs/pg/pg.log**************
// lustre内核中定义的magic数
link_ea_header leh_magic:0x11EAF1DF
link_ea_header leh_reccount:1
link_ea_header leh_len:48
// 在客户端执行ls -i /bigfs/pg/ 看到的inode就是下面这个
link_ea_entry parent inode=144115238826934273link_ea_entry pfid.f_seq=8589937617,pfid.f_oid=1,pfid.f_ver=0// 存储了该文件的名称名称
link_ea_entry filename=pg.log,filename_len=24// 查看pg.log文件的父目录的inode，和上面解析的保持一致[root@CentOS-Lustre-Client /bigfs]$ ls -l -i
total 4144115238826934273 drwxr-xr-x 2 root root 4096 Aug 25 23:19 pg

// lustre内核中定义的trusted.lma#define XATTR_NAME_LMA          "trusted.lma"

// 自解析lustre的属性，然后读取出来[root@CentOS-Lustre-Client ~/desc_extend_attribute]$ ./a.out  /bigfs/pg/pg.log 
******************trusted.lma on /bigfs/pg/pg.log**************
lustre_mdt_attrs lma_compat:0
lustre_mdt_attrs lma_incompat:0
// 解析到这个文件的inode=144115238826934274lustre_mdt_attrs self inode=144115238826934274// 详细解析fid的信息
lustre_mdt_attrs fid.f_seq=8589937617,fid.f_oid=2,fid.f_ver=0// 在lustre客户端查看目标文件的inode，和解析出来的保持一致[root@CentOS-Lustre-Client ~/desc_extend_attribute]$ ls -l -i /bigfs/pg/pg.log 
144115238826934274 -rw-r--r-- 1 root root 2097152 Aug 25 23:19 /bigfs/pg/pg.log

// lustre内核中定义的trusted.lma#define XATTR_NAME_LOV          "trusted.lov"

// lustre内核中定义的trusted.lma#define XATTR_NAME_LOV          "trusted.lov"

// 记录了这个文件在mdt上的fid文件每个分片所在ost上的位置，和fid
******************trusted.lov on /bigfs/pg/pg.log**************
lov_mds_md_v1 lmm_magic:0xBD10BD0
lov_mds_md_v1 lmm_pattern:1
lov_mds_md_v1 lmm_stripe_size:1048576
lov_mds_md_v1 lmm_stripe_count:2
lov_mds_md_v1 lmm_layout_gen:0
// 目标文件的fid
lov_mds_md_v1 mdt: oi_id=2,oi_seq=8589937617,fid.f_seq=2,fid.f_oid=3025,fid.f_ver=2// 目标文件分片的fid、所在的ost
lov_mds_md_v1 ost: 
   on ost: ost_index=1,oi_id=34,oi_seq=0,fid.f_seq=2,fid.f_oid=3025,fid.f_ver=2
   on ost: ost_index=2,oi_id=34,oi_seq=0,fid.f_seq=2,fid.f_oid=3025,fid.f_ver=2

#include <stdint.h>#include <stdlib.h>#include <stdio.h>#include <assert.h>#include <fcntl.h>#include <unistd.h>#include <string.h>#include <stdbool.h>#include <sys/types.h>#include <sys/stat.h>#include <sys/xattr.h>#define XATTR_SIZE_MAX 65536#define XATTR_LUSTRE_LOV "lustre.lov"#define XATTR_NAME_LINK "trusted.link"#define XATTR_NAME_LMA "trusted.lma"#define XATTR_NAME_LOV "trusted.lov"/**
 * File IDentifier.
 *
 * FID is a cluster-wide unique identifier of a file or an object (stripe).
 * FIDs are never reused.
 **/struct lu_fid{
    /**
     * FID sequence. Sequence is a unit of migration: all files (objects)
     * with FIDs from a given sequence are stored on the same server.
     * Lustre should support 2^64 objects, so even if each sequence
     * has only a single object we can still enumerate 2^64 objects.
     **/
    uint64_t f_seq;
    /* FID number within sequence. */
    uint32_t f_oid;
    /**
     * FID version, used to distinguish different versions (in the sense
     * of snapshots, etc.) of the same file system object. Not currently
     * used.
     **/
    uint32_t f_ver;} __attribute__((packed));/**
 * OST object IDentifier.
 */struct ost_id{
    union
    {
        struct
        {
            uint64_t oi_id;
            uint64_t oi_seq;
        } oi;
        struct lu_fid oi_fid;
    };} __attribute__((packed));#define lov_user_ost_data lov_user_ost_data_v1struct lov_user_ost_data_v1{                           /* per-stripe data structure */
    struct ost_id l_ost_oi; /* OST object ID */
    uint32_t l_ost_gen;     /* generation of this OST index */
    uint32_t l_ost_idx;     /* OST index in LOV */} __attribute__((packed));#define lov_user_md lov_user_md_v1struct lov_user_md_v1{                              /* LOV EA user data (host-endian) */
    uint32_t lmm_magic;        /* magic number = LOV_USER_MAGIC_V1 */
    uint32_t lmm_pattern;      /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */
    struct ost_id lmm_oi;      /* MDT parent inode id/seq (id/0 for 1.x) */
    uint32_t lmm_stripe_size;  /* size of stripe in bytes */
    uint16_t lmm_stripe_count; /* num stripes in use for this object */
    union
    {
        uint16_t lmm_stripe_offset; /* starting stripe offset in
                                     * lmm_objects, use when writing */
        uint16_t lmm_layout_gen;    /* layout generation number
                                     * used when reading */
    };
    struct lov_user_ost_data_v1 lmm_objects[0]; /* per-stripe data */} __attribute__((packed, __may_alias__));/***********************trusted.link****************/struct link_ea_header{
    uint32_t leh_magic;
    uint32_t leh_reccount;
    uint64_t leh_len; /* total size */
    uint32_t leh_overflow_time;
    uint32_t leh_padding;};/** Hardlink data is name and parent fid.
 * Stored in this crazy struct for maximum packing and endian-neutrality
 */struct link_ea_entry{
    /** uint16_t stored big-endian, unaligned */
    unsigned char lee_reclen[2];
    unsigned char lee_parent_fid[sizeof(struct lu_fid)];
    char lee_name[0];} __attribute__((packed));/**********************trusted.lma****************//**
 * Following struct for object attributes, that will be kept inode's EA.
 * Introduced in 2.0 release (please see b15993, for details)
 * Added to all objects since Lustre 2.4 as contains self FID
 */struct lustre_mdt_attrs{
    /**
     * Bitfield for supported data in this structure. From enum lma_compat.
     * lma_self_fid and lma_flags are always available.
     */
    uint32_t lma_compat;
    /**
     * Per-file incompat feature list. Lustre version should support all
     * flags set in this field. The supported feature mask is available in
     * LMA_INCOMPAT_SUPP.
     */
    uint32_t lma_incompat;
    /** FID of this inode */
    struct lu_fid lma_self_fid;};/**********************trusted.lov****************/#define lov_ost_data lov_ost_data_v1struct lov_ost_data_v1{                           /* per-stripe data structure (little-endian)*/
    struct ost_id l_ost_oi; /* OST object ID */
    uint32_t l_ost_gen;     /* generation of this l_ost_idx */
    uint32_t l_ost_idx;     /* OST index in LOV (lov_tgt_desc->tgts) */};#define lov_mds_md lov_mds_md_v1struct lov_mds_md_v1{                             /* LOV EA mds/wire data (little-endian) */
    uint32_t lmm_magic;       /* magic number = LOV_MAGIC_V1 */
    uint32_t lmm_pattern;     /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */
    struct ost_id lmm_oi;     /* LOV object ID */
    uint32_t lmm_stripe_size; /* size of stripe in bytes */
    /* lmm_stripe_count used to be uint32_t */
    uint16_t lmm_stripe_count;             /* num stripes in use for this object */
    uint16_t lmm_layout_gen;               /* layout generation number */
    struct lov_ost_data_v1 lmm_objects[0]; /* per-stripe data */};enum fid_seq{
    FID_SEQ_OST_MDT0 = 0,
    FID_SEQ_LLOG = 1, /* unnamed llogs */
    FID_SEQ_ECHO = 2,
    FID_SEQ_UNUSED_START = 3, /* Unused */
    FID_SEQ_UNUSED_END = 9,   /* Unused */
    FID_SEQ_LLOG_NAME = 10,   /* named llogs */
    FID_SEQ_RSVD = 11,
    FID_SEQ_IGIF = 12,
    FID_SEQ_IGIF_MAX = 0x0ffffffffULL,
    FID_SEQ_IDIF = 0x100000000ULL,
    FID_SEQ_IDIF_MAX = 0x1ffffffffULL,
    /* Normal FID sequence starts from this value, i.e. 1<<33 */
    FID_SEQ_START = 0x200000000ULL,
    /* sequence for local pre-defined FIDs listed in local_oid */
    FID_SEQ_LOCAL_FILE = 0x200000001ULL,
    FID_SEQ_DOT_LUSTRE = 0x200000002ULL,
    /* sequence is used for local named objects FIDs generated
     * by local_object_storage library */
    FID_SEQ_LOCAL_NAME = 0x200000003ULL,
    /* Because current FLD will only cache the fid sequence, instead
     * of oid on the client side, if the FID needs to be exposed to
     * clients sides, it needs to make sure all of fids under one
     * sequence will be located in one MDT. */
    FID_SEQ_SPECIAL = 0x200000004ULL,
    FID_SEQ_QUOTA = 0x200000005ULL,
    FID_SEQ_QUOTA_GLB = 0x200000006ULL,
    FID_SEQ_ROOT = 0x200000007ULL, /* Located on MDT0 */
    FID_SEQ_LAYOUT_RBTREE = 0x200000008ULL,
    /* sequence is used for update logs of cross-MDT operation */
    FID_SEQ_UPDATE_LOG = 0x200000009ULL,
    /* Sequence is used for the directory under which update logs
     * are created. */
    FID_SEQ_UPDATE_LOG_DIR = 0x20000000aULL,
    FID_SEQ_NORMAL = 0x200000400ULL,
    FID_SEQ_LOV_DEFAULT = 0xffffffffffffffffULL};#define __be32_to_cpu(x) ((uint32_t)((((uint32_t)(x) & (uint32_t)0x000000ffUL) << 24) | \
                                     (((uint32_t)(x) & (uint32_t)0x0000ff00UL) << 8) |  \
                                     (((uint32_t)(x) & (uint32_t)0x00ff0000UL) >> 8) |  \
                                     (((uint32_t)(x) & (uint32_t)0xff000000UL) >> 24)))#define __be64_to_cpu(x) ((uint64_t)((((uint64_t)(x) & (uint64_t)0x00000000000000ffULL) << 56) | \
                                     (((uint64_t)(x) & (uint64_t)0x000000000000ff00ULL) << 40) | \
                                     (((uint64_t)(x) & (uint64_t)0x0000000000ff0000ULL) << 24) | \
                                     (((uint64_t)(x) & (uint64_t)0x00000000ff000000ULL) << 8) |  \
                                     (((uint64_t)(x) & (uint64_t)0x000000ff00000000ULL) >> 8) |  \
                                     (((uint64_t)(x) & (uint64_t)0x0000ff0000000000ULL) >> 24) | \
                                     (((uint64_t)(x) & (uint64_t)0x00ff000000000000ULL) >> 40) | \
                                     (((uint64_t)(x) & (uint64_t)0xff00000000000000ULL) >> 56)))static int read_lustre_attr(const char *file, const char *attr_name, void **attr_buf){
    *attr_buf = malloc(XATTR_SIZE_MAX);
    assert(attr_buf != NULL);
    memset(*attr_buf, '\0', XATTR_SIZE_MAX);
    int fd = open(file, O_RDONLY);
    assert(fd != -1);
    ssize_t bytes_read = fgetxattr(fd, attr_name, *attr_buf, XATTR_SIZE_MAX);
    if (bytes_read < 0)
    {
        free(*attr_buf);
        *attr_buf = NULL;
        fprintf(stdout, "no attribute:%s on %s\n", attr_name, file);
        return -1;
    }
    return fd;}static inline bool fid_seq_is_igif(uint64_t seq){
    return seq >= FID_SEQ_IGIF && seq <= FID_SEQ_IGIF_MAX;}static inline bool fid_is_igif(const struct lu_fid *fid){
    return fid_seq_is_igif(fid->f_seq);}static inline uint64_t lu_igif_ino(const struct lu_fid *fid){
    return fid->f_seq;}static inline uint64_t fid2inode(const struct lu_fid *fid){
    uint64_t ino;
    uint64_t seq;

    if (fid_is_igif(fid))
    {
        ino = lu_igif_ino(fid);
        return ino;
    }

    seq = fid->f_seq;

    ino = (seq << 24) + ((seq >> 24) & 0xffffff0000ULL) + fid->f_oid;

    return ino ?: fid->f_oid;}int fetch_lustre_lov_attr(const char *file){
    void *attr_read_buf = NULL;
    int fd = read_lustre_attr(file, XATTR_LUSTRE_LOV, &attr_read_buf);
    if (fd == -1)
    {
        return fd;
    }
    struct lov_user_md *lum = (struct lov_user_md *)attr_read_buf;
    fprintf(stdout, "******************%s on %s**************\n", XATTR_LUSTRE_LOV, file);
    fprintf(stdout, "lmm_magic:0x%02X\n", lum->lmm_magic);
    fprintf(stdout, "lmm_pattern:%d\n", lum->lmm_pattern);
    fprintf(stdout, "lmm_stripe_size:%d\n", lum->lmm_stripe_size);
    fprintf(stdout, "lmm_stripe_count:%d\n", lum->lmm_stripe_count);
    fprintf(stdout, "lmm_layout_gen:%d\n", lum->lmm_layout_gen);

    for (int i = 0; i < lum->lmm_stripe_count; i++)
    {
        fprintf(stdout, "    ost_idx=%d,oi_id=%ld,oi_seq=%ld,fid.f_seq=%ld,fid.f_oid=%d,fid.f_ver=%d\n", lum->lmm_objects[i].l_ost_idx, lum->lmm_objects[i].l_ost_oi.oi.oi_id, lum->lmm_objects[i].l_ost_oi.oi.oi_seq, lum->lmm_objects[i].l_ost_oi.oi_fid.f_seq, lum->lmm_objects[i].l_ost_oi.oi_fid.f_oid, lum->lmm_objects[i].l_ost_oi.oi_fid.f_ver);
    }
    close(fd);
    free(attr_read_buf);
    attr_read_buf = NULL;
    return 0;}int fetch_trusted_link_attr(const char *file){
    void *attr_read_buf = NULL;
    int fd = read_lustre_attr(file, XATTR_NAME_LINK, &attr_read_buf);
    if (fd == -1)
    {
        return fd;
    }
    struct link_ea_header *leh = (struct link_ea_header *)attr_read_buf;
    fprintf(stdout, "******************%s on %s**************\n", XATTR_NAME_LINK, file);

    fprintf(stdout, "link_ea_header leh_magic:0x%02X\n", leh->leh_magic);
    fprintf(stdout, "link_ea_header leh_reccount:%d\n", leh->leh_reccount);
    fprintf(stdout, "link_ea_header leh_len:%d\n", leh->leh_len);

    struct link_ea_entry *link_ea = (struct link_ea_entry *)(leh + 1);

    struct lu_fid *pfid_tmp = (struct lu_fid *)&link_ea->lee_parent_fid;
    //  struct lu_fid pfid = *pfid_tmp;

    struct lu_fid pfid = {
        .f_seq = __be64_to_cpu(pfid_tmp->f_seq),
        .f_oid = __be32_to_cpu(pfid_tmp->f_oid),

        .f_ver = __be32_to_cpu(pfid_tmp->f_ver),
    };

    uint16_t file_name_len = (link_ea->lee_reclen[0] << 8) | link_ea->lee_reclen[1];
    char *file_name = (char *)calloc(file_name_len + 1, sizeof(char));
    memcpy(file_name, link_ea->lee_name, file_name_len);

    fprintf(stdout, "link_ea_entry parent inode=%ld\n", fid2inode(&pfid));
    fprintf(stdout, "link_ea_entry pfid.f_seq=%ld,pfid.f_oid=%d,pfid.f_ver=%d\n", __be64_to_cpu(pfid_tmp->f_seq), __be32_to_cpu(pfid_tmp->f_oid), __be32_to_cpu(pfid_tmp->f_ver));
    fprintf(stdout, "link_ea_entry filename=%s,filename_len=%d\n", file_name, file_name_len);

    close(fd);
    free(attr_read_buf);
    attr_read_buf = NULL;
    return 0;}int fetch_trusted_lma_attr(const char *file){
    void *attr_read_buf = NULL;
    int fd = read_lustre_attr(file, XATTR_NAME_LMA, &attr_read_buf);
    if (fd == -1)
    {
        return fd;
    }
    struct lustre_mdt_attrs *lma = (struct lustre_mdt_attrs *)attr_read_buf;
    fprintf(stdout, "******************%s on %s**************\n", XATTR_NAME_LMA, file);

    fprintf(stdout, "lustre_mdt_attrs lma_compat:%d\n", lma->lma_compat);
    fprintf(stdout, "lustre_mdt_attrs lma_incompat:%d\n", lma->lma_incompat);
    struct lu_fid fid = lma->lma_self_fid;

    fprintf(stdout, "lustre_mdt_attrs self inode=%ld\n", fid2inode(&fid));
    fprintf(stdout, "lustre_mdt_attrs fid.f_seq=%ld,fid.f_oid=%d,fid.f_ver=%d\n", fid.f_seq, fid.f_oid, fid.f_ver);
    close(fd);
    free(attr_read_buf);
    attr_read_buf = NULL;
    return 0;}int fetch_trusted_lov_attr(const char *file){
    void *attr_read_buf = NULL;
    int fd = read_lustre_attr(file, XATTR_NAME_LOV, &attr_read_buf);
    if (fd == -1)
    {
        return fd;
    }
    struct lov_mds_md *lmm = (struct lov_mds_md *)attr_read_buf;
    fprintf(stdout, "******************%s on %s**************\n", XATTR_NAME_LOV, file);

    fprintf(stdout, "lov_mds_md_v1 lmm_magic:0x%02X\n", lmm->lmm_magic);
    fprintf(stdout, "lov_mds_md_v1 lmm_pattern:%d\n", lmm->lmm_pattern);
    fprintf(stdout, "lov_mds_md_v1 lmm_stripe_size:%d\n", lmm->lmm_stripe_size);

    fprintf(stdout, "lov_mds_md_v1 lmm_stripe_count:%d\n", lmm->lmm_stripe_count);
    fprintf(stdout, "lov_mds_md_v1 lmm_layout_gen:%d\n", lmm->lmm_layout_gen);

    struct ost_id oid = lmm->lmm_oi;
    struct lu_fid fid = oid.oi_fid;
    fprintf(stdout, "lov_mds_md_v1 mdt: oi_id=%ld,oi_seq=%ld,fid.f_seq=%ld,fid.f_oid=%d,fid.f_ver=%d\n", oid.oi.oi_id, oid.oi.oi_seq, fid.f_seq, fid.f_oid, fid.f_ver);
    fprintf(stdout, "lov_mds_md_v1 ost: \n");
    for (int i = 0; i < lmm->lmm_stripe_count; i++)
    {
        struct lov_ost_data_v1 lov_tmp = lmm->lmm_objects[i];
        struct ost_id oid_tmp = lov_tmp.l_ost_oi;
        fprintf(stdout, "   on ost: ost_index=%d,oi_id=%ld,oi_seq=%ld,fid.f_seq=%ld,fid.f_oid=%d,fid.f_ver=%d\n", lov_tmp.l_ost_idx, oid_tmp.oi.oi_id, oid_tmp.oi.oi_seq, fid.f_seq, fid.f_oid, fid.f_ver);
    }
    close(fd);
    free(attr_read_buf);
    attr_read_buf = NULL;
    return 0;}int main(int argc, char *argv[]){
    // fetch_lustre_lov_attr(argv[1]);
    // fetch_trusted_link_attr(argv[1]);
    // fetch_trusted_lma_attr(argv[1]);
    fetch_trusted_lov_attr(argv[1]);
    return 0;}