分布式文件系统 fastdfs

零月

发布于 2018-04-25 16:11:00

1.8K0

发布于 2018-04-25 16:11:00

fastdfs

FastDFS是一款开源的轻量级分布式文件系统纯C实现，支持Linux、FreeBSD等UNIX系统类google FS，不是通用的文件系统，只能通过专有API访问，目前提供了C、Java和PHP API为互联网应用量身定做，解决大容量文件存储问题，追求高性能和高扩展性FastDFS可以看做是基于文件的key value pair存储系统，称作分布式文件存储服务更为合适。

主机	ip	端口（tracker）	端口（storage）
node1（tracker）	128.0.0.81	22122
node2（storage）	128.0.0.82		23000
node3（storage）	128.0.0.83		23000

安装tracker

[root@node1 ~]# wget https://github.com/happyfish100/libfastcommon/archive/V1.0.7.tar.gz
[root@node1 ~]# tar -zxvf V1.0.7.tar.gz
[root@node1 ~]# cd libfastcommon-1.0.7/
[root@node1 libfastcommon-1.0.7]# ls
HISTORY  INSTALL  libfastcommon.spec  make.sh  README  src
[root@node1 libfastcommon-1.0.7]# ./make.sh
[root@node1 libfastcommon-1.0.7]# ./make.sh install
[root@node1 ~]# wget https://github.com/happyfish100/fastdfs/archive/V5.05.tar.gz
[root@node1 ~]# tar -zxvf V5.05.tar.gz
[root@node1 ~]# cd fastdfs-5.05/
[root@node1 fastdfs-5.05]# ./make.sh
[root@node1 fastdfs-5.05]# ./make.sh install
[root@node1 ~]# cd /etc/fdfs/
[root@node1 fdfs]# ls
client.conf.sample  storage.conf.sample  tracker.conf.sample
[root@node1 fdfs]# cp tracker.conf.sample tracker.conf
#修改目录和端口
[root@node1 fdfs]# vim tracker.conf
base_path=/data/fastdfs/tracker
http.server_port=80
#创建目录
[root@node1 fdfs]# mkdir -pv /data/fastdfs/tracker
#做软链接
[root@node1 fdfs]# ln -s /usr/bin/fdfs_trackerd /usr/local/bin
[root@node1 fdfs]# ln -s /usr/bin/stop.sh /usr/local/bin
[root@node1 fdfs]# ln -s /usr/bin/restart.sh /usr/local/bin
#启动
[root@node1 fdfs]# service fdfs_trackerd start
Unit fdfs_trackerd.service could not be found.
Reloading systemd:                                         [  OK  ]
Starting fdfs_trackerd (via systemctl):                    [  OK  ]
#查看端口
[root@node1 fdfs]# netstat -nltp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 0.0.0.0:22122           0.0.0.0:*               LISTEN      22547/fdfs_trackerd

storage的安装与tracker安装类似，首先编译安装fastdfs，然后修改配置文件。

[root@node2 ~]# cd /etc/fdfs/
[root@node2 fdfs]# cp storage.conf.sample storage.conf
[root@node3 fdfs]# cp storage.conf.sample storage.conf
[root@node2 fdfs]# vim storage.conf
disabled=false #启用配置文件
group_name=group1 #组名，根据实际情况修改
port=23000 #设置 storage 的端口号
base_path=/data/fastdfs/storage #设置 storage 的日志目录（需预先创建）
store_path_count=1 #存储路径个数，需要和 store_path 个数匹配 store_path0=/data/fastdfs/storage #存储路径
tracker_server=128.0.0.81:22122 #tracker 服务器的 IP 地址和端口号
#创建目录
[root@node2 bin]# mkdir -pv /data/fastdfs/storage
#软链接
[root@node2 fdfs]# ln -s /usr/bin/fdfs_storaged /usr/local/bin
[root@node2 bin]# ln -s /usr/bin/fdfs_trackerd /usr/local/bin
[root@node2 bin]# ln -s /usr/bin/stop.sh /usr/local/bin
[root@node2 bin]# ln -s /usr/bin/restart.sh /usr/local/bin
#启动
[root@node2 bin]# systemctl start fdfs_storaged.service
#查看端口
[root@node2 bin]# netstat -nlatp|grep fdfs
tcp        0      0 0.0.0.0:23000           0.0.0.0:*               LISTEN      22431/fdfs_storaged 
tcp        0      0 128.0.0.82:53724        128.0.0.81:22122        ESTABLISHED 22431/fdfs_storaged
[root@node2 bin]# ll /data/fastdfs/storage
total 12
drwxr-xr-x 259 root root 8192 Dec  3 13:45 data
drwxr-xr-x   2 root root   26 Dec  3 13:41 logs

查看Tracker 和 Storage 服务的通信

[root@node2 ~]# /usr/bin/fdfs_monitor /etc/fdfs/storage.conf
[2017-12-03 14:29:19] DEBUG - base_path=/data/fastdfs/storage, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

server_count=1, server_index=0

tracker server is 128.0.0.81:22122

group count: 1

Group 1:
group name = group1
disk total space = 92115 MB
disk free space = 90561 MB
trunk free space = 0 MB
storage server count = 1
active server count = 1
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0

    Storage 1:
        id = 128.0.0.82
        ip_addr = 128.0.0.82 (node2)  ACTIVE
        http domain = 
        version = 5.05
        join time = 2017-12-03 13:41:51
        up time = 2017-12-03 13:45:35
        total storage = 92115 MB
        free storage = 90561 MB
        upload priority = 10
        store_path_count = 1
        subdir_count_per_path = 256
        storage_port = 23000
        storage_http_port = 8888
        current_write_path = 0
        source storage id = 
        if_trunk_server = 0
        connection.alloc_count = 256
        connection.current_count = 0
        connection.max_count = 0
        total_upload_count = 0
        success_upload_count = 0
        total_append_count = 0
        success_append_count = 0
        total_modify_count = 0
        success_modify_count = 0
        total_truncate_count = 0
        success_truncate_count = 0
        total_set_meta_count = 0
        success_set_meta_count = 0
        total_delete_count = 0
        success_delete_count = 0
        total_download_count = 0
        success_download_count = 0
        total_get_meta_count = 0
        success_get_meta_count = 0
        total_create_link_count = 0
        success_create_link_count = 0
        total_delete_link_count = 0
        success_delete_link_count = 0
        total_upload_bytes = 0
        success_upload_bytes = 0
        total_append_bytes = 0
        success_append_bytes = 0
        total_modify_bytes = 0
        success_modify_bytes = 0
        stotal_download_bytes = 0
        success_download_bytes = 0
        total_sync_in_bytes = 0
        success_sync_in_bytes = 0
        total_sync_out_bytes = 0
        success_sync_out_bytes = 0
        total_file_open_count = 0
        success_file_open_count = 0
        total_file_read_count = 0
        success_file_read_count = 0
        total_file_write_count = 0
        success_file_write_count = 0
        last_heart_beat_time = 2017-12-03 14:29:06
        last_source_update = 1970-01-01 08:00:00
        last_sync_update = 1970-01-01 08:00:00
        last_synced_timestamp = 1970-01-01 08:00:00

查看端口

[root@node3 ~]# netstat -nlatp | grep fdfs
tcp        0      0 0.0.0.0:23000           0.0.0.0:*               LISTEN      22381/fdfs_storaged 
tcp        0      0 128.0.0.83:23000        128.0.0.82:60548        ESTABLISHED 22381/fdfs_storaged 
tcp        0      0 128.0.0.83:60226        128.0.0.81:22122        ESTABLISHED 22381/fdfs_storaged 
tcp        0      0 128.0.0.83:54014        128.0.0.82:23000        ESTABLISHED 22381/fdfs_storaged

查看Tracker 和 Storage 服务的通信

[root@node3 ~]# /usr/bin/fdfs_monitor /etc/fdfs/storage.conf
[2017-12-03 14:46:15] DEBUG - base_path=/data/fastdfs/storage, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

server_count=1, server_index=0

tracker server is 128.0.0.81:22122

group count: 1

Group 1:
group name = group1
disk total space = 92115 MB
disk free space = 90560 MB
trunk free space = 0 MB
storage server count = 2
active server count = 2
storage server port = 23000
storage HTTP port = 8888
store path count = 1
subdir count per path = 256
current write server index = 0
current trunk file id = 0

    Storage 1:
        id = 128.0.0.82
        ip_addr = 128.0.0.82 (node2)  ACTIVE
        http domain = 
        version = 5.05
        join time = 2017-12-03 13:41:51
        up time = 2017-12-03 13:45:35
        total storage = 92115 MB
        free storage = 90561 MB
        upload priority = 10
        store_path_count = 1
        subdir_count_per_path = 256
        storage_port = 23000
        storage_http_port = 8888
        current_write_path = 0
        source storage id = 
        if_trunk_server = 0
        connection.alloc_count = 256
        connection.current_count = 1
        connection.max_count = 1
        total_upload_count = 0
        success_upload_count = 0
        total_append_count = 0
        success_append_count = 0
        total_modify_count = 0
        success_modify_count = 0
        total_truncate_count = 0
        success_truncate_count = 0
        total_set_meta_count = 0
        success_set_meta_count = 0
        total_delete_count = 0
        success_delete_count = 0
        total_download_count = 0
        success_download_count = 0
        total_get_meta_count = 0
        success_get_meta_count = 0
        total_create_link_count = 0
        success_create_link_count = 0
        total_delete_link_count = 0
        success_delete_link_count = 0
        total_upload_bytes = 0
        success_upload_bytes = 0
        total_append_bytes = 0
        success_append_bytes = 0
        total_modify_bytes = 0
        success_modify_bytes = 0
        stotal_download_bytes = 0
        success_download_bytes = 0
        total_sync_in_bytes = 0
        success_sync_in_bytes = 0
        total_sync_out_bytes = 0
        success_sync_out_bytes = 0
        total_file_open_count = 0
        success_file_open_count = 0
        total_file_read_count = 0
        success_file_read_count = 0
        total_file_write_count = 0
        success_file_write_count = 0
        last_heart_beat_time = 2017-12-03 14:46:05
        last_source_update = 1970-01-01 08:00:00
        last_sync_update = 1970-01-01 08:00:00
        last_synced_timestamp = 1970-01-01 08:00:00 
    Storage 2:
        id = 128.0.0.83
        ip_addr = 128.0.0.83 (node3)  ACTIVE
        http domain = 
        version = 5.05
        join time = 2017-12-03 14:44:08
        up time = 2017-12-03 14:44:31
        total storage = 92115 MB
        free storage = 90560 MB
        upload priority = 10
        store_path_count = 1
        subdir_count_per_path = 256
        storage_port = 23000
        storage_http_port = 8888
        current_write_path = 0
        source storage id = 128.0.0.82
        if_trunk_server = 0
        connection.alloc_count = 256
        connection.current_count = 1
        connection.max_count = 1
        total_upload_count = 0
        success_upload_count = 0
        total_append_count = 0
        success_append_count = 0
        total_modify_count = 0
        success_modify_count = 0
        total_truncate_count = 0
        success_truncate_count = 0
        total_set_meta_count = 0
        success_set_meta_count = 0
        total_delete_count = 0
        success_delete_count = 0
        total_download_count = 0
        success_download_count = 0
        total_get_meta_count = 0
        success_get_meta_count = 0
        total_create_link_count = 0
        success_create_link_count = 0
        total_delete_link_count = 0
        success_delete_link_count = 0
        total_upload_bytes = 0
        success_upload_bytes = 0
        total_append_bytes = 0
        success_append_bytes = 0
        total_modify_bytes = 0
        success_modify_bytes = 0
        stotal_download_bytes = 0
        success_download_bytes = 0
        total_sync_in_bytes = 0
        success_sync_in_bytes = 0
        total_sync_out_bytes = 0
        success_sync_out_bytes = 0
        total_file_open_count = 0
        success_file_open_count = 0
        total_file_read_count = 0
        success_file_read_count = 0
        total_file_write_count = 0
        success_file_write_count = 0
        last_heart_beat_time = 2017-12-03 14:46:00
        last_source_update = 1970-01-01 08:00:00
        last_sync_update = 1970-01-01 08:00:00
        last_synced_timestamp = 1970-01-01 08:00:00
#都是active通信成功
ip_addr = 128.0.0.82 (node2)  ACTIVE
ip_addr = 128.0.0.83 (node3)  ACTIVE

安装nginx

yum -y install gcc pcre pcre-devel zlib openssl
wget https://github.com/happyfish100/fastdfs-nginx-module/archive/master.zip
wget -c https://nginx.org/download/nginx-1.10.1.tar.gz

#配置/usr/local/src/fastdfs-nginx-module/src/目录下的config文件，把CORE_INCS和CORE_LIBS的所有路径都修改为/usr/include和/usr/lib
vim /root/fastdfs-nginx-module/src/config
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
CORE_LIBS="$CORE_LIBS -L/usr/lib -lfastcommon -lfdfsclient"
make && make install
/usr/local/nginx/sbin/nginx -V
/usr/local/nginx/sbin/nginx

配置fastdfs模块

[root@node2 ~]# cd fastdfs-nginx-module/src/
[root@node2 src]# ls
common.c  config            ngx_http_fastdfs_module.c
common.h  mod_fastdfs.conf
[root@node2 src]# cp mod_fastdfs.conf /etc/fdfs
[root@node2 src]# cd /etc/fdfs/
[root@node2 fdfs]# vim mod_fastdfs.conf
  #保存日志目录
  base_path=/data/fastdfs/storage 
  #tracker 服务器的 IP 地址以及端口号 storage_server_port=23000 #storage 服务器的端口号
  tracker_server=128.0.0.81:22122 
  #当前服务器的 group 名
  group_name=group1
  #文件 url 中是否有 group 名
  url_have_group_name = true
  #存储路径个数，需要和 store_path 个数匹配 
  store_path0=/data/fastdfs/storage #存储路径
  store_path_count=1 
  group_count = 2#设置组的个数
  # 从文件扩展名查找文件类型（ nginx时为true） 
  http.need_find_content_type=true 

  [group1]
  group_name=group1
  storage_server_port=23000
  store_path_count=1
  store_path0=/data/fastdfs/storage

配置nginx

[root@node2 ~]# cd fastdfs-5.05/
[root@node2 fastdfs-5.05]# cd conf/
[root@node2 conf]# cp anti-steal.jpg http.conf mime.types /etc/fdfs/
[root@node2 conf]# cd /usr/local/nginx/conf
[root@node2 conf]# vim nginx.conf
  location /group1/M00 {
            root /data/fastdfs/storage/;
            ngx_fastdfs_module;
      }
[root@node2 conf]# mkdir /data/fastdfs/storage/data/group1
[root@node2 conf]# ln -s /data/fastdfs/storage/data /data/fastdfs/storage/data/group1/M00

配置客户端

[root@node2 conf]# cd /etc/fdfs
[root@node2 fdfs]# cp client.conf.sample client.conf
base_path=/data/fastdfs/client
tracker_server=192.168.2.231:22122
mkdir /data/fastdfs/client

上传

[root@node2 ~]# /usr/bin/fdfs_test /etc/fdfs/client.conf upload timg.jpg
This is FastDFS client test program v5.05

Copyright (C) 2008, Happy Fish / YuQing

FastDFS may be copied only under the terms of the GNU General
Public License V3, which may be found in the FastDFS source kit.
Please visit the FastDFS Home Page http://www.csource.org/ 
for more detail.

[2017-12-03 18:21:45] DEBUG - base_path=/data/fastdfs/client, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

tracker_query_storage_store_list_without_group: 
    server 1. group_name=, ip_addr=128.0.0.82, port=23000
    server 2. group_name=, ip_addr=128.0.0.83, port=23000

group_name=group1, ip_addr=128.0.0.82, port=23000
storage_upload_by_filename
group_name=group1, remote_filename=M00/00/00/gAAAUloj0DmATy21AAMuQuM_dQs658.jpg
source ip address: 128.0.0.82
file timestamp=2017-12-03 18:21:45
file size=208450
file crc32=3812586763
example file url: http://128.0.0.82/group1/M00/00/00/gAAAUloj0DmATy21AAMuQuM_dQs658.jpg
storage_upload_slave_by_filename
group_name=group1, remote_filename=M00/00/00/gAAAUloj0DmATy21AAMuQuM_dQs658_big.jpg
source ip address: 128.0.0.82
file timestamp=2017-12-03 18:21:45
file size=208450
file crc32=3812586763
example file url: http://128.0.0.82/group1/M00/00/00/gAAAUloj0DmATy21AAMuQuM_dQs658_big.jpg