TFS测试环境部署文档
2014-03-21Linux撒加7969°c
A+ A-操作系统版本
CentOS 6.4 x86_64
软件版本
TFS：dev for out users
Nginx：1.4.5
Nginx_tfs：最新版本
Pcre：8.34
jemalloc：3.5.1
软件安装目录
TFS：/opt/tfs
Nginx：/opt/nginx
配置文件目录
TFS NameServer：/opt/tfs/conf/ns.conf
TFS DataServer：/opt/tfs/conf/ds.conf
软件启动脚本
TFS：/etc/init.d/rc.tfs
Nginx：/etc/init.d/rc.nginx
日志文件目录
TFS：/opt/tfs/logs
Nginx：/opt/nginx/logs
源码包目录
/root
IP地址划分及用途
10.1.207.110 Data Server Node1
10.1.207.111 Data Server Node2
10.1.207.112 Data Server Node3
10.1.207.114 Name Server+Nginx
分区信息
Data Server磁盘分区
/dev/sdb 20642428 19925684 0 100% /data1
/dev/sdc 20642428 19925696 0 100% /data2
分区文件系统为ext4
TFS安装（以下步骤在4台机器上都需要执行）
依赖包安装
1、添加第三方软件升级源
rpm -ivh https://mirrors.sohu.com/fedora-epel/6/x86_64/epel-release-6-8.noarch.rpm
2、安装依赖包
yum check-update
yum -y install zlib-devel libtool automake libuuid-devel readline-devel e2fsprogs-devel svn git
TFS Common Utils安装
1、获取TFS Common Utils
svn checkout -r 18 https://code.taobao.org/svn/tb-common-utils/trunk/ tb-common-utils
2、安装common utils
export TBLIB_ROOT=/opt/tfs
cd tb-common-utils
sh build.sh
TFS部署
1、获取并安装jemalloc
wget https://www.canonware.com/download/jemalloc/jemalloc-3.5.1.tar.bz2
tar jxf jemalloc-3.5.1.tar.bz2
cd jemalloc-3.5.1
./configure
make && make install
2、获取TFS并安装
svn co https://code.taobao.org/svn/tfs/branches/dev_for_outer_users tfs
cd tfs
./configure --prefix=/opt/tfs --with-tblib-root=/opt/tfs --without-tcmalloc --with-release
make && make install
cp /opt/tfs/scripts/tfs /etc/init.d/rc.tfs ---->TFS服务控制脚本，在4台机器上都执行
cp conf/ns.conf /opt/tfs/conf/ ----> NameServer服务器执行
cp conf/ds.conf /opt/tfs/conf/ ----> DataServer服务器执行
3、配置文件修改
NameServer ns.conf
[public]
log_size=1073741824
log_num = 4
log_level=warn
task_max_queue_size = 10240
port = 8100 ----->nameserver监听端口
work_dir=/opt/tfs ----->work directoy,设置为tfs的安装目录
dev_name= eth0 ----->device name，TFS通信的接口，此接口的ip地址若与ip_addr参数不一致，日志中将报错
thread_count =32
ip_addr = 10.1.207.114
[nameserver]
safe_mode_time = 300
ip_addr_list = 10.1.207.114|192.168.0.2
group_mask = 255.255.255.255
max_write_timeout = 3
cluster_id = 1
block_max_use_ratio = 98
block_max_size = 75497472
max_replication = 3
min_replication = 2
replicate_ratio = 50
max_write_filecount = 64
use_capacity_ratio = 96
heart_interval = 4
object_dead_max_time = 300
object_clear_max_time = 180
heart_thread_count = 4
heart_max_queue_size = 2048
report_block_thread_count = 6
report_block_max_queue_size = 32
report_block_hour_range = 2~4
report_block_time_interval = 1
repl_wait_time = 180
compact_delete_ratio = 10
compact_max_load = 200
compact_hour_range = 1~10
dump_stat_info_interval = 60000000
balance_percent = 0.05
add_primary_block_count = 3
task_percent_sec_size = 200
oplog_sync_max_slots_num = 1024
oplog_sync_thread_num = 1
group_count = 1
group_seq = 0
discard_newblk_safe_mode_time = 360
choose_target_server_random_max_num = 128
Data Server ds.conf 其中斜体字根据Data Server的监听端口、IP以及Device Name进行修改，红色字体在3台Data Server配置中保持不变
[public]
log_size=1073741824
log_num = 4
log_level=info
task_max_queue_size = 10240
port = 8200 ----->data server监听端口
work_dir=/opt/tfs ----->work directoy,设置为tfs的安装目录
dev_name= eth0 ----->device name，TFS通信的接口，此接口的ip地址若与ip_addr参数不一致，日志中将报错
thread_count = 32
ip_addr = 10.1.207.112 ----->dev_name所设置网口的ip
[dataserver]
ip_addr = 10.1.207.114 ----->nameserver ip地址
ip_addr_list = 10.1.207.114|192.168.0.2 ----->第一个IP为ns ip，第二个为ns 备机，若无ns备机，第二个ip随意指定，不指定将报错
port = 8100 ----->nameserver 监听端口
heart_interval = 2
check_interval = 2
replicate_threadcount = 1
block_max_size = 75497472
dump_visit_stat_interval = 60
backup_type = 1
backup_path = /mnt/tfs
mount_name = /data ------>TFS数据盘目录，不带disk index num
mount_maxsize = 19922944 ----->不能超过df输出列中Available，例如/dev/sdb在df中的Available为26065072，则此处不得超过26065072
base_filesystem_type = 1
superblock_reserve = 0
avg_file_size = 15360
mainblock_size = 75497472
extblock_size = 4194304
block_ratio = 0.5
hash_slot_ratio = 0.5
4、启动服务
§ 启动Name Server
在10.1.207.114上执行/etc/init.d/rc.tfs start_ns
§ 启动Data Server
在110-112先初始化分区，执行:
/opt/tfs/scripts/stfs clear 1,2
/opt/tfs/scripts/stfs format 1,2
此处1,2对应于目录/data1 /data2中的数字部分，该数字在TFS称为disk index number
/etc/init.d/rc.tfs start_ds 1,2 启动data server服务
5、停止服务
§ 停止Name Server
在10.1.207.114上执行/etc/init.d/rc.tfs stop_ns
§ 停止Data Server
在110-112上执行/etc/init.d/rc.tfs stop_ds 1,2
Nginx安装(此操作在10.1.207.114上执行)
1、安装Nginx_tfs所依赖的开源JSON库软件---yajl
wget -O yajl-2.0.1.tar.gz https://codeload.github.com/lloyd/yajl/legacy.tar.gz/2.0.1
tar zxf yajl-2.0.1.tar.gz
cd lloyd-yajl-f4b2b1a
./configure
make && make install
echo "/usr/local/lib" > /etc/ld.so.conf.d/yajl.conf
ldconfig
2、安装Nginx
下载nginx_tfs
git clone https://github.com/alibaba/nginx-tfs.git
下载Nginx 1.4.5
wget https://nginx.org/download/nginx-1.4.5.tar.gz
下载pcre 8.34
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.34.tar.gz
创建用户和组
groupadd www
useradd -g www -s /sbin/nologin -m www
解压并安装软件
tar zxf pcre-8.34.tar.gz
tar zxf nginx-1.4.5.tar.gz
cd nginx
./configure \
--prefix=/opt/nginx \
--user=www \
--group=www \
--without-select_module \
--without-poll_module \
--with-file-aio \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--without-http_ssi_module \
--without-http_autoindex_module \
--without-http_uwsgi_module \
--without-http_scgi_module \
--without-http_memcached_module \
--http-client-body-temp-path=/opt/nginx/cache/client/ \
--http-proxy-temp-path=/opt/nginx/cache/proxy/ \
--without-http_fastcgi_module \
--without-mail_pop3_module \
--without-mail_imap_module \
--without-mail_smtp_module \
--with-pcre=../pcre-8.34 \
--add-module=../nginx-tfs
make && make install
编辑/opt/nginx/conf/nginx.conf，清空该文件，输入以下内容：
user www www;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
error_log /var/log/nginx_error.log crit;
pid /var/run/nginx.pid;
worker_rlimit_nofile 60000;
events
{
use epoll;
worker_connections 8192;
multi_accept on;
}
http {
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
server_names_hash_max_size 512;
client_header_buffer_size 32k;
client_header_timeout 10;
client_body_buffer_size 16k;
client_body_timeout 10;
large_client_header_buffers 16 8k;
client_max_body_size 100m;
directio 4m;
server_tokens off;
sendfile on;
send_timeout 2;
tcp_nopush on;
tcp_nodelay on;
types_hash_max_size 2048;
keepalive_timeout 30;
keepalive_requests 100000;
reset_timedout_connection on;
open_file_cache max=200000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;
gzip on;
gzip_min_length 1k;
gzip_disable "MSIE [1-6]\.";
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_comp_level 5;
gzip_proxied expired no-cache no-store private auth;
gzip_types image/gif image/jpeg text/plain application/x-javascript text/css application/xml text/javascript;
gzip_vary on;
tfs_body_buffer_size 2m;
tfs_send_timeout 3s;
tfs_connect_timeout 3s;
tfs_read_timeout 3s;
tfs_upstream tfs_ns {
erver 127.0.0.1:8100;
type ns;
}
server {
listen 80;
server_name 10.1.207.114;
tfs_keepalive max_cached=100 bucket_count=10;
tfs_log /opt/nginx/logs/cronolog/tfs_access.log;
location / {
tfs_pass tfs://tfs_ns;
}
}
}
编辑Nginx启动脚本/etc/init.d/rc.nginx，内容如下：
#! /bin/sh
# Description: Startup script for nginx
# chkconfig: 2345 55 25
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DESC="Nginx Daemon"
NAME=nginx
DAEMON=/opt/nginx/sbin/$NAME
CONFIGFILE=/opt/nginx/conf/nginx.conf
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/rc.$NAME
set -e
[ -x "$DAEMON" ] || exit 0
do_start() {
$DAEMON -c $CONFIGFILE || echo -n "nginx already running"
}
do_stop() {
$DAEMON -c $CONFIGFILE -s stop || echo -n "nginx not running"
}
do_reload() {
$DAEMON -c $CONFIGFILE -s reload || echo -n "nginx can't reload"
}
case "$1" in
start)
echo -n "Starting $DESC: $NAME"
do_start
echo "."
;;
stop)
echo -n "Stopping $DESC: $NAME"
do_stop
echo "."
;;
reload)
echo -n "Reloading $DESC configuration..."
do_reload
echo "."
;;
restart)
echo -n "Restarting $DESC: $NAME"
do_stop
sleep 1
do_start
echo "."
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|reload|restart}" >&2
exit 3
;;
esac
exit 0
编辑好脚本后，启动Nginx
chmod +x /etc/init.d/rc.nginx
/etc/init.d/rc.nginx start
测试
在命令行下通过curl来测试是否能够上传、读取图片文件
此时在浏览器里，输入https://10.1.207.114/v1/tfs/T1ARxTByJT1RCvBVdK即可显示configuration3.png文件的内容了
