Nginx动态upstream实战

1、背景

Nginx后端服务发生变化时，需要手动修改upstream列表并reload，相对比较麻烦，特别是针对一些长连接应用中reload会早成断链和建链风暴，对后端应用造成较大的压力，因此在实际项目应用中需要一种动态修改upstream的机制，无需重启或者reload Nginx。本文简单描述了基于ngx_http_dyups_module模块实现动态修改更新upstream列表的实现。

2、关键模块依赖

Dynamic Upstream : https://github.com/yzprofile/ngx_http_dyups_module
健康检查：https://github.com/yaoweibin/nginx_upstream_check_module
一致性hash：https://github.com/replay/ngx_http_consistent_hash

3、Niginx编译脚本

下面的编译脚本支持lua、动态upstream等模块，已在生产环境实际运行。

#!/bin/bash

set -e

NGINX_VERSION="1.17.3"
NGINX_TARBALL="nginx-${NGINX_VERSION}.tar.gz"
PCRE_VERSION="8.43"
PCRE_TARBALL="pcre-${PCRE_VERSION}.tar.gz"
OPENSSL_VERSION="1.1.1d"
OPENSSL_TARBALL="openssl-${OPENSSL_VERSION}.tar.gz"
OPENSSL_PATCH_URL="http://www.linuxfromscratch.org/patches/downloads/openssl/openssl-1.1.1d-upstream_fix-1.patch"
ZLIB_VERSION="1.2.11"
ZLIB_TARBALL="zlib-${ZLIB_VERSION}.tar.gz"
LUA_JIT_VERSION="2.1-20190912"
LUA_JIT_TARBALL="luajit2-${LUA_JIT_VERSION}.tar.gz"
LUA_NGINX_VERSION="0.10.15"
LUA_NGINX_TARBALL="lua-nginx-module-${LUA_NGINX_VERSION}.tar.gz"
DEVEL_KIT_VERSION="0.3.1"
DEVEL_KIT_TARBALL="ngx_devel_kit-${DEVEL_KIT_VERSION}.tar.gz"
LUA_RESTY_CORE_VERSION="0.1.17"
LUA_RESTY_CORE_TARBALL="lua-resty-core-${LUA_RESTY_CORE_VERSION}.tar.gz"
LUA_RESTY_LRUCACHE_VERSION="0.09"
LUA_RESTY_LRUCACHE_TARBALL="lua-resty-lrucache-${LUA_RESTY_LRUCACHE_VERSION}.tar.gz"
LUA_RESTY_HTTP_VERSION="0.09"
LUA_RESTY_HTTP_TARBALL="lua-resty-http-${LUA_RESTY_LRUCACHE_VERSION}.tar.gz"

CWD=$(pwd)
TMP_DIR="${CWD}/.tmp/"
INSTALL_DIR=$HOME/opt/nginx

if [[ "${1}" == "clean" ]]; then
  rm -rf ${INSTALL_DIR}
  rm -rf .tmp
  exit 0
fi

if [[ -d "${INSTALL_DIR}" ]]; then
  rm -rf ${INSTALL_DIR}
fi

if [ ! -d "${TMP_DIR}" ] ; then
  mkdir ${TMP_DIR}
fi

cd ${TMP_DIR}

if [[ ! -d "${NGINX_TARBALL%.tar.gz}" ]]; then
  wget "http://nginx.org/download/${NGINX_TARBALL}"
  tar xvzf "${NGINX_TARBALL}" && rm -f "${NGINX_TARBALL}"
fi

if [[ ! -d "${PCRE_TARBALL%.tar.gz}" ]]; then
  wget "https://ftp.pcre.org/pub/pcre/${PCRE_TARBALL}"
  tar xvzf "${PCRE_TARBALL}" && rm -f "${PCRE_TARBALL}"
fi

if [[ ! -d "${OPENSSL_TARBALL%.tar.gz}" ]]; then
  wget "http://www.openssl.org/source/${OPENSSL_TARBALL}"
  tar xvzf "${OPENSSL_TARBALL}" && rm -f "${OPENSSL_TARBALL}"
  cd "${OPENSSL_TARBALL%.tar.gz}"
  wget "${OPENSSL_PATCH_URL}" -O fix-pod-syntax.patch
  patch -p1 < ./fix-pod-syntax.patch
  cd ${TMP_DIR}
fi

if [[ ! -d "${ZLIB_TARBALL%.tar.gz}" ]]; then
  wget "http://zlib.net/${ZLIB_TARBALL}"
  tar xvzf "${ZLIB_TARBALL}" && rm -rf "${ZLIB_TARBALL}"
fi

if [[ ! -d "${LUA_JIT_TARBALL%.tar.gz}" ]]; then
  wget -O "${LUA_JIT_TARBALL}" "https://codeload.github.com/openresty/luajit2/tar.gz/v${LUA_JIT_VERSION}"
  tar xvzf "${LUA_JIT_TARBALL}" && rm -rf "${LUA_JIT_TARBALL}"
  cd luajit2-${LUA_JIT_VERSION}
  make
  make install PREFIX="${TMP_DIR}opt/luajit/"
  cd ${TMP_DIR}
fi

if [[ ! -d "${LUA_NGINX_TARBALL%.tar.gz}" ]]; then
  wget -O "${LUA_NGINX_TARBALL}" "https://codeload.github.com/openresty/lua-nginx-module/tar.gz/v${LUA_NGINX_VERSION}"
  tar xvzf "${LUA_NGINX_TARBALL}" && rm -rf "${LUA_NGINX_TARBALL}"
fi

if [[ ! -d "${DEVEL_KIT_TARBALL%.tar.gz}" ]]; then
  wget -O ${DEVEL_KIT_TARBALL} "https://codeload.github.com/simplresty/ngx_devel_kit/tar.gz/v${DEVEL_KIT_VERSION}"
  tar xvzf "${DEVEL_KIT_TARBALL}" && rm -rf "${DEVEL_KIT_TARBALL}"
fi

if [[ ! -d "nginx_upstream_check_module" ]]; then
  git clone https://github.com/yaoweibin/nginx_upstream_check_module.git
fi

if [[ ! -d "ngx_http_dyups_module" ]]; then
  git clone https://github.com/yzprofile/ngx_http_dyups_module.git
fi

if [[ ! -d "ngx_http_consistent_hash" ]]; then
  git clone https://github.com/replay/ngx_http_consistent_hash.git
fi

export LUAJIT_LIB=${TMP_DIR}opt/luajit/lib 
export LUAJIT_INC=${TMP_DIR}opt/luajit/include/luajit-2.1/

cd "nginx-${NGINX_VERSION}"

./configure \
  --with-cpu-opt=generic \
  --prefix=${INSTALL_DIR} \
  --conf-path=${INSTALL_DIR}/nginx.conf \
  --http-log-path=${INSTALL_DIR}/logs/access.log \
  --error-log-path=${INSTALL_DIR}/logs/error.log \
  --lock-path=${INSTALL_DIR}/run/nginx.lock \
  --pid-path=${INSTALL_DIR}/run/nginx.pid \
  --http-client-body-temp-path=${INSTALL_DIR}/run/tmp/body \
  --http-fastcgi-temp-path=${INSTALL_DIR}/run/tmp/fastcgi \
  --http-proxy-temp-path=${INSTALL_DIR}/run/tmp/proxy \
  --http-scgi-temp-path=${INSTALL_DIR}/run/tmp/scgi \
  --http-uwsgi-temp-path=${INSTALL_DIR}/run/tmp/uwsgi \
  --with-pcre=../pcre-${PCRE_VERSION} \
  --sbin-path=. \
  --with-ld-opt="-L${LUAJIT_LIB} ${LUAJIT_LIB}/libluajit-5.1.a -ldl" \
  --with-openssl=../openssl-${OPENSSL_VERSION} \
  --with-http_ssl_module \
  --with-http_stub_status_module \
  --with-http_gzip_static_module \
  --with-zlib=../zlib-${ZLIB_VERSION} \
  --with-pcre \
  --with-debug \
  --with-file-aio \
  --with-http_v2_module \
  --with-stream \
  --with-stream_ssl_module \
  --without-http_geo_module \
  --without-http_map_module \
  --without-http_fastcgi_module \
  --without-http_uwsgi_module \
  --without-http_scgi_module \
  --without-http_memcached_module \
  --without-http_empty_gif_module \
  --without-mail_pop3_module \
  --without-mail_imap_module \
  --without-mail_smtp_module \
  --add-module=../lua-nginx-module-${LUA_NGINX_VERSION} \
  --add-module=../ngx_http_dyups_module \
  --add-module=../nginx_upstream_check_module\
  --add-module=../ngx_devel_kit-${DEVEL_KIT_VERSION} \
  --add-module=../ngx_http_consistent_hash\

sed -i "/CFLAGS/s/ \-O //g" objs/Makefile

make && make install



#rm -rf ${INSTALL_DIR}/html
rm -rf ${INSTALL_DIR}/fastcgi*
rm -rf ${INSTALL_DIR}/koi*
rm -rf ${INSTALL_DIR}/win*
rm -rf ${INSTALL_DIR}/scgi*
rm -rf ${INSTALL_DIR}/uwsgi*

if [ ! -d "${INSTALL_DIR}/run/tmp/" ] ; then
  mkdir ${INSTALL_DIR}/run/tmp/
fi
if [ ! -d "${INSTALL_DIR}/lua/" ] ; then
  mkdir ${INSTALL_DIR}/lua/
fi


echo -e "#user nobody; \n\
worker_processes auto; \n\n\
events { \n\
    worker_connections  1024; \n\
} \n\n\
http { \n\
    include       mime.types; \n\
    default_type  application/octet-stream; \n\n\
    lua_load_resty_core off; \n\
    sendfile        on; \n\
    keepalive_timeout  65; \n\n\n\
    #upstream backend { \n\
    #    consistent_hash \$remote_addr; \n\
    #    server 127.0.0.1:28080; \n\
    #} \n\n\n\
    server { \n\
        listen       80; \n\
        server_name  localhost; \n\
        location / { \n\
            root   html; \n\
            index  index.html index.htm; \n\
        } \n\
    } \n\n\n\
    server { \n\
        listen 8001; \n\
        location / { \n\
            dyups_interface; \n\
        } \n\
    } \n\
}" > ${INSTALL_DIR}/nginx.conf

4、配置示例

请将配置文件中的相关路径和域名替换成自己的路径及域名。

user nobody nogroup; 
worker_processes auto; 

events { 
    worker_connections  1024; 
} 

http { 
    include       mime.types; 
    default_type  application/octet-stream; 

    lua_load_resty_core off; 
    sendfile        on; 
    keepalive_timeout  65; 

    access_log /home/lijun/logs/nginx/access.log;
    error_log /home/lijun/logs/nginx/error.log;

    gzip on;

    limit_req_zone $binary_remote_addr zone=allips:10m rate=30r/m;
    http2_max_concurrent_streams 1024;

    gzip_vary on;
    gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
    proxy_cache_path /home/lijun/opt/nginx/run/tmp/cache levels=1:2 keys_zone=imgcache:100m inactive=1d max_size=10g;
    proxy_cache_valid 200 301 302 5m;

    upstream yee_dynamic_backend { 
        consistent_hash $remote_addr;
        server 127.0.0.1:58080;
    } 

    server {
        listen 0.0.0.0:443 ssl http2;
        server_name yee.im;
        server_tokens off;

        ssl_certificate /home/lijun/.ssl/yee.im/yee.im.cer;
        ssl_certificate_key /home/lijun/.ssl/yee.im/yee.im.key;
        ssl_protocols      TLSv1.2 TLSv1.3;

        if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})") {
                set $year $1;
                set $month $2;
                set $day $3;
        }
        access_log /home/lijun/logs/yee.im/access-$year-$month-$day.log;

        client_max_body_size 50m;
        client_body_buffer_size 256k;
        client_header_buffer_size 512k;
        large_client_header_buffers 4 512k;

        location / {
            proxy_ignore_client_abort on;
            proxy_set_header Host $host;
            proxy_set_header Scheme $scheme;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto https;
            proxy_set_header X-Forwarded-Port  443;

            send_timeout 600;
            proxy_connect_timeout 600;
            proxy_send_timeout 600;
            proxy_read_timeout 600;
            proxy_buffer_size 256k;
            proxy_buffers 4 256k;
            proxy_busy_buffers_size 256k;
            proxy_next_upstream error timeout invalid_header http_500 http_503 http_404;
            proxy_max_temp_file_size 128m;

            proxy_redirect off ;
            set $yee_dynamic_ups yee_dynamic_backend;
            proxy_pass    http://$yee_dynamic_ups;
        }
    }


    server { 
        listen 8001; 
        location / {
            allow 127.0.0.1; 
            deny all;
            dyups_interface; 
        } 
    } 
}

注意 set $yee_dynamic_ups yee_dynamic_backend; 不要遗漏。

consistent_hash $remote_addr; 基于客户端ip的一致性hash负载策略。

5、操作示例

查看所有的upstream和服务器信息

curl http://127.0.0.1:8001/detail

查看所有的upstream

curl http://127.0.0.1:8001/list

根据upstream名称查看服务器信息

curl http://127.0.0.1:8001/upstream/[name]

更新upstream

curl -d "server 127.0.0.1:58080; " http://127.0.0.1:8001/upstream/[name]

删除upstream

curl -i -X DELETE http://127.0.0.1:8001/upstream/[name]