diff --git a/10/Dockerfile b/10/Dockerfile index b4b1297..3b1d867 100644 --- a/10/Dockerfile +++ b/10/Dockerfile @@ -1,32 +1,85 @@ FROM colovu/ubuntu:18.04 -ENV PG_MAJOR=10 \ - PG_VERSION="10.12*" \ - GPG_KEY='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8' \ - PGDATA=/srv/data/postgresql \ - PATH=$PATH:/usr/lib/postgresql/10/bin +ARG app_ver=10 +ARG LOCAL_SERVER= + +ENV APP_NAME=postgresql \ + APP_EXEC=postgres \ + APP_USER=postgres \ + APP_GROUP=postgres \ + APP_VERSION=${app_ver} + +ENV APP_BASE_DIR=/usr/lib/${APP_NAME}/${APP_VERSION} \ + APP_DEF_DIR=/etc/${APP_NAME} \ + APP_CONF_DIR=/srv/conf/${APP_NAME} \ + APP_DATA_DIR=/srv/data/${APP_NAME} \ + APP_DATA_LOG_DIR=/srv/datalog/${APP_NAME} \ + APP_CACHE_DIR=/var/cache/${APP_NAME} \ + APP_RUN_DIR=/var/run/${APP_NAME} \ + APP_LOG_DIR=/var/log/${APP_NAME} \ + APP_CERT_DIR=/srv/cert/${APP_NAME} \ + APP_WWW_DIR=/srv/www + +# PGDATA 用于指定默认配置文件路径 +ENV PG_MAJOR=${APP_VERSION} \ + PGDATA=${APP_DATA_DIR}/${APP_VERSION} \ + PATH="${APP_BASE_DIR}/bin:${PATH}" LABEL \ - "Version"="v${PG_MAJOR}" \ - "Description"="Docker image for PostgreSQL ${PG_MAJOR} based on Ubuntu 18.04." \ - "Dockerfile"="https://github.com/colovu/docker-postgres" \ + "Version"="v${APP_VERSION}" \ + "Description"="Docker image for PostgreSQL ${APP_VERSION}." \ + "Dockerfile"="https://github.com/colovu/docker-${APP_NAME}" \ "Vendor"="Endial Fang (endial@126.com)" +COPY prebuilds / + +# 安装 locales 并修改默认编码 RUN set -eux; \ -# 确保程序使用静默安装，而非交互模式 +# 设置程序使用静默安装，而非交互模式；类似tzdata等程序需要使用静默安装 export DEBIAN_FRONTEND=noninteractive; \ - groupadd -r postgres; \ - useradd -r -g postgres -s /usr/sbin/nologin -d /var/lib/postgresql postgres; \ \ - mkdir -p /var/lib/postgresql /srv/conf/postgresql ${PGDATA} /var/log/postgresql /var/run/postgresql; \ + apt-get update; \ + apt-get install -y --no-install-recommends locales; \ + localedef -c -i en_US -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8; \ + echo 'en_GB.UTF-8 UTF-8\nen_US.UTF-8 UTF-8' >> /etc/locale.gen && locale-gen; \ + update-locale LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 LC_MESSAGES=POSIX && dpkg-reconfigure locales; + +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 + +RUN set -eux; \ +# 设置程序使用静默安装，而非交互模式；类似tzdata等程序需要使用静默安装 + export DEBIAN_FRONTEND=noninteractive; \ \ +# 设置入口脚本的可执行权限 + chmod +x /usr/local/bin/entrypoint.sh; \ + \ +# 为应用创建对应的组、用户、相关目录 + APP_DIRS="${APP_DEF_DIR:-} ${APP_CONF_DIR:-} ${APP_DATA_DIR:-} ${APP_CACHE_DIR:-} ${APP_RUN_DIR:-} ${APP_LOG_DIR:-} ${APP_CERT_DIR:-} ${APP_WWW_DIR:-} ${APP_DATA_LOG_DIR:-}"; \ + groupadd -r ${APP_GROUP}; \ + useradd -r -g ${APP_GROUP} -s /usr/sbin/nologin ${APP_USER}; \ + mkdir -p ${APP_DIRS}; \ + \ +# 应用软件包及依赖 + appDeps=" \ + postgresql-${PG_MAJOR} \ + postgresql-common \ + libnss-wrapper \ + xz-utils \ + tzdata \ + "; \ + \ +# 安装临时使用的软件包，在使用完后会进行删除 fetchDeps=" \ dirmngr \ gnupg \ "; \ - apt update; \ - apt install -y --no-install-recommends ${fetchDeps}; \ + savedAptMark="$(apt-mark showmanual) ${appDeps}"; \ + apt-get update; \ + apt-get install -y --no-install-recommends ${fetchDeps}; \ \ + GPG_KEY='B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8'; \ export GNUPGHOME="$(mktemp -d)"; \ gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "${GPG_KEY}"|| \ gpg --batch --keyserver pgp.mit.edu --recv-keys "$GPG_KEY" || \ @@ -39,40 +92,46 @@ RUN set -eux; \ \ echo "deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main ${PG_MAJOR}" >> /etc/apt/sources.list; \ echo "deb-src http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main ${PG_MAJOR}" >> /etc/apt/sources.list; \ - apt update; \ + apt-get update; \ + apt-get install -y --no-install-recommends ${appDeps}; \ \ - apt install -y --no-install-recommends \ - postgresql-${PG_MAJOR}=${PG_VERSION} \ - postgresql-common \ - libnss-wrapper \ - xz-utils \ - tzdata \ - ; \ - \ -# reconfigure tzdata for China +# 为中国区使用重新配置tzdata信息 ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime; \ dpkg-reconfigure -f noninteractive tzdata; \ \ - chown -Rf postgres:postgres /var/lib/postgresql /srv/conf/postgresql ${PGDATA} /var/log/postgresql /var/run/postgresql; \ -# this 777 will be replaced by 700 or 755 at runtime (allows semi-arbitrary "--user" values) - chmod 777 /var/lib/postgresql /srv/conf/postgresql ${PGDATA} /var/log/postgresql /var/run/postgresql; \ +# 检测是否存在overrides脚本文件，如果存在，执行 + { [ ! -e "/usr/local/overrides/overrides-${APP_VERSION}.sh" ] || /bin/bash "/usr/local/overrides/overrides-${APP_VERSION}.sh"; }; \ \ - apt purge -y --auto-remove ${fetchDeps}; \ - apt autoclean -y; \ +# 设置临时目录的权限信息，设置为777是为了保证后续使用`--user`或`gosu`时，可以更改目录对应的用户属性信息；运行时会被更改为700或755 + chown -Rf ${APP_USER}:${APP_GROUP} ${APP_DIRS}; \ + chmod 777 ${APP_DIRS}; \ + \ +# 查找新安装的应用及应用依赖软件包，并标识为'manual'，防止后续自动清理时被删除 + apt-mark auto '.*' > /dev/null; \ + { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark; }; \ + find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual; \ + \ +# 删除安装的临时依赖软件包，清理缓存 + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false ${fetchDeps}; \ + apt-get autoclean -y; \ rm -rf /var/lib/apt/lists/*; \ \ -# make the sample config easier to munge (and "correct by default") - dpkg-divert --add --rename --divert "/usr/share/postgresql/postgresql.conf.sample.dpkg" "/usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample"; \ - cp -v /usr/share/postgresql/postgresql.conf.sample.dpkg /usr/share/postgresql/postgresql.conf.sample; \ - ln -sv ../postgresql.conf.sample "/usr/share/postgresql/$PG_MAJOR/"; - - -COPY ./entrypoint.sh /usr/local/bin/ +# 验证安装的软件是否可以正常运行，常规情况下放置在命令行的最后 + gosu ${APP_USER} postgres --version; VOLUME ["/srv/conf", "/srv/data", "/var/log", "/var/run"] +# 默认使用gosu切换为新建用户启动，必须保证端口在1024之上 EXPOSE 5432 +# 容器初始化命令，默认存放在：/usr/local/bin/entrypoint.sh ENTRYPOINT ["entrypoint.sh"] -CMD ["postgres"] +# 应用程序的服务命令，必须使用非守护进程方式运行 +CMD ["postgres", "--config-file=/srv/conf/postgresql/10/main/postgresql.conf"] diff --git a/10/entrypoint.sh b/10/entrypoint.sh deleted file mode 100755 index 121be57..0000000 --- a/10/entrypoint.sh +++ /dev/null @@ -1,348 +0,0 @@ -#!/bin/bash -# docker entrypoint script - -set -Eeo pipefail - -LOG_RAW() { - local type="$1"; shift - printf '%s [%s] Entrypoint: %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$type" "$*" -} -LOG_I() { - LOG_RAW Note "$@" -} -LOG_W() { - LOG_RAW Warn "$@" >&2 -} -LOG_E() { - LOG_RAW Error "$@" >&2 - exit 1 -} - -LOG_I "Initial container for PostgreSQL" - -# allow the container to be started with `--user` or `-u` -# if [ "$1" = 'app-name' -a "$(id -u)" = '0' ]; then -# echo "[i] Restart container with user: user-name" -# echo "" -# exec gosu user-name "$0" "$@" -# fi - -# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) - -# 检测"_FILE"文件，并从文件中读取信息作为参数值；环境变量不允许 VAR 与 VAR_FILE 方式并存 -# -# usage: file_env VAR [DEFAULT] -# ie: file_env 'XYZ_DB_PASSWORD' 'example' -file_env() { - local var="$1" - local fileVar="${var}_FILE" - local def="${2:-}" - if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then - echo >&2 "error: both $var and $fileVar are set (but are exclusive)" - exit 1 - fi - local val="$def" - if [ "${!var:-}" ]; then - val="${!var}" - elif [ "${!fileVar:-}" ]; then - val="$(< "${!fileVar}")" - fi - export "$var"="$val" - unset "$fileVar" -} - -# check to see if this file is being run or sourced from another script -_is_sourced() { - # https://unix.stackexchange.com/a/215279 - [ "${#FUNCNAME[@]}" -ge 2 ] \ - && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ - && [ "${FUNCNAME[1]}" = 'source' ] -} - -# 使用root用户运行时，创建默认的数据库存储目录，并修改对应目录所属用户为"postgres" -docker_create_db_directories() { - local user; user="$(id -u)" - - LOG_I "Check directories used by postgres" - mkdir -p "/var/log/postgresql" - mkdir -p "/var/run/postgresql" - mkdir -p "${PGDATA}" - - mkdir -p "/srv/conf/postgresql/initdb.d" - [ ! -e /srv/conf/postgresql/postgresql.conf ] && cp -rf /usr/share/postgresql/postgresql.conf.sample /srv/conf/postgresql/postgresql.conf - - mkdir -p "/srv/conf/postgresql-common" - [ ! -e /srv/conf/postgresql-common/createcluster.conf ] && cp -rf /etc/postgresql-common/createcluster.conf /srv/conf/postgresql-common/createcluster.conf - - # 创建数据库日志存储目录，修改相应目录的所属用户信息 - if [ -n "$POSTGRES_INITDB_WALDIR" ]; then - mkdir -p "$POSTGRES_INITDB_WALDIR" - if [ "$user" = '0' ]; then - find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + - fi - chmod 700 "$POSTGRES_INITDB_WALDIR" - fi - - # 允许容器使用`--user`参数启动，修改相应目录的所属用户信息 - if [ "$user" = '0' ]; then - find "${PGDATA}" \! -user postgres -exec chown postgres '{}' + - find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + - find /var/log/postgresql \! -user postgres -exec chown postgres '{}' + - find /srv/conf/postgresql \! -user postgres -exec chown postgres '{}' + - find /srv/conf/postgresql-common \! -user postgres -exec chown postgres '{}' + - chmod 0755 /var/log/postgresql /var/run/postgresql /srv/conf/postgresql /srv/conf/postgresql-common - chmod 0700 ${PGDATA} - # 解决使用gosu后，nginx: [emerg] open() "/dev/stdout" failed (13: Permission denied) - chmod 0622 /dev/stdout /dev/stderr - fi -} - -# 针对 ${PGDATA} 目录为空时，使用'initdb'初始化数据目录；同时创建 POSTGRES_USER 定义的同名数据库用户 -# 用户需要传给`initdb`的参数，可通过环境变量 POSTGRES_INITDB_ARGS 传输，或直接使用命令行参数传输到当前函数 -# `initdb`会自动创建以下数据库："postgres", "template0", "template1" -docker_init_database_dir() { - # "initdb" 需要当前用户UID在 "/etc/passwd" 中存在，因此，如果需要时， 我们使用"nss_wrapper"虚拟相关用户 - if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then - export LD_PRELOAD='/usr/lib/libnss_wrapper.so' - export NSS_WRAPPER_PASSWD="$(mktemp)" - export NSS_WRAPPER_GROUP="$(mktemp)" - echo "postgres:x:$(id -u):$(id -g):PostgreSQL:${PGDATA}:/bin/false" > "$NSS_WRAPPER_PASSWD" - echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" - fi - - if [ -n "$POSTGRES_INITDB_WALDIR" ]; then - set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" - fi - - eval 'initdb --username="$POSTGRES_USER" --pwfile=<(echo "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' - - # unset/cleanup "nss_wrapper" bits - if [ "${LD_PRELOAD:-}" = '/usr/lib/libnss_wrapper.so' ]; then - rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" - unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP - fi -} - -# 如果 POSTGRES_PASSWORD 超过100字节，打印警告信息 -# 如果 POSTGRES_PASSWORD 为空且 POSTGRES_HOST_AUTH_METHOD 不为 'trust'，打印错误信息并退出 -# 如果 POSTGRES_HOST_AUTH_METHOD 设置为 'trust'，打印警告信息 -docker_verify_minimum_env() { - if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then - cat >&2 <<-'EOWARN' - WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. - This will not work if used via PGPASSWORD with "psql". - https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) - https://github.com/docker-library/postgres/issues/507 - EOWARN - fi - if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then - # The - option suppresses leading tabs but *not* spaces. :) - cat >&2 <<-'EOE' - Error: Database is uninitialized and superuser password is not specified. - You must specify POSTGRES_PASSWORD to a non-empty value for the - superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". - You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all - connections without a password. This is *not* recommended. - See PostgreSQL documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - EOE - exit 1 - fi - if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then - cat >&2 <<-'EOWARN' - ******************************************************************************** - WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow - anyone with access to the Postgres port to access your database without - a password, even if POSTGRES_PASSWORD is set. See PostgreSQL - documentation about "trust": - https://www.postgresql.org/docs/current/auth-trust.html - In Docker's default configuration, this is effectively any other - container on the same system. - It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace - it with "-e POSTGRES_PASSWORD=password" instead to set a password in - "docker run". - ******************************************************************************** - EOWARN - fi -} - -# usage: docker_process_init_files [file [file [...]]] -# ie: docker_process_init_files /always-initdb.d/* -# process initializer files, based on file extensions and permissions -docker_process_init_files() { - # psql here for backwards compatiblilty "${psql[@]}" - psql=( docker_process_sql ) - - echo - local f - for f; do - case "$f" in - *.sh) - # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 - # https://github.com/docker-library/postgres/pull/452 - if [ -x "$f" ]; then - echo "$0: running $f" - "$f" - else - echo "$0: sourcing $f" - . "$f" - fi - ;; - *.sql) echo "$0: running $f"; docker_process_sql -f "$f"; echo ;; - *.sql.gz) echo "$0: running $f"; gunzip -c "$f" | docker_process_sql; echo ;; - *.sql.xz) echo "$0: running $f"; xzcat "$f" | docker_process_sql; echo ;; - *) echo "$0: ignoring $f" ;; - esac - echo - done -} - -# Execute sql script, passed via stdin (or -f flag of pqsl) -# usage: docker_process_sql [psql-cli-args] -# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' -# ie: docker_process_sql -f my-file.sql -# ie: docker_process_sql > "${PGDATA}/pg_hba.conf" -} - -# start socket-only postgresql server for setting up or running scripts -# all arguments will be passed along as arguments to `postgres` (via pg_ctl) -docker_temp_server_start() { - if [ "$1" = 'postgres' ]; then - shift - fi - - # internal start of server in order to allow setup using psql client - # does not listen on external TCP/IP and waits until start finishes - set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" - - PGUSER="${PGUSER:-$POSTGRES_USER}" \ - pg_ctl -D "${PGDATA}" \ - -o "$(printf '%q ' "$@")" \ - -w start -} - -# stop postgresql server after done setting up user and running scripts -docker_temp_server_stop() { - PGUSER="${PGUSER:-postgres}" \ - pg_ctl -D "${PGDATA}" -m fast -w stop -} - -# 检测可能导致postgres执行后直接退出的命令，如"--help"；如果存在，直接返回 0 -_pg_want_help() { - local arg - for arg; do - case "$arg" in - -'?'|--help|--describe-config|-V|--version) - return 0 - ;; - esac - done - return 1 -} - -_main() { - # 如果命令行参数是以配置参数("-")开始，修改执行命令，确保使用postgres命令启动服务器 - if [ "${1:0:1}" = '-' ]; then - set -- postgres "$@" - fi - - # 命令行参数以postgres起始，且不包含直接返回的命令(如：-V、--version、--help)时，执行初始化操作 - if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then - docker_setup_env - - # 以root用户运行时，设置数据存储目录与权限；设置完成后，会使用gosu重新以"postgres"用户运行当前脚本 - docker_create_db_directories - if [ "$(id -u)" = '0' ]; then - echo "[i] Restart container with user: postgres" - echo "" - exec gosu postgres "$0" "$@" - fi - - # 检测数据库存储目录是否为空；如果为空，进行初始化操作 - if [ -z "$DATABASE_ALREADY_EXISTS" ]; then - docker_verify_minimum_env - - # 检测目录权限，防止初始化失败 - ls /srv/conf/postgresql/initdb.d/ > /dev/null - - docker_init_database_dir - pg_setup_hba_conf - - # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless - # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS - export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" - docker_temp_server_start "$@" - - docker_setup_db - docker_process_init_files /srv/conf/postgresql/initdb.d/* - - docker_temp_server_stop - unset PGPASSWORD - - echo - echo "[i] PostgreSQL init process complete; ready for start up." - echo - else - echo - echo "[i] PostgreSQL Database directory appears to contain a database; Skipping initialization" - echo - fi - - echo "[i] Start Application." - fi - - # 执行命令行 - exec "$@" -} - -if ! _is_sourced; then - _main "$@" -fi diff --git a/10/prebuilds/usr/local/bin/appcommon.sh b/10/prebuilds/usr/local/bin/appcommon.sh new file mode 100644 index 0000000..363d8c7 --- /dev/null +++ b/10/prebuilds/usr/local/bin/appcommon.sh @@ -0,0 +1,761 @@ +#!/bin/bash +# +# 应用通用业务处理函数 + +# 加载依赖脚本 +. /usr/local/scripts/liblog.sh +. /usr/local/scripts/libfile.sh +. /usr/local/scripts/libfs.sh +. /usr/local/scripts/libos.sh +. /usr/local/scripts/libcommon.sh +. /usr/local/scripts/libservice.sh +. /usr/local/scripts/libvalidations.sh + +# 函数列表 + +# 配置 libnss_wrapper 以使得 PostgreSQL 命令可以以任意用户身份执行 +# 全局变量: +# PG_* +postgresql_enable_nss_wrapper() { + if ! getent passwd "$(id -u)" &> /dev/null && [ -e /usr/lib/libnss_wrapper.so ]; then + export LD_PRELOAD='/usr/lib/libnss_wrapper.so' + export NSS_WRAPPER_PASSWD="$(mktemp)" + export NSS_WRAPPER_GROUP="$(mktemp)" + echo "postgres:x:$(id -u):$(id -g):PostgreSQL:${PG_DATA_DIR}:/bin/false" > "$NSS_WRAPPER_PASSWD" + echo "postgres:x:$(id -g):" > "$NSS_WRAPPER_GROUP" + fi +} + +# 加载应用使用的环境变量初始值，该函数在相关脚本中以eval方式调用 +# 全局变量: +# ENV_* : 容器使用的全局变量 +# PG_* : 应用配置文件使用的全局变量，变量名根据配置项定义 +# 返回值: +# 可以被 'eval' 使用的序列化输出 +docker_app_env() { + # 以下变量已经在创建镜像时定义，可直接使用 + # APP_NAME、APP_EXEC、APP_USER、APP_GROUP、APP_VERSION + # APP_BASE_DIR、APP_DEF_DIR、APP_CONF_DIR、APP_CERT_DIR、APP_DATA_DIR、APP_DATA_LOG_DIR、APP_CACHE_DIR、APP_RUN_DIR、APP_LOG_DIR + cat <<"EOF" +# Debug log message +export ENV_DEBUG=${ENV_DEBUG:-false} + +# Paths +export PG_BASE_DIR="${PG_BASE_DIR:-${APP_BASE_DIR}}" +export PG_DATA_DIR="${PG_DATA_DIR:-${APP_DATA_DIR}/${PG_MAJOR}}" +export PG_DATALOG_DIR="${PG_DATALOG_DIR:-${APP_DATA_LOG_DIR}}" +export PG_CONF_DIR="${PG_CONF_DIR:-${APP_CONF_DIR}}" +export PG_LOG_DIR="${PG_LOG_DIR:-${APP_LOG_DIR}}" +export PG_BIN_DIR="${PG_BIN_DIR:-${PG_BASE_DIR}/bin}" +export PG_CONF_FILE="${PG_CONF_DIR}/${PG_MAJOR}/main/postgresql.conf" +export PG_HBA_FILE="${PG_CONF_DIR}/${PG_MAJOR}/main/pg_hba.conf" +export PG_IDENT_FILE="${PG_CONF_DIR}/${PG_MAJOR}/main/pg_ident.conf" +export PG_RECOVERY_FILE="${PG_DATA_DIR}/recovery.conf" +export PG_PID_FILE="${APP_RUN_DIR}/postgresql.pid" +export PG_LOG_FILE="${PG_LOG_DIR}/postgresql.log" + + +# Users +export PG_DAEMON_USER="${PG_DAEMON_USER:-${APP_USER}}" +export PG_DAEMON_GROUP="${PG_DAEMON_GROUP:-${APP_GROUP}}" + +# Cluster configuration +export PG_CLUSTER_APP_NAME=${PG_CLUSTER_APP_NAME:-walreceiver} +export PG_REPLICATION_MODE="${PG_REPLICATION_MODE:-master}" + +export PG_MASTER_HOST="${PG_MASTER_HOST:-}" +export PG_MASTER_PORT_NUMBER="${PG_MASTER_PORT_NUMBER:-5432}" +export PG_NUM_SYNCHRONOUS_REPLICAS="${PG_NUM_SYNCHRONOUS_REPLICAS:-0}" +export PG_REPLICATION_USER="${PG_REPLICATION_USER:-}" +export PG_REPLICATION_PASSWORD="${PG_REPLICATION_PASSWORD:-}" + +export PG_SYNCHRONOUS_COMMIT_MODE="${PG_SYNCHRONOUS_COMMIT_MODE:-on}" +export PG_FSYNC="${PG_FSYNC:-on}" + +# PostgreSQL settings +export PG_INIT_MAX_TIMEOUT=${PG_INIT_MAX_TIMEOUT:-60} +export PG_INITDB_ARGS="${PG_INITDB_ARGS:-}" +export PG_INITDB_WAL_DIR="${PG_INITDB_WAL_DIR:-${PG_DATALOG_DIR}}" +export PG_PORT_NUMBER="${PG_PORT_NUMBER:-5432}" + +# PostgreSQL TLS Settings + +# PostgreSQL LDAP Settings +export PG_ENABLE_LDAP="${PG_ENABLE_LDAP:-no}" +export PG_LDAP_URL="${PG_LDAP_URL:-}" +export PG_LDAP_PREFIX="${PG_LDAP_PREFIX:-}" +export PG_LDAP_SUFFIX="${PG_LDAP_SUFFIX:-}" +export PG_LDAP_SERVER="${PG_LDAP_SERVER:-}" +export PG_LDAP_PORT="${PG_LDAP_PORT:-}" +export PG_LDAP_SCHEME="${PG_LDAP_SCHEME:-}" +export PG_LDAP_TLS="${PG_LDAP_TLS:-}" +export PG_LDAP_BASE_DN="${PG_LDAP_BASE_DN:-}" +export PG_LDAP_BIND_DN="${PG_LDAP_BIND_DN:-}" +export PG_LDAP_BIND_PASSWORD="${PG_LDAP_BIND_PASSWORD:-}" +export PG_LDAP_SEARCH_ATTR="${PG_LDAP_SEARCH_ATTR:-}" +export PG_LDAP_SEARCH_FILTER="${PG_LDAP_SEARCH_FILTER:-}" + +# Authentication +export PG_ALLOW_EMPTY_PASSWORD="${PG_ALLOW_EMPTY_PASSWORD:-no}" +export PG_USERNAME="${PG_USERNAME:-postgres}" +export PG_PASSWORD="${PG_PASSWORD:-}" +export PG_DATABASE="${PG_DATABASE:-postgres}" + +export PG_INITSCRIPTS_USERNAME="${PG_INITSCRIPTS_USERNAME:-${PG_USERNAME}}" +export PG_INITSCRIPTS_PASSWORD="${PG_INITSCRIPTS_PASSWORD:-${PG_PASSWORD}}" +EOF + + if [[ -f "${PG_POSTGRES_PASSWORD_FILE:-}" ]]; then + cat <<"EOF" +export PG_POSTGRES_PASSWORD="$(< "${PG_POSTGRES_PASSWORD_FILE}")" +EOF + else + cat <<"EOF" +export PG_POSTGRES_PASSWORD="${PG_POSTGRES_PASSWORD:-}" +EOF + fi + + if [[ -f "${PG_PASSWORD_FILE:-}" ]]; then + cat <<"EOF" +export PG_PASSWORD="$(< "${PG_PASSWORD_FILE}")" +EOF + fi + + if [[ -f "${PG_REPLICATION_PASSWORD_FILE:-}" ]]; then + cat <<"EOF" +export PG_REPLICATION_PASSWORD="$(< "${PG_REPLICATION_PASSWORD_FILE}")" +EOF + fi +} + +# 将变量配置更新至配置文件 +# 参数: +# $1 - 文件 +# $2 - 变量 +# $3 - 值（列表） +postgresql_common_conf_set() { + local file="${1:?missing file}" + local key="${2:?missing key}" + shift + shift + local values=("$@") + + if [[ "${#values[@]}" -eq 0 ]]; then + LOG_E "missing value" + return 1 + elif [[ "${#values[@]}" -ne 1 ]]; then + for i in "${!values[@]}"; do + postgresql_common_conf_set "$file" "${key[$i]}" "${values[$i]}" + done + else + value="${values[0]}" + # Check if the value was set before + if grep -q "^[#\\s]*$key\s*=.*" "$file"; then + # Update the existing key + replace_in_file "$file" "^[#\\s]*${key}\s*=.*" "${key} = \'${value}\'" false + else + # 增加一个新的配置项；如果在其他位置有类似操作，需要注意换行 + printf "%s = %s" "$key" "$value" >>"$file" + fi + fi +} + +# 更新 postgresql.conf 配置文件中指定变量值 +# 全局变量: +# PG_CONF_FILE +# 变量: +# $1 - 变量 +# $2 - 值（列表） +postgresql_conf_set() { + postgresql_common_conf_set "${PG_CONF_FILE}" "$@" +} + +# 更新 pg_hba.conf 配置文件中指定变量值 +# 全局变量: +# PG_HBA_FILE +# 变量: +# $1 - 变量 +# $2 - 值（列表） +postgresql_hba_set() { + postgresql_common_conf_set "${PG_HBA_FILE}" "$@" +} + +# 更新 pg_ident.conf 配置文件中指定变量值 +# 全局变量: +# PG_IDENT_FILE +# 变量: +# $1 - 变量 +# $2 - 值（列表） +postgresql_ident_set() { + postgresql_common_conf_set "${PG_IDENT_FILE}" "$@" +} + +# 更新 recover.conf 配置文件中指定变量值 +# 全局变量: +# PG_CONF_FILE +# 变量: +# $1 - 变量 +# $2 - 值（列表） +postgresql_recover_set() { + postgresql_common_conf_set "${PG_RECOVERY_FILE}" "$@" +} + +# 修改 PostgreSQL 应用指定配置文件的配置项 +# 全局变量: +# PG_* +# 参数: +# $1 - 配置项 +# $2 - 值 +# $3 - 配置文件 (默认值: $PG_CONF_FILE) +postgresql_set_property() { + local -r property="${1:?missing property}" + local -r value="${2:?missing value}" + local -r conf_file="${3:?missing config-file}" + local psql_conf + + replace_in_file "$conf_file" "^#*\s*${property}\s*=.*" "${property} = '${value}'" false +} + +# 检测用户参数信息是否满足条件; 针对部分权限过于开放情况，打印提示信息 +# 全局变量： +# PG_* +app_verify_minimum_env() { + local error_code=0 + LOG_D "Validating settings in PG_* env vars..." + + # Auxiliary functions + print_validation_error() { + LOG_E "$1" + error_code=1 + } + + empty_password_enabled_warn() { + LOG_W "You set the environment variable PG_ALLOW_EMPTY_PASSWORD=${PG_ALLOW_EMPTY_PASSWORD}. For safety reasons, do not use this flag in a production environment." + } + empty_password_error() { + print_validation_error "The $1 environment variable is empty or not set. Set the environment variable PG_ALLOW_EMPTY_PASSWORD=yes to allow the container to be started with blank passwords. This is recommended only for development." + } + if is_boolean_yes "$PG_ALLOW_EMPTY_PASSWORD"; then + empty_password_enabled_warn + else + if [[ -z "$PG_PASSWORD" ]]; then + empty_password_error "PG_PASSWORD" + fi + if (( ${#PG_PASSWORD} > 100 )); then + print_validation_error "The password cannot be longer than 100 characters. Set the environment variable PG_PASSWORD with a shorter value" + fi + if [[ -n "$PG_USERNAME" ]] && [[ -z "$PG_PASSWORD" ]]; then + empty_password_error "PG_PASSWORD" + fi + if [[ -n "$PG_USERNAME" ]] && [[ "$PG_USERNAME" != "postgres" ]] && [[ -n "$PG_PASSWORD" ]] && [[ -z "$PG_DATABASE" ]]; then + print_validation_error "In order to use a custom PostgreSQL user you need to set the environment variable PG_DATABASE as well" + fi + fi + + if [[ -n "$PG_REPLICATION_MODE" ]]; then + if [[ "$PG_REPLICATION_MODE" = "master" ]]; then + if (( PG_NUM_SYNCHRONOUS_REPLICAS < 0 )); then + print_validation_error "The number of synchronous replicas cannot be less than 0. Set the environment variable PG_NUM_SYNCHRONOUS_REPLICAS" + fi + elif [[ "$PG_REPLICATION_MODE" = "slave" ]]; then + if [[ -z "$PG_MASTER_HOST" ]]; then + print_validation_error "Slave replication mode chosen without setting the environment variable PG_MASTER_HOST. Use it to indicate where the Master node is running" + fi + if [[ -z "$PG_REPLICATION_USER" ]]; then + print_validation_error "Slave replication mode chosen without setting the environment variable PG_REPLICATION_USER. Make sure that the master also has this parameter set" + fi + else + print_validation_error "Invalid replication mode. Available options are 'master/slave'" + fi + # Common replication checks + if [[ -n "$PG_REPLICATION_USER" ]] && [[ -z "$PG_REPLICATION_PASSWORD" ]]; then + empty_password_error "PG_REPLICATION_PASSWORD" + fi + else + if is_boolean_yes "$PG_ALLOW_EMPTY_PASSWORD"; then + empty_password_enabled_warn + else + if [[ -z "$PG_PASSWORD" ]]; then + empty_password_error "PG_PASSWORD" + fi + if [[ -n "$PG_USERNAME" ]] && [[ -z "$PG_PASSWORD" ]]; then + empty_password_error "PG_PASSWORD" + fi + fi + fi + + if ! is_yes_no_value "$PG_ENABLE_LDAP"; then + empty_password_error "The values allowed for PG_ENABLE_LDAP are: yes or no" + fi + + if is_boolean_yes "$PG_ENABLE_LDAP" && [[ -n "$PG_LDAP_URL" ]] && [[ -n "$PG_LDAP_SERVER" ]]; then + empty_password_error "You can not set PG_LDAP_URL and PG_LDAP_SERVER at the same time. Check your LDAP configuration." + fi + + [[ "$error_code" -eq 0 ]] || exit "$error_code" +} + +# 初始化 pg_hba.conf 文件，增加 LDAP 配置；同时保留本地认证 +# 全局变量: +# PG_* +postgresql_ldap_auth_configuration() { + LOG_I "Generating LDAP authentication configuration" + local ldap_configuration="" + + if [[ -n "$PG_LDAP_URL" ]]; then + ldap_configuration="ldapurl=\"${PG_LDAP_URL}\"" + else + ldap_configuration="ldapserver=${PG_LDAP_SERVER}" + + [[ -n "$PG_LDAP_PREFIX" ]] && ldap_configuration+=" ldapprefix=\"${PG_LDAP_PREFIX}\"" + [[ -n "$PG_LDAP_SUFFIX" ]] && ldap_configuration+=" ldapsuffix=\"${PG_LDAP_SUFFIX}\"" + [[ -n "$PG_LDAP_PORT" ]] && ldap_configuration+=" ldapport=${PG_LDAP_PORT}" + [[ -n "$PG_LDAP_BASE_DN" ]] && ldap_configuration+=" ldapbasedn=\"${PG_LDAP_BASE_DN}\"" + [[ -n "$PG_LDAP_BIND_DN" ]] && ldap_configuration+=" ldapbinddn=\"${PG_LDAP_BIND_DN}\"" + [[ -n "$PG_LDAP_BIND_PASSWORD" ]] && ldap_configuration+=" ldapbindpasswd=${PG_LDAP_BIND_PASSWORD}" + [[ -n "$PG_LDAP_SEARCH_ATTR" ]] && ldap_configuration+=" ldapsearchattribute=${PG_LDAP_SEARCH_ATTR}" + [[ -n "$PG_LDAP_SEARCH_FILTER" ]] && ldap_configuration+=" ldapsearchfilter=\"${PG_LDAP_SEARCH_FILTER}\"" + [[ -n "$PG_LDAP_TLS" ]] && ldap_configuration+=" ldaptls=${PG_LDAP_TLS}" + [[ -n "$PG_LDAP_SCHEME" ]] && ldap_configuration+=" ldapscheme=${PG_LDAP_SCHEME}" + fi + + cat << EOF > "$PG_HBA_FILE" +local all all trust +host all postgres 0.0.0.0/0 trust +host all postgres ::/0 trust +host all all 0.0.0.0/0 ldap $ldap_configuration +host all all ::/0 ldap $ldap_configuration +EOF +} + +# 修改 pg_hba.conf 文件，增加主从复制从服务器认证许可 +# 全局变量: +# PG_* +postgresql_add_replication_to_pghba() { + local replication_auth="trust" + if [[ -n "$PG_REPLICATION_PASSWORD" ]]; then + replication_auth="md5" + fi + cat << EOF >> "$PG_HBA_FILE" +host replication all 0.0.0.0/0 ${replication_auth} +host replication all ::/0 ${replication_auth} +EOF +} + +# 初始化 pg_hba.conf 文件 +# 全局变量: +# PG_* +postgresql_password_auth_configuration() { + LOG_I "Generating local authentication configuration" + cat << EOF > "$PG_HBA_FILE" +local all all trust +host all all 0.0.0.0/0 trust +host all all ::/0 trust +EOF +} + +# 更改默认监听地址为 "*"，以对容器外提供服务；默认配置文件应当为仅监听 localhost(127.0.0.1) +postgresql_enable_remote_connections() { + postgresql_conf_set "listen_addresses" "*" +} + +# 以后台方式启动Zookeeper服务，并等待启动就绪 +# 全局变量: +# PG_* +# ENV_DEBUG +postgresql_start_server_bg() { + is_postgresql_running && return + + # -w wait until operation completes (default) + # -W don't wait until operation completes + # -D location of the database storage area + # -l write (or append) server log to FILENAME + # -o command line options to pass to postgres or initdb + local -r pg_ctl_flags=("-W" "-D" "$PG_DATA_DIR" "-l" "$PG_LOG_FILE" "-o" "--config-file=$PG_CONF_FILE --external_pid_file=$PG_PID_FILE --hba_file=$PG_HBA_FILE") + LOG_I "Starting PostgreSQL in background..." + local pg_ctl_cmd=() + if _is_run_as_root; then + pg_ctl_cmd+=("gosu" "$PG_DAEMON_USER") + fi + pg_ctl_cmd+=(pg_ctl) + if is_boolean_yes "${ENV_DEBUG}"; then + "${pg_ctl_cmd[@]}" "start" "${pg_ctl_flags[@]}" + else + "${pg_ctl_cmd[@]}" "start" "${pg_ctl_flags[@]}" >/dev/null 2>&1 + fi + + local -r pg_isready_args=("-h" "localhost" "-p" "${PG_PORT_NUMBER}" "-U" "postgres") + local counter=$PG_INIT_MAX_TIMEOUT + LOG_I "Starting check PostgreSQL is ready status..." + while ! pg_isready "${pg_isready_args[@]}" >/dev/null 2>&1; do + sleep 1 + counter=$((counter - 1 )) + if (( counter <= 0 )); then + LOG_E "PostgreSQL is not ready after $PG_INIT_MAX_TIMEOUT seconds" + exit 1 + fi + done + LOG_D "PostgreSQL is ready for service..." +} + +# 停止 PostgreSQL 后台服务 +# 全局变量: +# PG_PID_FILE +postgresql_stop_server() { + LOG_I "Stopping PostgreSQL..." + stop_service_using_pid "$PG_PID_FILE" +} + +# 检测 PostgreSQL 后台服务是否在运行中 +# 全局变量: +# PG_PID_FILE +# 返回值: +# 布尔值 +is_postgresql_running() { + local pid + pid="$(get_pid_from_file "$PG_PID_FILE")" + + if [[ -z "$pid" ]]; then + false + else + is_service_running "$pid" + fi +} + +# 使用运行中的 PostgreSQL 服务执行 SQL 操作 +# 全局变量: +# ENV_DEBUG +# PG_* +# 参数: +# $1 - 需要操作的数据库名 +# $2 - 操作使用的用户名 +# $3 - 操作用户密码 +# $4 - 主机 +# $5 - 端口 +# $6 - 扩展参数 (如: -tA) +postgresql_execute() { + local -r db="${1:-}" + local -r user="${2:-postgres}" + local -r pass="${3:-}" + local -r host="${4:-localhost}" + local -r port="${5:-${PG_PORT_NUMBER}}" + local -r opts="${6:-}" + + local args=( "-h" "$host" "-p" "$port" "-U" "$user" ) + local cmd=("psql") + [[ -n "$db" ]] && args+=( "-d" "$db" ) + [[ -n "$opts" ]] && args+=( "$opts" ) + LOG_D "Execute args: ${args[@]}" + if is_boolean_yes "${ENV_DEBUG}"; then + PGPASSWORD=$pass "${cmd[@]}" "${args[@]}" + else + PGPASSWORD=$pass "${cmd[@]}" "${args[@]}" >/dev/null 2>&1 + fi +} + +# 在重新启动容器时，删除标志文件及 postmaster PID 文件 (容器重新启动) +# 全局变量: +# PG_* +postgresql_clean_from_restart() { + local -r -a files=( + "$PG_DATA_DIR"/postmaster.pid + "$PG_DATA_DIR"/standby.signal + "$PG_DATA_DIR"/recovery.signal + "$PG_PID_FILE" + ) + + for file in "${files[@]}"; do + if [[ -f "$file" ]]; then + LOG_I "Cleaning stale $file file" + rm "$file" + fi + done +} + +# 生成初始 postgres.conf 配置 +# 全局变量: +# PG_* +postgresql_default_postgresql_config() { + LOG_I "Modify postgresql.conf with default values..." + postgresql_conf_set "wal_level" "hot_standby" + postgresql_conf_set "max_wal_size" "400MB" + postgresql_conf_set "max_wal_senders" "16" + postgresql_conf_set "wal_keep_segments" "12" + postgresql_conf_set "hot_standby" "on" + if (( PG_NUM_SYNCHRONOUS_REPLICAS > 0 )); then + postgresql_conf_set "synchronous_commit" "$PG_SYNCHRONOUS_COMMIT_MODE" + postgresql_conf_set "synchronous_standby_names" "${PG_NUM_SYNCHRONOUS_REPLICAS} (\"${PG_CLUSTER_APP_NAME}\")" + fi + postgresql_conf_set "fsync" "$PG_FSYNC" +} + +# 生成初始 pg_hba.conf 配置 +# 全局变量: +# PG_* +postgresql_default_hba_config() { + LOG_I "Modify pg_hba.conf with default values..." + + if is_boolean_yes "$PG_ENABLE_LDAP"; then + postgresql_ldap_auth_configuration + else + postgresql_password_auth_configuration + fi + + if [[ -n "$PG_PASSWORD" ]]; then + LOG_I "Configuring md5 encrypt" + postgresql_hba_set "trust" "md5" + fi + +} + +# 为 Slava 模式工作的节点创建 recovery.conf 文件 +# 全局变量: +# PG_* +postgresql_configure_recovery() { + LOG_I "Setting up streaming replication slave..." + if (( PG_MAJOR >= 12 )); then + # 版本为12以上时， Slave 节点配置保存在 postgresql.conf 文件中 + postgresql_conf_set "primary_conninfo" "host=${PG_MASTER_HOST} port=${PG_MASTER_PORT_NUMBER} user=${PG_REPLICATION_USER} password=${PG_REPLICATION_PASSWORD} application_name=${PG_CLUSTER_APP_NAME}" + postgresql_conf_set "promote_trigger_file" "/tmp/postgresql.trigger.${PG_MASTER_PORT_NUMBER}" + touch "$PG_DATA_DIR"/standby.signal + else + # 版本低于12时， Slave 节点配置保存在 recover.conf 文件中 + cp -f "/usr/share/postgresql/${PG_MAJOR}/recovery.conf.sample" "$PG_RECOVERY_FILE" + chmod 600 "$PG_RECOVERY_FILE" + postgresql_recover_set "standby_mode" "on" + postgresql_recover_set "primary_conninfo" "host=${PG_MASTER_HOST} port=${PG_MASTER_PORT_NUMBER} user=${PG_REPLICATION_USER} password=${PG_REPLICATION_PASSWORD} application_name=${PG_CLUSTER_APP_NAME}" + postgresql_recover_set "trigger_file" "/tmp/postgresql.trigger.${PG_MASTER_PORT_NUMBER}" + fi +} + +# 为默认的数据库用户 postgres 设置密码 +# 全局变量: +# PG_* +# 参数: +# $1 - 用户密码 +postgresql_alter_postgres_user() { + local -r escaped_password="${1//\'/\'\'}" + LOG_I "Changing password of postgres" + echo "ALTER ROLE postgres WITH PASSWORD '$escaped_password';" | postgresql_execute +} + +# 为数据库 $PG_DATABASE 创建管理员账户 +# 全局变量: +# PG_* +postgresql_create_admin_user() { + local -r escaped_password="${PG_PASSWORD//\'/\'\'}" + LOG_I "Creating user ${PG_USERNAME}" + echo "CREATE ROLE \"${PG_USERNAME}\" WITH LOGIN CREATEDB PASSWORD '${escaped_password}';" | postgresql_execute + + LOG_I "Granting access to \"${PG_USERNAME}\" to the database \"${PG_DATABASE}\"" + echo "GRANT ALL PRIVILEGES ON DATABASE \"${PG_DATABASE}\" TO \"${PG_USERNAME}\"\;" | postgresql_execute "" "postgres" "$PG_POSTGRES_PASSWORD" +} + +# 为 master-slave 复制模式创建用户 +# 全局变量: +# PG_* +postgresql_create_replication_user() { + local -r escaped_password="${PG_REPLICATION_PASSWORD//\'/\'\'}" + LOG_I "Creating replication user $PG_REPLICATION_USER" + echo "CREATE ROLE \"$PG_REPLICATION_USER\" REPLICATION LOGIN ENCRYPTED PASSWORD '$escaped_password'" | postgresql_execute +} + +# 创建用户自定义数据库 $PG_DATABASE +# 全局变量: +# PG_* +postgresql_create_custom_database() { + echo "CREATE DATABASE \"$PG_DATABASE\"" | postgresql_execute "" "postgres" "" "localhost" +} + +# 应用默认初始化操作 +# 执行完毕后，会在 ${APP_DATA_DIR} 目录中生成 .app_init_flag 及 .data_init_flag 文件 +docker_app_init() { + postgresql_clean_from_restart + LOG_D "Check init status of ${APP_NAME}..." + + # 检测配置文件是否存在 + if [[ ! -f "${APP_DATA_DIR}/.app_init_flag" ]]; then + LOG_I "No injected configuration file found, creating default config files..." + postgresql_default_postgresql_config + postgresql_default_hba_config + + if [[ "$PG_REPLICATION_MODE" = "master" ]]; then + [[ -n "$PG_REPLICATION_USER" ]] && postgresql_add_replication_to_pghba + else + postgresql_configure_recovery + fi + + echo "$(date '+%Y-%m-%d %H:%M:%S') : Init success." >> ${APP_DATA_DIR}/.app_init_flag + else + LOG_I "User injected custom configuration detected!" + fi + + if [[ ! -f "${APP_DATA_DIR}/.data_init_flag" ]]; then + LOG_I "Deploying PostgreSQL from scratch..." + if [[ "$PG_REPLICATION_MODE" = "master" ]]; then + postgresql_master_init_db + postgresql_start_server_bg + [[ "$PG_DATABASE" != "postgres" ]] && postgresql_create_custom_database + + # 为数据库授权；默认用户不为 postgres 时，需要创建管理员账户 + LOG_D "Set password for postgres user" + if [[ "$PG_USERNAME" = "postgres" ]]; then + postgresql_alter_postgres_user "$PG_PASSWORD" + else + if [[ -n "$PG_POSTGRES_PASSWORD" ]]; then + postgresql_alter_postgres_user "$PG_POSTGRES_PASSWORD" + fi + postgresql_create_admin_user + fi + [[ -n "$PG_REPLICATION_USER" ]] && postgresql_create_replication_user + else + postgresql_slave_init_db + fi + + echo "$(date '+%Y-%m-%d %H:%M:%S') : Init success." > ${APP_DATA_DIR}/.data_init_flag + else + LOG_I "Deploying PostgreSQL with persisted data..." + fi +} + +# 用户自定义的应用初始化操作，依次执行目录preinitdb.d中的初始化脚本 +# 执行完毕后，会在 ${APP_DATA_DIR} 目录中生成 .custom_preinit_flag 文件 +docker_custom_preinit() { + # 检测用户配置文件目录是否存在initdb.d文件夹，如果存在，尝试执行目录中的初始化脚本 + if [ -d "/srv/conf/${APP_NAME}/preinitdb.d" ]; then + # 检测数据存储目录是否存在已初始化标志文件；如果不存在，进行初始化操作 + if [ ! -f "${APP_DATA_DIR}/.custom_preinit_flag" ]; then + LOG_I "Process custom pre-init scripts from /srv/conf/${APP_NAME}/preinitdb.d..." + + # 检测目录权限，防止初始化失败 + ls "/srv/conf/${APP_NAME}/preinitdb.d/" > /dev/null + + docker_process_init_files /srv/conf/${APP_NAME}/preinitdb.d/* + + echo "$(date '+%Y-%m-%d %H:%M:%S') : Init success." > ${APP_DATA_DIR}/.custom_preinit_flag + LOG_I "Custom preinit for ${APP_NAME} complete." + else + LOG_I "Custom preinit for ${APP_NAME} already done before, skipping initialization." + fi + fi +} + +# 用户自定义的应用初始化操作，依次执行目录initdb.d中的初始化脚本 +# 执行完毕后，会在 ${APP_DATA_DIR} 目录中生成 .custom_init_flag 文件 +docker_custom_init() { + # 检测用户配置文件目录是否存在initdb.d文件夹，如果存在，尝试执行目录中的初始化脚本 + if [ -d "/srv/conf/${APP_NAME}/initdb.d" ]; then + # 检测数据存储目录是否存在已初始化标志文件；如果不存在，进行初始化操作 + if [[ -n $(find "/srv/conf/${APP_NAME}/initdb.d/" -type f -regex ".*\.\(sh\|sql\|sql.gz\)") ]] && [[ ! -f "${APP_DATA_DIR}/.custom_init_flag" ]]; then + LOG_I "Process custom init scripts from /srv/conf/${APP_NAME}/initdb.d..." + + postgresql_start_server_bg + find "/srv/conf/${APP_NAME}/initdb.d/" -type f -regex ".*\.\(sh\|sql\|sql.gz\)" | sort | while read -r f; do + case "$f" in + *.sh) + if [[ -x "$f" ]]; then + LOG_D "Executing $f"; "$f" + else + LOG_D "Sourcing $f"; . "$f" + fi + ;; + *.sql) LOG_D "Executing $f"; postgresql_execute "$PG_DATABASE" "$PG_INITSCRIPTS_USERNAME" "$PG_INITSCRIPTS_PASSWORD" < "$f";; + *.sql.gz) LOG_D "Executing $f"; gunzip -c "$f" | postgresql_execute "$PG_DATABASE" "$PG_INITSCRIPTS_USERNAME" "$PG_INITSCRIPTS_PASSWORD";; + *) LOG_D "Ignoring $f" ;; + esac + done + + echo "$(date '+%Y-%m-%d %H:%M:%S') : Init success." > ${APP_DATA_DIR}/.custom_init_flag + LOG_I "Custom init for ${APP_NAME} complete." + else + LOG_I "Custom init for ${APP_NAME} already done before, skipping initialization." + fi + fi + + # 停止初始化启动的 PostgreSQL 后台服务 + postgresql_stop_server + + # 删除第一次运行生成的日志文件 + rm -rf "$PG_LOG_FILE" + + # 绑定所有 IP ，启用远程访问 + postgresql_enable_remote_connections +} + +# 初始化 Master 节点数据库 +# 全局变量: +# PG_* +# 返回值: +# 布尔值 +postgresql_master_init_db() { + local envExtraFlags=() + local initdb_args=() + if [[ -n "${PG_INITDB_ARGS}" ]]; then + read -r -a envExtraFlags <<< "$PG_INITDB_ARGS" + initdb_args+=("${envExtraFlags[@]}") + fi + #initdb+=("-o" "--config-file=$PG_CONF_FILE --external_pid_file=$PG_PID_FILE --hba_file=$PG_HBA_FILE") + initdb_args+=("--waldir=$PG_INITDB_WAL_DIR") + + local initdb_cmd=() + if _is_run_as_root; then + initdb_cmd+=("gosu" "$PG_DAEMON_USER") + fi + initdb_cmd+=(initdb) + + LOG_I "Initializing PostgreSQL database" + + if [[ -n "${initdb_args[*]}" ]]; then + LOG_I "extra initdb arguments: ${initdb_args[*]}" + fi + + if is_boolean_yes "${ENV_DEBUG}"; then + "${initdb_cmd[@]}" -E UTF8 -D "$PG_DATA_DIR" -U "postgres" "${initdb_args[@]}" + else + "${initdb_cmd[@]}" -E UTF8 -D "$PG_DATA_DIR" -U "postgres" "${initdb_args[@]}" >/dev/null 2>&1 + fi +} + +# 初始化 Slave 节点数据库 +# 全局变量: +# PG_* +# 返回值: +# 布尔值 +postgresql_slave_init_db() { + LOG_I "Waiting for replication master to accept connections (${PG_INIT_MAX_TIMEOUT} timeout)..." + local -r check_args=("-U" "$PG_REPLICATION_USER" "-h" "$PG_MASTER_HOST" "-p" "$PG_MASTER_PORT_NUMBER" "-d" "postgres") + local check_cmd=() + if _is_run_as_root; then + check_cmd=("gosu" "$PG_DAEMON_USER") + fi + check_cmd+=(pg_isready) + local ready_counter=$PG_INIT_MAX_TIMEOUT + + while ! PGPASSWORD=$PG_REPLICATION_PASSWORD "${check_cmd[@]}" "${check_args[@]}";do + sleep 1 + ready_counter=$(( ready_counter - 1 )) + if (( ready_counter <= 0 )); then + LOG_E "PostgreSQL master is not ready after $PG_INIT_MAX_TIMEOUT seconds" + exit 1 + fi + done + + LOG_I "Replicating the initial database" + local -r backup_args=("-D" "$PG_DATA_DIR" "-U" "$PG_REPLICATION_USER" "-h" "$PG_MASTER_HOST" "-p" "$PG_MASTER_PORT_NUMBER" "-X" "stream" "-w" "-v" "-P") + local backup_cmd=() + if _is_run_as_root; then + backup_cmd+=("gosu" "$PG_DAEMON_USER") + fi + backup_cmd+=(pg_basebackup) + + local replication_counter=$PG_INIT_MAX_TIMEOUT + while ! PGPASSWORD=$PG_REPLICATION_PASSWORD "${backup_cmd[@]}" "${backup_args[@]}";do + LOG_D "Backup command failed. Sleeping and trying again" + sleep 1 + replication_counter=$(( replication_counter - 1 )) + if (( replication_counter <= 0 )); then + LOG_E "Slave replication failed after trying for $PG_INIT_MAX_TIMEOUT seconds" + exit 1 + fi + done +} diff --git a/10/prebuilds/usr/local/bin/entrypoint.sh b/10/prebuilds/usr/local/bin/entrypoint.sh new file mode 100644 index 0000000..4140ffa --- /dev/null +++ b/10/prebuilds/usr/local/bin/entrypoint.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# 容器入口脚本 + +set -o errexit +set -o nounset +set -o pipefail +# set -o xtrace # Uncomment this line for debugging purpose + +# 加载依赖脚本 +. /usr/local/scripts/liblog.sh +. /usr/local/scripts/libcommon.sh + +. /usr/local/bin/appcommon.sh + +APP_DIRS="${APP_DEF_DIR:-} ${APP_HOME_DIR:-} ${APP_CONF_DIR:-} ${APP_DATA_DIR:-} ${APP_CACHE_DIR:-} ${APP_RUN_DIR:-} ${APP_LOG_DIR:-} ${APP_CERT_DIR:-} ${APP_WWW_DIR:-} ${APP_DATA_LOG_DIR:-}"; \ + +# 加载环境变量, docker_app_env()函数在文件 app-common.sh 中定义 +eval "$(docker_app_env)" + +APP_DIRS="${APP_DIRS} ${PG_DATA_DIR}" + +# 打印镜像欢迎信息 +docker_print_welcome + +# 检测数据卷，创建默认的关联目录，并拷贝所必须的默认配置文件及初始化文件 +docker_ensure_dir_and_configs() { + local user_id; user_id="$(id -u)" + + for dir in ${APP_DIRS}; do + LOG_D "Check directory $dir" + ensure_dir_exists "$dir" + done + + # 检测指定文件是否在配置文件存储目录存在，如果不存在则拷贝（新挂载数据卷、手动删除都会导致不存在） + LOG_D "Check config files" + ensure_config_file_exist ${APP_DEF_DIR}/* +} + +_main() { + # 如果命令行参数是以配置参数("-")开始，修改执行命令，确保使用可执行应用命令启动服务器 + if [ "${1:0:1}" = '-' ]; then + set -- "${APP_EXEC}" "$@" + fi + + # 命令行参数以可执行应用命令起始，且不包含直接返回的命令(如：-V、--version、--help)时，执行初始化操作 + if [ "$1" = "${APP_EXEC}" ] && ! docker_command_help "$@"; then + # 检测 ENV_* PG_* 环境变量是否有效 + app_verify_minimum_env + + # 检测应用需要使用的目录是否存在，并设置相应用户权限 + docker_ensure_dir_and_configs + + # 以root用户运行时，会使用gosu重新以"APP_USER"用户运行当前脚本 + LOG_D "Check if run as root" + if _is_run_as_root; then + LOG_D "Change permissions when run as root" + + # 以root用户启动时，修改相应目录的所属用户信息为APP_USER，确保切换用户时，权限正常 + for dir in ${APP_DIRS}; do + LOG_D "Change ownership and permissions of $dir" + configure_permissions_ownership "$dir" -f 755 -d 755 -u "${APP_USER}" + done + + # 解决 PostgreSQL 目录权限过于开放，无法初始化问题：FATAL: data directory "/srv/data/postgresql" has group or world access + LOG_D "Lack of permissions on data directory: ${PG_DATA_DIR}" + chmod -R 0700 ${PG_DATA_DIR} ${APP_DATA_DIR} + + # 解决使用gosu后，nginx: [emerg] open() "/dev/stdout" failed (13: Permission denied) + LOG_D "Change permissions of stdout/stderr to 0622" + chmod 0622 /dev/stdout /dev/stderr + + LOG_I "Restart container with default user: ${APP_USER}" + exec gosu "${APP_USER}" "$0" "$@" + fi + + # 执行预初始化操作 + docker_custom_preinit + + # 执行应用初始化操作 + docker_app_init + + # 执行用户自定义初始化脚本 + docker_custom_init + fi + + LOG_I "Start container with: $@" + # 执行命令行 + exec "$@" +} + +# 脚本入口命令 +if ! _is_sourced; then + _main "$@" +fi diff --git a/10/prebuilds/usr/local/license/LICENSE b/10/prebuilds/usr/local/license/LICENSE new file mode 100644 index 0000000..80e3bb7 --- /dev/null +++ b/10/prebuilds/usr/local/license/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Endial Fang (endial@126.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/10/prebuilds/usr/local/overrides/overrides-10.sh b/10/prebuilds/usr/local/overrides/overrides-10.sh new file mode 100644 index 0000000..9cb0957 --- /dev/null +++ b/10/prebuilds/usr/local/overrides/overrides-10.sh @@ -0,0 +1,29 @@ +#!/bin/bash -e + +POSTGRESQL_CONF="${APP_DEF_DIR}/${PG_MAJOR}/main/postgresql.conf" + +# 在安装完应用后，使用该脚本修改默认配置文件中部分配置项 +# 如果相应的配置项已经定义整体环境变量，则不需要在这里修改 +echo "Process overrides for default configs..." +#sed -i -E 's/^listeners=/d' "$KAFKA_HOME/config/server.properties" + +# 设置默认监听地址为 localhost ，防止初始化操作期间外部链接，在容器初始化完成后修改为监听所有地址 +sed -i -E "s/^#listen_addresses .*/listen_addresses = \'localhost\'/g" ${POSTGRESQL_CONF} + +sed -i -E "s/^data_directory .*/data_directory = \'\/srv\/data\/postgresql\/${PG_MAJOR}\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^hba_file .*/hba_file = \'\/srv\/conf\/postgresql\/${PG_MAJOR}\/main\/pg_hba.conf\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^ident_file .*/ident_file = \'\/srv\/conf\/postgresql\/${PG_MAJOR}\/main\/pg_ident.conf\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#external_pid_file .*/external_pid_file = \'\/var\/run\/postgresql\/postgresql.pid\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^max_connections .*/max_connections = 2000/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#password_encryption .*/password_encryption = md5/g" ${POSTGRESQL_CONF} + +sed -i -E "s/^#log_destination .*/log_destination = \'stderr\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#logging_collector .*/logging_collector = on/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#log_directory .*/log_directory = \'\/var\/log\/postgresql\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#log_filename .*/log_filename = \'postgresql-\%Y-\%m-\%d_\%H\%M\%S.log\'/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#log_truncate_on_rotation .*/log_truncate_on_rotation = on/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#log_rotation_age .*/log_rotation_age = 1d/g" ${POSTGRESQL_CONF} +sed -i -E "s/^#log_rotation_size .*/log_rotation_size = 0/g" ${POSTGRESQL_CONF} +sed -i -E "s/^log_timezone .*/log_timezone = \'Asia\/Shanghai\'/g" ${POSTGRESQL_CONF} + +sed -i -E "s/^#include_dir .*/include_dir = \'conf\.d\'/g" ${POSTGRESQL_CONF} diff --git a/10/prebuilds/usr/local/scripts/libcommon.sh b/10/prebuilds/usr/local/scripts/libcommon.sh new file mode 100644 index 0000000..b76479e --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libcommon.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# + +# shellcheck disable=SC1091 + +BOLD='\033[1m' + +# 加载依赖项 +. /usr/local/scripts/liblog.sh + +# 函数列表 + +# 打印包含包含Logo的欢迎信息 +# 全局变量: +# APP_NAME +print_image_welcome_page() { + local github_url="https://github.com/colovu/docker-${APP_NAME}" + if [ x"${WELCOME_MESSAGE:-}" = "x" ]; then + LOG_I "" + LOG_I " ######## ######## ### ######## ### ## ### ##" + LOG_I " ### ## ### ## ### ### ## ### ## ### ##" + LOG_I " ### ### ## ### ### ## ### ## ### ##" + LOG_I " ### ### ## ### ### ## ### ## ### ##" + LOG_I " ### ### ## ### ### ## ### ## ### ##" + LOG_I " ### ## ### ## ### ### ## #### ### ##" + LOG_I "######## ######## ######## ######## ## ########" + LOG_I "" + LOG_I "Welcome to the ${BOLD}${APP_NAME}${RESET} container" + LOG_I "Project on Github: ${BOLD}${github_url}${RESET}" + LOG_I "Send us your feedback at ${BOLD}endial@126.com${RESET}" + LOG_I "" + + export WELCOME_MESSAGE=1 + fi +} + +# 根据需要打印欢迎信息 +# 全局变量: +# ENV_DISABLE_WELCOME_MESSAGE +# APP_NAME +docker_print_welcome() { + if [[ -z "${ENV_DISABLE_WELCOME_MESSAGE:-}" ]]; then + if [[ -n "$APP_NAME" ]]; then + print_image_welcome_page + fi + fi +} + +# 检测可能导致容器执行后直接退出的命令，如"--help"；如果存在，直接返回 0 +# 参数: +# $1 - 待检测的参数表 +docker_command_help() { + local arg + for arg; do + case "$arg" in + -'?'|--help|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +# 根据脚本扩展名及权限，执行相应的初始化脚本 +# 参数: +# $1 - 文件列表，支持路径通配符 +# 使用: +# docker_process_init_files [file [file [...]]] +# 例子: +# docker_process_init_files /src/conf/${APP_NAME}/initdb.d/* +docker_process_init_files() { + echo + local f + for f; do + case "$f" in + *.sh) + if [ -x "$f" ]; then + LOG_I "$0: running $f" + "$f" + else + LOG_I "$0: sourcing $f" + . "$f" + fi + ;; + *) LOG_W "$0: ignoring $f" ;; + esac + echo + done +} + +# 检测应用相应的配置文件是否存在，如果不存在，则从默认配置文件目录拷贝一份 +# 默认配置文件路径：/etc/${APP_NAME} +# 目标配置文件路径：/srv/conf/${APP_NAME} +# 参数： +# $* - 文件及目录列表字符串，以" "分割 +# 例子: +# ensure_config_file_exist /etc/${APP_NAME}/* +ensure_config_file_exist() { + local f + + LOG_D "Parameter: $@" + while [ "$#" -gt 0 ]; do + f="${1}" + LOG_D "Process ${f}" + if [ -d ${f} ]; then + dist="$(echo ${f} | sed -e 's/\/etc/\/srv\/conf/g')" + [ ! -d "${dist}" ] && LOG_I "Create directory: ${dist}" && mkdir -p "${dist}" + [[ ! -z $(ls -A "$f") ]] && ensure_config_file_exist ${f}/* + else + dist="$(echo ${f} | sed -e 's/\/etc/\/srv\/conf/g')" + [ ! -e "${dist}" ] && LOG_I "Copy: ${f} ===> ${dist}" && cp "${f}" "${dist}" && rm -rf "/srv/data/${APP_NAME}/.app_init_flag" + fi + shift + done +} + +# 检测当前用户是否为 root +# 返回值: +# 布尔值 +_is_run_as_root() { + if [[ "$(id -u)" = "0" ]]; then + LOG_D "Run as root" + true + else + LOG_D "User id: $(id -u)" + false + fi +} + +# 检测当前脚本是被直接执行的，还是从其他脚本中使用 "source" 调用的 +_is_sourced() { + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} diff --git a/10/prebuilds/usr/local/scripts/libdownload.sh b/10/prebuilds/usr/local/scripts/libdownload.sh new file mode 100644 index 0000000..7c79454 --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libdownload.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# 从服务器（列表）下载相应软件包 + +# Constants +#CV_BASE="http://archive.colovu.com/dist-files/" +#CV_BASE="http://10.37.129.2/dist-files/" +CV_BASE="" + +# 检测软件包签名是否正确 +# 参数： +# $1 - 软件包签名文件 +# $2 - 软件包文件 +# $3 - PGPKEY +check_pgp() { + local name_asc=${1:?missing asc file name} + local name=${2:?missing file name} + local keys="${3:?missing key id}" + + GNUPGHOME="$(mktemp -d)" + for key in $keys; do + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "${key}" || + gpg --batch --keyserver pgp.mit.edu --recv-keys "${key}" || + gpg --batch --keyserver keys.gnupg.net --recv-keys "${key}" || + gpg --batch --keyserver keyserver.pgp.com --recv-keys "${key}"; + done + gpg --batch --verify "$name_asc" "$name" + command -v gpgconf > /dev/null && gpgconf --kill all + rm -rf "$GNUPGHOME" "$name_asc" +} + +# 从私有服务器下载软件包，如果不存在，则从官网服务器下载 +# 参数： +# $1 - 软件包全名(字符串) +# $2 - 官网路径(字符串) +# $3 - "-c"/"--checksum" +# $4 - 软件包SHA256值 +# $3 - "-g"/"--pgpkey" +# $4 - 用于软件包签名的KEY ID +# 例子： +# . /usr/local/scripts/libdownload.sh && download_dist "java" "11.0.7-0" --checksum 02a1fc9b79b11617ad39221667f6a34209f5c45ca908268f8ba6c264a2577ee2 +download_dist() { + local name="${1:?name is required}" + local base_urls="${2:?url is required}" + local package_sha256="" + local pgp_key="" + local success="" + + # 获取SHA256或PGP KEY + shift 2 + while [ "$#" -gt 0 ]; do + case "$1" in + -c|--checksum) + shift + package_sha256="${1:?missing package checksum}" + ;; + -g|--pgpkey) + shift + pgp_key="${1:?missing package PGP key}" + ;; + *) + echo "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + echo "Downloading $name package" + for url in $CV_BASE $base_urls; do + if wget -O "$name" "$url$name" && [ -s "$name" ]; then + if [ -n "$pgp_key" ]; then + wget -O "$name.asc" "$url$name.asc" + if [ ! -e "$name.asc" ]; then + wget -O "$name.asc" "$url$name.sig" + fi + fi + success=1 + break + fi + done + + if [ -n "$success" ]; then + if [ -n "$package_sha256" ]; then + echo "Verifying package whith sha256" + echo "$package_sha256 *${name}" | sha256sum --check - + fi + + if [ -n "$pgp_key" ]; then + echo "Verifying package with PGP" + check_pgp "$name.asc" "$name" "$pgp_key" + fi + else + [ -n "$success" ] + fi +} diff --git a/10/prebuilds/usr/local/scripts/libfile.sh b/10/prebuilds/usr/local/scripts/libfile.sh new file mode 100644 index 0000000..9da26fa --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libfile.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# 文件操作函数库 + +# 函数列表 + +# 检测"*_FILE"文件，并从文件中读取信息作为参数值；环境变量不允许 VAR 与 VAR_FILE 方式并存 +# 变量： +# $1 - 需要设置的环境变量名称 +# $2 - 该变量对应的默认值（Option） +# +# 使用: file_env ENV_VAR [DEFAULT] +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + LOG_E "Both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + + export "$var"="$val" + unset "$fileVar" +} + +# 使用规则表达式在文件中替换数据 +# 参数: +# $1 - 文件名 +# $2 - 正则表达式 +# $3 - 替代数据表达式 +# $4 - 是否使用POSIX表达式. Default: true +replace_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local substitute_regex="${3:?substitute regex is required}" + local posix_regex=${4:-true} + + local result + + # 因部分系统兼容性问题，需要防止使用 'sed in-place' 方式操作 + if [[ $posix_regex = true ]]; then + result="$(sed -E "s@$match_regex@$substitute_regex@g" "$filename")" + else + result="$(sed "s@$match_regex@$substitute_regex@g" "$filename")" + fi + echo "$result" > "$filename" +} + +# 使用规则表达式在文件中删除数据 +# 参数: +# $1 - 文件名 +# $2 - 正则表达式 +# $3 - 是否使用POSIX表达式. Default: true +remove_in_file() { + local filename="${1:?filename is required}" + local match_regex="${2:?match regex is required}" + local posix_regex=${3:-true} + local result + + # 因部分系统兼容性问题，需要防止使用 'sed in-place' 方式操作 + if [[ $posix_regex = true ]]; then + result="$(sed -E "/$match_regex/d" "$filename")" + else + result="$(sed "/$match_regex/d" "$filename")" + fi + echo "$result" > "$filename" +} diff --git a/10/prebuilds/usr/local/scripts/libfs.sh b/10/prebuilds/usr/local/scripts/libfs.sh new file mode 100644 index 0000000..2ab7c7d --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libfs.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# 文件管理函数库 + +# 加载依赖项 +. /usr/local/scripts/liblog.sh + +# 函数列表 + +# Ensure a file/directory is owned (user and group) but the given user +# Arguments: +# $1 - filepath +# $2 - owner +ensure_owned_by() { + local path="${1:?path is missing}" + local owner="${2:?owner is missing}" + + chown "$owner":"$owner" "$path" +} + +# 检测目录是否存在，如果不存在则创建，同时修改为指定的用户 +# Arguments: +# $1 - directory +# $2 - owner +ensure_dir_exists() { + local dir="${1:?directory is missing}" + local owner="${2:-}" + + mkdir -p "${dir}" + if [[ -n $owner ]]; then + ensure_owned_by "$dir" "$owner" + fi +} + +# 检测目录是否存在或为空 +# 参数: +# $1 - 目录路径 +# 返回值: +# 布尔值 +is_dir_empty() { + local dir="${1:?missing directory}" + + if [[ ! -e "$dir" ]] || [[ -z "$(ls -A "$dir")" ]]; then + true + else + false + fi +} + +# 循环设置目录中子目录及文件权限 +# 参数: +# $1 - paths (as a string). +# Flags: +# -f|--file-mode - 文件权限模式 +# -d|--dir-mode - 目录权限模式 +# -u|--user - 用户 +# -g|--group - 用户组 +configure_permissions_ownership() { + local -r paths="${1:?paths is missing}" + local dir_mode="" + local file_mode="" + local user="" + local group="" + + # Validate arguments + shift 1 + while [ "$#" -gt 0 ]; do + case "$1" in + -f|--file-mode) + shift + file_mode="${1:?missing mode for files}" + ;; + -d|--dir-mode) + shift + dir_mode="${1:?missing mode for directories}" + ;; + -u|--user) + shift + user="${1:?missing user}" + ;; + -g|--group) + shift + group="${1:?missing group}" + ;; + *) + LOG_E "Invalid command line flag $1" >&2 + return 1 + ;; + esac + shift + done + + read -r -a filepaths <<< "$paths" + for p in "${filepaths[@]}"; do + if [[ -e "$p" ]]; then + LOG_D "Check directory $p" + if [[ -n $dir_mode ]]; then + LOG_D "Change permissions to 755 of directories in $p" + find -L "$p" -type d -exec chmod "$dir_mode" '{}' + + fi + if [[ -n $file_mode ]]; then + LOG_D "Change permissions to 755 of files in $p" + find -L "$p" -type f -exec chmod "$file_mode" '{}' + + fi + if [[ -n $user ]] && [[ -n $group ]]; then + LOG_D "Change ownership to ${user}:${group} of files and directories in $p" + find -L "$p" \! -user ${user} -or \! -group ${group} -exec chown -L "$user":"$group" '{}' + + elif [[ -n $user ]] && [[ -z $group ]]; then + LOG_D "Change user to ${user} of files and directories in $p" + find -L "$p" \! -user ${user} -exec chown -L "$user" '{}' + + elif [[ -z $user ]] && [[ -n $group ]]; then + LOG_D "Change groupto ${group} of files and directories in $p" + find -L "$p" \! -group ${group} -exec chgrp -L "$group" '{}' + + fi + else + LOG_E "$p does not exist" + fi + done +} \ No newline at end of file diff --git a/10/prebuilds/usr/local/scripts/liblog.sh b/10/prebuilds/usr/local/scripts/liblog.sh new file mode 100644 index 0000000..8901e8c --- /dev/null +++ b/10/prebuilds/usr/local/scripts/liblog.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# 日志处理函数库 + +# 定义颜色信息 +RESET='\033[0m' +RED='\033[31;1m' +GREEN='\033[32;2m' +YELLOW='\033[33;1m' +MAGENTA='\033[36;2m' +CYAN='\033[35;2m' +BLUE='\033[34;2m' + +# 函数列表 + +# 输出实际日志信息 +# 参数: +# $1 - 日志类型 +# $2 - 日志信息 +LOG_RAW() { + local type="$1"; shift + case "${type}" in + x) printf "${CYAN}${APP_NAME:-} ${MAGENTA}%s ${RESET}${BLUE}DEBUG${RESET} %b\n" "$(date "+%T.%2N")" "${*}" ;; + I) printf "${CYAN}${APP_NAME:-} ${MAGENTA}%s ${RESET}${GREEN}INFO ${RESET} %b\n" "$(date "+%T.%2N")" "${*}";; + W) printf "${CYAN}${APP_NAME:-} ${MAGENTA}%s ${RESET}${YELLOW}WARN ${RESET} %b\n" "$(date "+%T.%2N")" "${*}";; + E) printf "${CYAN}${APP_NAME:-} ${MAGENTA}%s ${RESET}${RED}ERROR${RESET} %b\n" "$(date "+%T.%2N")" "${*}";; + esac +} + +# 输出调试类日志信息，尽量少使用 +# 参数: +# $1 - 日志类型 +# $2 - 日志信息 +LOG_D() { + local -r bool="${ENV_DEBUG:-false}" + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + LOG_RAW x "$@" + fi +} + +# 输出提示信息类日志信息 +# 参数: +# $1 - 日志类型 +# $2 - 日志信息 +LOG_I() { + shopt -s nocasematch + LOG_RAW I "$@" +} + +# 输出警告类日志信息至sterr +# 参数: +# $1 - 日志类型 +# $2 - 日志信息 +LOG_W() { + LOG_RAW W "$@" >&2 +} + +# 输出错误类日志信息至sterr，并退出脚本 +# 参数: +# $1 - 日志类型 +# $2 - 日志信息 +LOG_E() { + LOG_RAW E "$@" >&2 +} diff --git a/10/prebuilds/usr/local/scripts/libnet.sh b/10/prebuilds/usr/local/scripts/libnet.sh new file mode 100644 index 0000000..be3c1ff --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libnet.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# 网络管理函数库 + +# shellcheck disable=SC1091 + +# 加载依赖项 +. /usr/local/scripts/liblog.sh + +# 函数列表 + +# 解析主机名为 IP +# 参数: +# $1 - 待解析的主机名 +# 返回值: +# IP 地址 +######################### +dns_lookup() { + local host="${1:?host is missing}" + getent ahosts "$host" | awk '/STREAM/ {print $1 }' +} + +# 等待主机名解析，并返回 IP +# 参数: +# $1 - 主机名 +# $2 - 重试次数 +# $3 - 重试间隔（秒） +# 返回值: +# - IP 地址 +wait_for_dns_lookup() { + local hostname="${1:?hostname is missing}" + local retries="${2:-5}" + local seconds="${3:-1}" + check_host() { + if [[ $(dns_lookup "$hostname") == "" ]]; then + false + else + true + fi + } + # Wait for the host to be ready + retry_while "check_host ${hostname}" "$retries" "$seconds" + dns_lookup "$hostname" +} + +# 获取机器的 IP +# 返回值: +# - IP 地址 +get_machine_ip() { + dns_lookup "$(hostname)" +} + +# 检测提供的参数是否为可解析地址的主机名 +# 参数: +# $1 - 待检测值 +# 返回值: +# 布尔值 +is_hostname_resolved() { + local -r host="${1:?missing value}" + if [[ -n "$(dns_lookup "$host")" ]]; then + true + else + false + fi +} + +# 解析 URL +# 参数: +# $1 - URI 字符串 +# $2 - 待解析参数字符串。有效值 (scheme, authority, userinfo, host, port, path, query or fragment) +# 返回值: +# 字符串 +parse_uri() { + local uri="${1:?uri is missing}" + local component="${2:?component is missing}" + + # Solution based on https://tools.ietf.org/html/rfc3986#appendix-B with + # additional sub-expressions to split authority into userinfo, host and port + # Credits to Patryk Obara (see https://stackoverflow.com/a/45977232/6694969) + local -r URI_REGEX='^(([^:/?#]+):)?(//((([^@/?#]+)@)?([^:/?#]+)(:([0-9]+))?))?(/([^?#]*))?(\?([^#]*))?(#(.*))?' + # || | ||| | | | | | | | | | + # |2 scheme | ||6 userinfo 7 host | 9 port | 11 rpath | 13 query | 15 fragment + # 1 scheme: | |5 userinfo@ 8 :... 10 path 12 ?... 14 #... + # | 4 authority + # 3 //... + local index=0 + case "$component" in + scheme) + index=2 + ;; + authority) + index=4 + ;; + userinfo) + index=6 + ;; + host) + index=7 + ;; + port) + index=9 + ;; + path) + index=10 + ;; + query) + index=13 + ;; + fragment) + index=14 + ;; + *) + stderr_print "unrecognized component $component" + return 1 + ;; + esac + [[ "$uri" =~ $URI_REGEX ]] && echo "${BASH_REMATCH[${index}]}" +} + \ No newline at end of file diff --git a/10/prebuilds/usr/local/scripts/libos.sh b/10/prebuilds/usr/local/scripts/libos.sh new file mode 100644 index 0000000..680c498 --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libos.sh @@ -0,0 +1,158 @@ +#!/bin/bash +# +# 操作系统控制函数库 + +# shellcheck disable=SC1091 + +# 加载依赖项 +. /usr/local/scripts/liblog.sh + +# 函数列表 + +# 检测指定用户账户是否存在 +# 参数: +# $1 - 用户账户 +# 返回值: +# 布尔值 +user_exists() { + local user="${1:?user is missing}" + id "$user" >/dev/null 2>&1 +} + +# 检测指定用户分组是否存在 +# 参数: +# $1 - 用户组 +# 返回值: +# 布尔值 +group_exists() { + local group="${1:?group is missing}" + getent group "$group" >/dev/null 2>&1 +} + +# 确保用户组存在，如果不存在则创建相应用户组 +# 参数: +# $1 - 用户组 +ensure_group_exists() { + local group="${1:?group is missing}" + + if ! group_exists "$group"; then + groupadd "$group" >/dev/null 2>&1 + fi +} + +# 确保用户组及用户账户存在，如果不存在则创建相应用户组及账户 +# 参数: +# $1 - 用户 +# $2 - 用户组 +ensure_user_exists() { + local user="${1:?user is missing}" + local group="${2:-}" + + if ! user_exists "$user"; then + useradd "$user" >/dev/null 2>&1 + fi + + if [[ -n "$group" ]]; then + ensure_group_exists "$group" + fi + + usermod -a -G "$group" "$user" >/dev/null 2>&1 +} + +# 获取系统可用内存 +# 返回值: +# 内存大小(MB) +get_total_memory() { + echo $(($(grep MemTotal /proc/meminfo | awk '{print $2}') / 1024)) +} + +# 获取以定量方式描述的内存大小 +# 参数: +# $1 - 内存大小 (可选) +# 返回值: +# 基于定量内存大小的内存大小描述 +get_machine_size() { + local memory="${1:-}" + if [[ -z "$memory" ]]; then + debug "Memory was not specified, detecting available memory automatically" + memory="$(get_total_memory)" + fi + sanitized_memory=$(convert_to_mb "$memory") + if [[ "$sanitized_memory" -gt 26000 ]]; then + echo 2xlarge + elif [[ "$sanitized_memory" -gt 13000 ]]; then + echo xlarge + elif [[ "$sanitized_memory" -gt 6000 ]]; then + echo large + elif [[ "$sanitized_memory" -gt 3000 ]]; then + echo medium + elif [[ "$sanitized_memory" -gt 1500 ]]; then + echo small + else + echo micro + fi +} + +# 获取已定义的所有内存大小描述 +# 返回值: +# 内存大小描述 +get_supported_machine_sizes() { + echo micro small medium large xlarge 2xlarge +} + +# 将以字符串表示的内存大小转换为以MB为单位的内存大小值 (i.e. 2G -> 2048) +# 参数: +# $1 - 内存大小 +# 返回值: +# 内存大小值(以MB为单位) +convert_to_mb() { + local amount="${1:-}" + if [[ $amount =~ ^([0-9]+)(M|G) ]]; then + size="${BASH_REMATCH[1]}" + unit="${BASH_REMATCH[2]}" + if [[ "$unit" = "G" ]]; then + amount="$((size * 1024))" + else + amount="$size" + fi + fi + echo "$amount" +} + +# 如果禁用调试模式，将输出信息重定向至 /dev/null +# 全局变量: +# ENV_DEBUG +# 参数: +# $@ - 待执行的命令 +debug_execute() { + local -r bool="${ENV_DEBUG:-false}" + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + "$@" >/dev/null 2>&1 + else + "$@" + fi +} + +# 重试执行命令 +# 参数: +# $1 - cmd (as a string) +# $2 - 最大尝试次数. Default: 12 +# $3 - 重试前等待时间(秒). Default: 5 +# 返回值: +# 布尔值 +retry_while() { + local -r cmd="${1:?cmd is missing}" + local -r retries="${2:-12}" + local -r sleep_time="${3:-5}" + local return_value=1 + + read -r -a command <<< "$cmd" + for ((i = 1 ; i <= retries ; i+=1 )); do + "${command[@]}" && return_value=0 && break + sleep "$sleep_time" + done + return $return_value +} + + \ No newline at end of file diff --git a/10/prebuilds/usr/local/scripts/libservice.sh b/10/prebuilds/usr/local/scripts/libservice.sh new file mode 100644 index 0000000..a4ab489 --- /dev/null +++ b/10/prebuilds/usr/local/scripts/libservice.sh @@ -0,0 +1,131 @@ +#!/bin/bash +# +# 服务管理函数库 + +# shellcheck disable=SC1091 + +# Load Generic Libraries +. /usr/local/scripts/libvalidations.sh + +# 函数列表 + +# 获取并返回服务 PID +# 参数: +# $1 - PID 文件 +# 返回值: +# PID +get_pid_from_file() { + local pid_file="${1:?pid file is missing}" + + if [[ -f "$pid_file" ]]; then + if [[ -n "$(< "$pid_file")" ]] && [[ "$(< "$pid_file")" -gt 0 ]]; then + echo "$(< "$pid_file")" + fi + fi +} + +# 检测 PID 对应的服务是否在运行中 +# 参数: +# $1 - PID +# 返回值: +# Boolean +is_service_running() { + local pid="${1:?pid is missing}" + + kill -0 "$pid" 2>/dev/null +} + +# 通过发送信号停止一个指定的服务 +# 参数: +# $1 - PID 文件 +# $2 - 信号 (可选) +stop_service_using_pid() { + local pid_file="${1:?pid file is missing}" + local signal="${2:-}" + local pid + + pid="$(get_pid_from_file "$pid_file")" + [[ -z "$pid" ]] || ! is_service_running "$pid" && return + + if [[ -n "$signal" ]]; then + kill "-${signal}" "$pid" + else + kill "$pid" + fi + + local counter=10 + while [[ "$counter" -ne 0 ]] && is_service_running "$pid"; do + sleep 1 + counter=$((counter - 1)) + done +} + +# 为指定的服务生成一个监控配置文件 +# Arguments: +# $1 - 服务名 +# $2 - PID 文件 +# $3 - 启动命令 +# $4 - 停止命令 +# Flags: +# --disabled - Whether to disable the monit configuration +generate_monit_conf() { + local service_name="${1:?service name is missing}" + local pid_file="${2:?pid file is missing}" + local start_command="${3:?start command is missing}" + local stop_command="${4:?stop command is missing}" + local monit_conf_dir="/etc/monit/conf.d" + local disabled="no" + + # Parse optional CLI flags + shift 4 + while [[ "$#" -gt 0 ]]; do + case "$1" in + --disabled) + shift + disabled="$1" + ;; + *) + echo "Invalid command line flag ${1}" >&2 + return 1 + ;; + esac + shift + done + + is_boolean_yes "$disabled" && conf_suffix=".disabled" + mkdir -p "$monit_conf_dir" + cat >"${monit_conf_dir}/${service_name}.conf${conf_suffix:-}" <"${logrotate_conf_dir}/${service_name}" <= 0 )); then + true + else + false + fi +} + +# 检测数据是否为布尔值 '1' 或字符串 'yes/true' +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +is_boolean_yes() { + local -r bool="${1:-}" + # comparison is performed without regard to the case of alphabetic characters + shopt -s nocasematch + if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then + true + else + false + fi +} + +# 检测数据是否为字符串 'yes/no' +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +is_yes_no_value() { + local -r bool="${1:-}" + if [[ "$bool" =~ ^(yes|no)$ ]]; then + true + else + false + fi +} + +# 检测数据是否为字符串 'true/false' +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +is_true_false_value() { + local -r bool="${1:-}" + if [[ "$bool" =~ ^(true|false)$ ]]; then + true + else + false + fi +} + +# 检测提供的参数是否为空字符串或未定义 +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +is_empty_value() { + local -r val="${1:-}" + if [[ -z "$val" ]]; then + true + else + false + fi +} + +# 检测数据是否为有效的端口号 +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 或 错误消息 +validate_port() { + local value + local unprivileged=0 + + # Parse flags + while [[ "$#" -gt 0 ]]; do + case "$1" in + -unprivileged) + unprivileged=1 + ;; + --) + shift + break + ;; + -*) + stderr_print "unrecognized flag $1" + return 1 + ;; + *) + break + ;; + esac + shift + done + + if [[ "$#" -gt 1 ]]; then + echo "too many arguments provided" + return 2 + elif [[ "$#" -eq 0 ]]; then + stderr_print "missing port argument" + return 1 + else + value=$1 + fi + + if [[ -z "$value" ]]; then + echo "the value is empty" + return 1 + else + if ! is_int "$value"; then + echo "value is not an integer" + return 2 + elif [[ "$value" -lt 0 ]]; then + echo "negative value provided" + return 2 + elif [[ "$value" -gt 65535 ]]; then + echo "requested port is greater than 65535" + return 2 + elif [[ "$unprivileged" = 1 && "$value" -lt 1024 ]]; then + echo "privileged port requested" + return 3 + fi + fi +} + +# 检测数据是否为有效的IPv4地址 +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +validate_ipv4() { + local ip="${1:?ip is missing}" + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + read -r -a ip_array <<< "$(tr '.' ' ' <<< "$ip")" + [[ ${ip_array[0]} -le 255 && ${ip_array[1]} -le 255 \ + && ${ip_array[2]} -le 255 && ${ip_array[3]} -le 255 ]] + stat=$? + fi + return $stat +} + +# 校验字符串格式 +# 参数: +# $1 - 待检测的数据 +# 返回值: +# 布尔值 +validate_string() { + local string + local min_length=-1 + local max_length=-1 + + # Parse flags + while [ "$#" -gt 0 ]; do + case "$1" in + -min-length) + shift + min_length=${1:-} + ;; + -max-length) + shift + max_length=${1:-} + ;; + --) + shift + break + ;; + -*) + stderr_print "unrecognized flag $1" + return 1 + ;; + *) + break + ;; + esac + shift + done + + if [ "$#" -gt 1 ]; then + stderr_print "too many arguments provided" + return 2 + elif [ "$#" -eq 0 ]; then + stderr_print "missing string" + return 1 + else + string=$1 + fi + + if [[ "$min_length" -ge 0 ]] && [[ "${#string}" -lt "$min_length" ]]; then + echo "string length is less than $min_length" + return 1 + fi + if [[ "$max_length" -ge 0 ]] && [[ "${#string}" -gt "$max_length" ]]; then + echo "string length is great than $max_length" + return 1 + fi +} \ No newline at end of file