diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fdbc153 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,38 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ +/.mvn/ + +## service +/logs +/_deploy +/_doc diff --git a/.gitignore b/.gitignore index 32de00b..d77aaa5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,34 +1,270 @@ -HELP.md +# gitignore,多数参考 https://github.com/github/gitignore +# Usage: 整个文件复制进项目,并屏蔽不想要的内容 + +# 项目相关 +logs/ + +######################################################################## +# 1. 语言相关 +######################################################################## + +################################ java ############################## +*.class +*.log +*.ctxt +.mtj.tmp/ +*.jar +*.war +*.nar +*.ear +*.tar.gz +hs_err_pid* + +################################ golang ############################# +#*.exe +#*.exe~ +#*.dll +#*.so +#*.dylib +#*.test + +## Output of the go coverage tool, specifically when used with LiteIDE +#*.out + +######################################################################## +# 2. 环境相关 +######################################################################## + +########################## node, webpack, vite ######################### +#node_modules/ +#dist/ +# +## local env files +#.env.local +#.env.*.local +# +## Log files +#npm-debug.log* +#yarn-debug.log* +#yarn-error.log* + +################################# maven ############################# target/ -!.mvn/wrapper/maven-wrapper.jar -!**/src/main/**/target/ -!**/src/test/**/target/ - -### STS ### -.apt_generated -.classpath -.factorypath -.project -.settings -.springBeans -.sts4-cache - -### IntelliJ IDEA ### -.idea -*.iws +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +pom.xml.next +release.properties +dependency-reduced-pom.xml +buildNumber.properties +.mvn/timing.properties +# https://github.com/takari/maven-wrapper#usage-without-binary-jar +.mvn/wrapper/maven-wrapper.jar + +################################ gradle ############################# +#.gradle +#**/build/ +#!src/**/build/ +#gradle-app.setting +#!gradle-wrapper.jar +#.gradletasknamecache + +######################################################################## +# IDE +######################################################################## + +################################ jetbrains ############################# +# 遵照 jetbrains 官方建议,提交有关项目风格的内容,忽略有关私有环境和数据源的项目 +# 但方案不同,为避免插件的 xml 文件,采用了全忽略但保留某些文件的方案 +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +.idea/** + +!.idea/codeStyles/ +!.idea/codeStyles/** +!.idea/sqldialects.xml + *.iml *.ipr -### NetBeans ### -/nbproject/private/ +# CMake +cmake-build-*/ + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +################################ Eclipse ############################# +/.metadata +/bin/ +/tmp/ +/*.tmp +/*.bak +/*.swp +/*~.nib +/local.properties +/.settings/ +/.loadpath +/.recommenders +/.externalToolBuilders/ +/*.launch +/*.pydevproject +/.cproject +/.autotools +/.factorypath +/.buildpath +/.target +/.tern-project +/.texlipse +/.springBeans +/.recommenders/ +/.apt_generated/ +/.apt_generated_test/ +/.cache-main +/.scala_dependencies +/.worksheet + +################################ NetBeans ############################# +**/nbproject/private/ +**/nbproject/Makefile-*.mk +**/nbproject/Package-*.bash +/build/ /nbbuild/ /dist/ /nbdist/ /.nb-gradle/ -build/ -!**/src/main/**/build/ -!**/src/test/**/build/ -### VS Code ### -.vscode/ -/.mvn/ +################################ VSCode ############################# +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +######################################################################## +# - 杂项 +######################################################################## +# ansible +*.retry + +# diff +*.patch +*.diff + +# Microsoft Office +~$*.doc* +Backup of *.doc* +~$*.xls* +*.xlk +~$*.ppt* +*.~vsd* + +# Redis +*.rdb + +# TortoiseGit +/.tgitconfig + +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json + +######################################################################## +# - 系统相关 +######################################################################## +################################# macOS ################################ +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +################################ Windows ################################# + +# Windows thumbnail cache files +Thumbs.db +Thumbs.db:encryptable +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +################################## Linux ############################### +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### service ### +mybatis-gen/report.txt diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 0000000..aac5413 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,11 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..79ee123 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..35b5a03 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,67 @@ +ARG MAVEN_IMAGE=maven:3-adoptopenjdk-11 +ARG MAVEN_RUNTIME=adoptopenjdk:11-jre-hotspot + +# ----------------------------------------------------------------------------- +# Build 构建,这部分最好提取出来利用 CI 执行,可以有 maven 缓存使用 +# ----------------------------------------------------------------------------- +FROM ${MAVEN_IMAGE} AS build + +WORKDIR /application + +COPY build/settings.xml /usr/share/maven/ref/settings-docker.xml + +# pre dependency,需要按照项目结构,将所有的 pom.xml 都复制进去 +COPY pom.xml ./pom.xml +RUN mvn -B -f /application/pom.xml -s /usr/share/maven/ref/settings-docker.xml dependency:resolve + +# build +COPY src /application/src +RUN mvn -f /application/pom.xml -s /usr/share/maven/ref/settings-docker.xml clean package -DskipTests + +# ----------------------------------------------------------------------------- +# spring-boot unpacker +# ----------------------------------------------------------------------------- +FROM ${MAVEN_RUNTIME} as unpacker + +WORKDIR application + +ARG JAR_FILE=/application/target/*.jar +COPY --from=build ${JAR_FILE} application.jar +RUN java -Djarmode=layertools -jar application.jar extract + +# ----------------------------------------------------------------------------- +# Runtime +# ----------------------------------------------------------------------------- +FROM ${MAVEN_RUNTIME} + +LABEL maintainer="Mark Zhou " + +# 支持的环境变量: +# - NAME 应用名称 +# - JAVA_OPTS jvm 参数 +# - JAVA_AGENT jvm agent + +WORKDIR /application +HEALTHCHECK --interval=3s --timeout=5s --start-period=120s CMD curl -f http://localhost:60002/actuator || exit 1 + +# 以下为单一 jar 文件运行方法 +# COPY --from=build target/*.jar ./app.jar + +# 以下四行为 springboot 2.3.0+ 分层打包 +# https://docs.spring.io/spring-boot/docs/2.3.0.RELEASE/reference/html/spring-boot-features.html#writing-the-dockerfile +# https://docs.spring.io/spring-boot/docs/2.4.0/reference/htmlsingle/#boot-features-container-images +# https://docs.spring.io/spring-boot/docs/2.5.3/reference/htmlsingle/#features.container-images +COPY --from=unpacker application/dependencies/ ./ +COPY --from=unpacker application/spring-boot-loader/ ./ +COPY --from=unpacker application/snapshot-dependencies/ ./ +COPY --from=unpacker application/application/ ./ + +# add startup.sh,为了防止换行符,后边有转换 CRLF -> LF ,因此这个文件必须最后加入,不然上边的 extract 就失去了意义 +COPY build/startup-springboot.sh ./startup-springboot.sh +RUN sed -i $'s/\r$//' startup-springboot.sh + +ENTRYPOINT ["sh", "startup-springboot.sh"] + +# 60001 springmvc, 60002 spring-actuator +EXPOSE 60001 +EXPOSE 60002 diff --git a/Dockerfile.buildkit b/Dockerfile.buildkit new file mode 100644 index 0000000..e4603db --- /dev/null +++ b/Dockerfile.buildkit @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.3 + +ARG MAVEN_IMAGE=maven:3-adoptopenjdk-11 +ARG MAVEN_RUNTIME=adoptopenjdk:11-jre-hotspot + +# ----------------------------------------------------------------------------- +# Build 构建,使用了 buildx 的缓存 +# ----------------------------------------------------------------------------- +FROM ${MAVEN_IMAGE} AS build + +WORKDIR /application + +COPY build/settings.xml /usr/share/maven/ref/settings-docker.xml + +# copy 复制所有文件,建议有选择的只把 src 和 lib 复制进去,CI 中不会有log等文件就无所谓了 +COPY . . + +# build +RUN --mount=type=cache,target=/usr/share/maven/ref/repository mvn -f /application/pom.xml -s /usr/share/maven/ref/settings-docker.xml clean package -DskipTests + +# ----------------------------------------------------------------------------- +# spring-boot unpacker +# ----------------------------------------------------------------------------- +FROM ${MAVEN_RUNTIME} as unpacker + +WORKDIR application + +ARG JAR_FILE=/application/target/*.jar +COPY --from=build ${JAR_FILE} application.jar +RUN java -Djarmode=layertools -jar application.jar extract + +# ----------------------------------------------------------------------------- +# Runtime +# ----------------------------------------------------------------------------- +FROM ${MAVEN_RUNTIME} + +LABEL maintainer="Mark Zhou " + +# 支持的环境变量: +# - NAME 应用名称 +# - JAVA_OPTS jvm 参数 +# - JAVA_AGENT jvm agent + +WORKDIR /application +HEALTHCHECK --interval=3s --timeout=5s --start-period=120s CMD curl -f http://localhost:60002/actuator || exit 1 + +# 以下为单一 jar 文件运行方法 +# COPY --from=build target/*.jar ./app.jar + +# 以下四行为 springboot 2.3.0+ 分层打包 +# https://docs.spring.io/spring-boot/docs/2.3.0.RELEASE/reference/html/spring-boot-features.html#writing-the-dockerfile +# https://docs.spring.io/spring-boot/docs/2.4.0/reference/htmlsingle/#boot-features-container-images +# https://docs.spring.io/spring-boot/docs/2.5.3/reference/htmlsingle/#features.container-images +COPY --from=unpacker application/dependencies/ ./ +COPY --from=unpacker application/spring-boot-loader/ ./ +COPY --from=unpacker application/snapshot-dependencies/ ./ +COPY --from=unpacker application/application/ ./ + +# add startup.sh,为了防止换行符,后边有转换 CRLF -> LF ,因此这个文件必须最后加入,不然上边的 extract 就失去了意义 +COPY build/startup-springboot.sh ./startup-springboot.sh +RUN sed -i $'s/\r$//' startup-springboot.sh + +ENTRYPOINT ["sh", "startup-springboot.sh"] + +# 60001 springmvc, 60002 spring-actuator +EXPOSE 60001 +EXPOSE 60002 diff --git a/_deploy/README.md b/_deploy/README.md new file mode 100644 index 0000000..4d2c18e --- /dev/null +++ b/_deploy/README.md @@ -0,0 +1,82 @@ +# 部署用脚本 + +整个项目流程为 +- 决定环境 +- 修改配置 +- 编译、打包 +- push + +线上机器: +- pull image +- pull _deploy +- sh _deploy/init.sh +- (SPUG环境)sh _deploy/spug.sh +- sh update.sh + +日常维护脚本: +- tail.sh 查看日志 +- exec.sh 进入容器 +- update.sh 更新版本,仅在有tag变更后可用 + +## pre + +决定环境后第一步,先改 _config.sh + +这个步骤应该在项目计划部署时,就写入 git 中 + +## build + +无CI/CD: + +``` +执行 _deploy/build.sh 打包并上传 +``` + +有 CI/CD: + +``` +……有了再说 +确保Docker 版本支持 BuildKit(19.03+),执行 _deploy/build.sh 打包并上传 + +或者,以后有按照 CI/CD 特性 而定义的特殊流程 +``` + +## spug 配置参考 + +构建脚本 +```bash +TAG=${SPUG_GIT_BRANCH}-$(date +%y.%m.%d)-${SPUG_GIT_COMMIT_ID:0:7} +bash _deploy/build.sh ${TAG} +``` + +部署脚本 +```bash +echo "IMAGE: $( -j DNAT --to-destination <容器ip>:<容器内部端口> + # 取消端口映射规则 + iptables -t nat -D DOCKER -p tcp -d 0/0 --dport <容器外部端口> -j DNAT --to-destination <容器ip>:<容器内部端口> + docker-compose exec ${SERVICE_NAME} java -jar arthas-boot.jar --target-ip 0.0.0.0 + ``` + - 或者,使用服务 Arthas Tunnel 服务 + +- 怎么配合网关上下线切流,需要脚本控制,计划使用 Haproxy 方案,机器太少暂且搁置 diff --git a/_deploy/_config.sh b/_deploy/_config.sh new file mode 100644 index 0000000..aec06d4 --- /dev/null +++ b/_deploy/_config.sh @@ -0,0 +1,16 @@ +# 该脚本随项目 _deploy 分发,脚本内参数仅与 docker镜像 docker-compose 定义有关 + +set -e + +# 镜像地址,不包括tag +IMAGE_BASE=registry.cn-beijing.aliyuncs.com/zzmark/echo + +# 服务名、监控服务名,仅支持小写、下划线 +SERVICE_NAME=echo +MONITOR_SERVICE_NAME=monitor_echo + +# 服务域名 +DOMAIN="template-api.leadpcom.com" + +# 日志位置,会被 init.sh 执行时的环境变量覆盖掉 +LOG_DIR=/data/log/${SERVICE_NAME} diff --git a/_deploy/build.sh b/_deploy/build.sh new file mode 100644 index 0000000..2e0bbd7 --- /dev/null +++ b/_deploy/build.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# docker 构建脚本,完全使用 Dockerfile 构建流 +# 编译、打包、生成 Docker Image 全都靠 Dockerfile 流程 +# +# usage: +# build.sh xxx # 输入编译目标的 TAG 版本 +# +# mark_zhou[zz.mark06@gmail.com] + +set -e + +_DEPLOY_DIR=$(cd `dirname $0`; pwd) +WORKSPACE_DIR=$(pwd) + +source ${_DEPLOY_DIR}/_config.sh + +if [ ! -n "$1" ];then + echo "缺少 tag 参数 无法执行脚本" + exit 1 +fi + +IMAGE_TAG=$1 +IMAGE=${IMAGE_BASE}:${IMAGE_TAG} + +# 判断 docker ,高版本使用 buildx 低版本使用普通的 +# 18.09 后,docker 支持 buildkit 特性,可以更好的使用缓存 +# 按照官方建议,最好可以使用 buildx 来进行构建,但这个部分官方按照插件分发,即使版本足够也可能无法使用,所以有具体的判断 +# 为保兼容性,项目维护两个 Dockerfile +DOCKER_VERSION=$(docker version -f '{{.Client.Version}}') +BUILDX_SUPPORT_VERSION=18.09 + +echo "Build 并 Push 镜像 ${IMAGE}" + +DOCKERFILE=${WORKSPACE_DIR}/Dockerfile +if [[ $(echo "${BUILDX_SUPPORT_VERSION} ${DOCKER_VERSION}" | tr " " "\n" | sort -V | head -n 1) != ${DOCKER_VERSION} ]]; then + echo "Docker 版本 ${DOCKER_VERSION} 支持 BuildKit,使用 buildkit 文件" + export DOCKER_BUILDKIT=1 + DOCKERFILE=${WORKSPACE_DIR}/Dockerfile.buildkit +fi + +CHECK_BUILDKIT=$(docker buildx || true) + +if [[ ${CHECK_BUILDKIT} =~ "BuildKit" ]]; then + echo "使用 BuildKit buildx 命令打包" + docker buildx build -f ${DOCKERFILE} . --tag ${IMAGE} --push + echo "推送完毕" +else + echo "Docker 版本 ${DOCKER_VERSION} 不支持 buildx,使用 Legacy 方案" + docker build -f ${DOCKERFILE} . --tag ${IMAGE} + docker push ${IMAGE} + echo "推送完毕" +fi + +# 存留版本信息 +echo ${IMAGE_TAG} > ${_DEPLOY_DIR}/version.txt diff --git a/_deploy/docker-compose.yml.template b/_deploy/docker-compose.yml.template new file mode 100644 index 0000000..184fe53 --- /dev/null +++ b/_deploy/docker-compose.yml.template @@ -0,0 +1,44 @@ +version: "3.7" +services: + ${SERVICE_NAME}: + image: ${IMAGE} + restart: always + networks: + - gateway_edge + - internal + labels: + traefik.enable: 'true' + traefik.docker.network: gateway_edge + + traefik.http.routers.${SERVICE_NAME}.service: ${SERVICE_NAME} + traefik.http.routers.${SERVICE_NAME}.rule: Host(\`${DOMAIN}\`) + traefik.http.routers.${SERVICE_NAME}.entrypoints: http + traefik.http.services.${SERVICE_NAME}.loadbalancer.server.port: 8080 + + traefik.http.routers.${MONITOR_SERVICE_NAME}.service: ${MONITOR_SERVICE_NAME} + traefik.http.routers.${MONITOR_SERVICE_NAME}.rule: Host(\`${DOMAIN}\`) + traefik.http.routers.${MONITOR_SERVICE_NAME}.entrypoints: monitor + traefik.http.services.${MONITOR_SERVICE_NAME}.loadbalancer.server.port: 8081 + + # healthCheck + traefik.http.services.${SERVICE_NAME}.loadbalancer.healthCheck.path: /actuator + traefik.http.services.${SERVICE_NAME}.loadbalancer.healthCheck.scheme: http + traefik.http.services.${SERVICE_NAME}.loadbalancer.healthCheck.port: 8081 + volumes: + - "./application.yml:/application/config/application-platform.yml" + - ${LOG_DIR}:/application/logs + + redis: + image: redis + restart: always + command: redis-server --appendonly yes + networks: + - internal + +networks: + internal: + gateway_edge: + name: gateway_edge + external: true + +# 基于 spring-boot service template v1.0 diff --git a/_deploy/init.sh b/_deploy/init.sh new file mode 100644 index 0000000..8609f91 --- /dev/null +++ b/_deploy/init.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# 部署环境初始化脚本 +# 需要先修改同在 _deploy 下的 _config.sh 文件 +# 该脚本负责执行以下操作: +# - docker-compose.yml 文件生成 +# - 生成 update.sh exec.sh tail.sh restart.sh 几个常用脚本 +# docker-compose.yml.template 文件随项目固定,基本不会变动。若要修改,请通过 git 提交 +# 该脚本随项目 _deploy 分发,脚本内参数仅与 docker-compose 定义有关 +# +# usage: +# ./init.sh xxxx/xxxxx:xxx # 提供完整 image ,直接使用,优先度高 +# ./init.sh # 提供 version.txt 文件,写入 image 使用 +# +# env: +# - LOG_DIR 日志存放目录,优先于 _config.sh +# +# mark_zhou[zz.mark06@gmail.com] + +set -e + +_DEPLOY_DIR=$(cd `dirname $0`; pwd) +WORKSPACE_DIR=$(pwd) + +# 保存环境变量中的 LOG_DIR +_LOG_DIR=${LOG_DIR} + +echo "加载配置 ${_DEPLOY_DIR}/_config.sh" +source ${_DEPLOY_DIR}/_config.sh + +# 优先使用脚本后的参数 +if [ ! -n "$1" ];then + # 无参数则从文件获取 + VERSION_FILE_PATH=${_DEPLOY_DIR}/version.txt + if [ ! -f ${VERSION_FILE_PATH} ]; then + echo "缺少 version 参数 无法执行脚本" + exit 1 + else + export IMAGE_TAG=$(< ${VERSION_FILE_PATH}) + export IMAGE=${IMAGE_BASE}:${IMAGE_TAG} + fi +else + export IMAGE=$1 +fi + +# 以下生成部分 +echo "准备生成镜像 ${IMAGE} 相关运行文件" + +# 脚本执行前,环境变量中存在 LOG_DIR,优先使用 +if [ -z ${_LOG_DIR} ];then + _LOG_DIR=${LOG_DIR} +fi + +# docker-compose.yml +LOG_DIR=${_LOG_DIR} +echo "生成 docker-compose.yml 到 ${WORKSPACE_DIR}/docker-compose.yml" +eval "cat < ${WORKSPACE_DIR}/docker-compose.yml + +# update.sh +echo "生成 update.sh 到 ${WORKSPACE_DIR}/update.sh" +export __SERVICE_NAME=${SERVICE_NAME} +eval "cat < ${WORKSPACE_DIR}/update.sh + +# exec.sh +echo "生成 exec.sh 到 ${WORKSPACE_DIR}/exec.sh" +cat << EOF > ${WORKSPACE_DIR}/exec.sh +docker-compose exec ${SERVICE_NAME} bash +EOF + +# tail.sh +echo "生成 tail.sh 到 ${WORKSPACE_DIR}/tail.sh" +cat << EOF > ${WORKSPACE_DIR}/tail.sh +docker-compose logs -f --tail 20 +EOF + +# restart.sh +echo "生成 restart.sh 到 ${WORKSPACE_DIR}/restart.sh" +cat << EOF > ${WORKSPACE_DIR}/restart.sh +docker-compose restart ${SERVICE_NAME} +EOF + +# chmod +x +chmod +x ${WORKSPACE_DIR}/update.sh ${WORKSPACE_DIR}/exec.sh ${WORKSPACE_DIR}/tail.sh ${WORKSPACE_DIR}/restart.sh + +# 写一个空的 application.yml +touch ${WORKSPACE_DIR}/application.yml diff --git a/_deploy/spug.sh b/_deploy/spug.sh new file mode 100644 index 0000000..f2c8c86 --- /dev/null +++ b/_deploy/spug.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# spug运维平台 配置文件处理脚本,会由 spug 部署逻辑执行 +# 需要根据业务定制 +# +# 基于 spring-boot 2.3+ template v1.0 环境 +# +# 使用 application.yml 作为项目配置文件 +# 脚本将环境变量中 __SPUG__开头的配置,写入文件中 +# +# mark_zhou[zz.mark06@gmail.com] + +set -e + +_DEPLOY_DIR=$(cd `dirname $0`; pwd) +WORKSPACE_DIR=$(pwd) + +if [ ! -n "$1" ];then + echo "需要输入 应用-标识符" + exit 1 +fi + +APP_TAG=$1 +APP_TAG=${APP_TAG^^} +APP_TAG=${APP_TAG/-/_} +echo "应用标识符: ${APP_TAG}" + +ENV_NAME=_SPUG_${APP_TAG}_APPLICATION_YML +echo "ENV_NAME: ${ENV_NAME}" +APPLICATION_YML="$(eval echo '$'$ENV_NAME})" +echo "json: ${APPLICATION_YML}" + +YAML=$(echo "${APPLICATION_YML}" | docker run -i --rm simplealpine/json2yaml) +echo "配置文件 application.yml" +echo ${YAML} + +cat < ${WORKSPACE_DIR}/application.yml +${YAML} +EOF diff --git a/_deploy/update.sh.template b/_deploy/update.sh.template new file mode 100644 index 0000000..bde018a --- /dev/null +++ b/_deploy/update.sh.template @@ -0,0 +1,51 @@ +#!/bin/bash +# 更新服务并检测可用性 +# usage: +# ./update.sh ${service_name} + +set -e + +SERVICE_NAME=\$1 + +if [ ! -n "\$1" ];then + SERVICE_NAME=${__SERVICE_NAME} +fi + +HEALTHY="healthy" +STARTING="starting" +UNHEALTHY="unhealthy" + +# update +docker-compose -p \${SERVICE_NAME} up -d + +# check healthy +if [[ \${SERVICE_NAME} = '' ]]; then + echo "ignore healthy check." + exit 0 +fi + +RESIDUE=20 +while(( \${RESIDUE} > 0 )) +do + let "RESIDUE--" + set +e + STATUS=\$(docker inspect --format '{{.State.Health.Status}}' \${SERVICE_NAME}_\${SERVICE_NAME}_1) + CODE=\$? + set -e + if [[ \${CODE} = 1 ]]; then + echo "该服务不支持健康检查" + exit 1 + fi + if [[ \${STATUS} = \${STARTING} ]]; then + echo "Health check: \${STATUS}" + sleep 3 + elif [[ \${STATUS} = \${HEALTHY} ]]; then + echo \${STATUS} + exit 0 + else + echo "error! status: \${STATUS}" + exit 1 + fi +done +echo "TIMEOUT! 一分钟服务未进入健康状态,请手动检查状态. Last status: \${STATUS}" +exit 0 diff --git a/build/settings.xml b/build/settings.xml new file mode 100644 index 0000000..ee58beb --- /dev/null +++ b/build/settings.xml @@ -0,0 +1,19 @@ + + + + + + huaweicloud + repo1,repo,central + https://mirrors.huaweicloud.com/repository/maven/ + + + + /usr/share/maven/ref/repository + diff --git a/build/startup-springboot.sh b/build/startup-springboot.sh new file mode 100644 index 0000000..5f15e25 --- /dev/null +++ b/build/startup-springboot.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# 启动脚本 +# 该脚本部分取自 vjkit,都做了注释,按需引用 +# 仅支持 java 11+ +# +# 支持的环境变量: +# - $NAME 应用名称 +# - $JAVA_OPTS jvm 参数 +# - $JAVA_AGENT jvm agent + +# 基本参数 +# appid 关系到日志文件名和一些其他的设置 +APPID=${NAME:-application} + +BASEPATH=$(cd `dirname $0`; pwd) +JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}') + +# ************************************************************** + +# ************************** 以下JVM部分 ************************ + +# Enable coredump 容器环境暂不清楚是否可用,观望一手 +# ulimit -c unlimited + +## Memory Options, 根据实际情况进行调整,建议为内存总量一半, 容器环境,靠容器本体限制 +# MEM_OPTS="-Xmx512m -Xms512m -XX:NewRatio=1 -XX:MetaspaceSize=192M -XX:MaxMetaspaceSize=192M" + +# 启动时预申请内存 +# MEM_OPTS="$MEM_OPTS -XX:+AlwaysPreTouch" + +# 如果线程数较多,函数的递归较少,线程栈内存可以调小节约内存,默认1M。 +MEM_OPTS="$MEM_OPTS -Xss512k" + +# 堆外内存的最大值默认约等于堆大小,可以显式将其设小,获得一个比较清晰的内存总量预估 +#MEM_OPTS="$MEM_OPTS -XX:MaxDirectMemorySize=2g" + +# 根据JMX/VJTop的观察,调整二进制代码区大小避免满了之后不能再JIT,JDK7/8,是否打开多层编译的默认值都不一样 +#MEM_OPTS="$MEM_OPTS -XX:ReservedCodeCacheSize=240M" + + +## GC Options## + +GC_OPTS="-XX:+UseG1GC" + +# System.gc() 使用并行算法 +GC_OPTS="$GC_OPTS -XX:+ExplicitGCInvokesConcurrent" + +# 根据应用的对象生命周期设定,减少事实上的老生代对象在新生代停留时间,加快YGC速度 +# GC_OPTS="$GC_OPTS -XX:MaxTenuringThreshold=3" + +# 如果OldGen较大,加大YGC时扫描OldGen关联的卡片,加快YGC速度,默认值256较低 +# GC_OPTS="$GC_OPTS -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=1024" + +# 如果JVM并不独占机器,机器上有其他较繁忙的进程在运行,将GC线程数设置得比默认值(CPU核数*5/8 )更低以减少竞争,反而会大大加快YGC速度。 +#GC_OPTS="$GC_OPTS -XX:ParallelGCThreads=12 -XX:ConcGCThreads=6" + + +## GC log Options, only for JDK7/JDK8 ## + +# 默认使用/dev/shm 内存文件系统避免在高IO场景下写GC日志时被阻塞导致STW时间延长 +if [ -d /dev/shm/ ]; then + GC_LOG_FILE=/dev/shm/gc-${APPID}.log +else + mkdir -p ${BASEPATH}/gc/ + GC_LOG_FILE=${BASEPATH}/gc/gc-${APPID}.log +fi + +if [ -f ${GC_LOG_FILE} ]; then + GC_LOG_BACKUP=${BASEPATH}/gc/gc-${APPID}-$(date +'%Y%m%d_%H%M%S').log + echo "saving gc log ${GC_LOG_FILE} to ${GC_LOG_BACKUP}" + mkdir -p ${BASEPATH}/gc/ + mv ${GC_LOG_FILE} ${GC_LOG_BACKUP} +fi + +#打印GC日志,包括时间戳,晋升老生代失败原因,应用实际停顿时间(含GC及其他原因) 暂且没摸清java11该怎么打印 +#GCLOG_OPTS="-Xloggc:${GC_LOG_FILE} -XX:+PrintGCDetails -Xlog::::time,level,tags,safepoint" + + +# 打印安全点日志,找出GC日志里非GC的停顿的原因,会损失很多性能 +#GCLOG_OPTS="$GCLOG_OPTS -XX:+PrintSafepointStatistics -XX:PrintSafepointStatisticsCount=1 -XX:+UnlockDiagnosticVMOptions -XX:-DisplayVMOutput -XX:+LogVMOutput -XX:LogFile=/dev/shm/vm-${APPID}.log" + + +## Optimization Options## + +OPTIMIZE_OPTS="-XX:-UseBiasedLocking -XX:AutoBoxCacheMax=20000 -Djava.security.egd=file:/dev/./urandom" + + +# 关闭PerfData写入,避免高IO场景GC时因为写PerfData文件被阻塞,但会使得jstats,jps不能使用 +#OPTIMIZE_OPTS="$OPTIMIZE_OPTS -XX:+PerfDisableSharedMem" + +# 关闭多层编译,减少应用刚启动时的JIT导致的可能超时,以及避免部分函数C1编译后最终没被C2编译。 但导致函数没有被初始C1编译。 +#if [[ "$JAVA_VERSION" > "1.8" ]]; then +# OPTIMIZE_OPTS="$OPTIMIZE_OPTS -XX:-TieredCompilation" +#fi + +# 如果希望无论函数的热度如何,最终JIT所有函数,关闭GC时将函数调用次数减半。 +#OPTIMIZE_OPTS="$OPTIMIZE_OPTS -XX:-UseCounterDecay" + +## Trouble shooting Options## +mkdir -p ${BASEPATH}/jvm/ +SHOOTING_OPTS="-XX:+PrintCommandLineFlags -XX:-OmitStackTraceInFastThrow -XX:ErrorFile=${BASEPATH}/jvm/hs_err_%p.log" + + +# OOM 时进行HeapDump,但此时会产生较高的连续IO,如果是容器环境,有可能会影响他的容器 +# mkdir -p ${BASEPATH}/dump/ +# SHOOTING_OPTS="$SHOOTING_OPTS -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${BASEPATH}/dump/" + + +# 在非生产环境,打开JFR进行性能记录(生产环境要收License) +#SHOOTING_OPTS="$SHOOTING_OPTS -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:+UnlockDiagnosticVMOptions -XX:+DebugNonSafepoints" + + +## JMX Options## + +#开放JMX本地访问,设定端口号,可使用 jconsole 介入。容器场景不能限定 localhost -Djava.rmi.server.hostname=127.0.0.1 +JMX_OPTS="-Dcom.sun.management.jmxremote.port=9012 -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false" + + +## Other Options## + +## 远程 debug +# OTHER_OPTS="-Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005" + + +## All together ## + +JAVA_OPTS=${JAVA_OPTS:-"$MEM_OPTS $GC_OPTS $GCLOG_OPTS $OPTIMIZE_OPTS $SHOOTING_OPTS $JMX_OPTS $OTHER_OPTS"} +echo JAVA_OPTS=$JAVA_OPTS + +# **************************************************************** + +exec java $JAVA_OPTS $JAVA_AGENT org.springframework.boot.loader.JarLauncher "$@" diff --git a/pom.xml b/pom.xml index 759e94e..f74551b 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.3.1.RELEASE + 2.5.3 top.zzmark @@ -24,6 +24,21 @@ spring-boot-starter-web + + cn.hutool + hutool-all + 5.7.9 + + + com.github.oshi + oshi-core + 5.6.1 + + + org.springframework.boot + spring-boot-starter-actuator + + org.springframework.boot spring-boot-devtools diff --git a/src/main/java/top/zzmark/echo/EchoApplication.java b/src/main/java/top/zzmark/echo/EchoApplication.java index 3b32ec2..4e63cc8 100644 --- a/src/main/java/top/zzmark/echo/EchoApplication.java +++ b/src/main/java/top/zzmark/echo/EchoApplication.java @@ -1,11 +1,28 @@ package top.zzmark.echo; +import org.apache.catalina.filters.RemoteIpFilter; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.context.annotation.Bean; @SpringBootApplication public class EchoApplication { + /** + * 识别反向代理的 X-Forwarded-* 标记,获取客户端的实际 ip + * 需要上层转发层的支持。 + *

+ * 该 filter 不遵循 RFC 7239, spring-mvc 提供了一个名为 ForwardedFilter 的拦截器,但并不知道该怎么用 + *

+ * webflux 有更好的解决方案,不需要这个 + */ + @Bean + @ConditionalOnClass(RemoteIpFilter.class) + public RemoteIpFilter remoteIpFilter() { + return new RemoteIpFilter(); + } + public static void main(String[] args) { SpringApplication.run(EchoApplication.class, args); } diff --git a/src/main/java/top/zzmark/echo/controller/Controller.java b/src/main/java/top/zzmark/echo/controller/Controller.java index f11eaa0..59e2d3a 100644 --- a/src/main/java/top/zzmark/echo/controller/Controller.java +++ b/src/main/java/top/zzmark/echo/controller/Controller.java @@ -1,5 +1,8 @@ package top.zzmark.echo.controller; +import cn.hutool.system.SystemUtil; +import cn.hutool.system.oshi.CpuInfo; +import cn.hutool.system.oshi.OshiUtil; import org.springframework.web.bind.annotation.*; import top.zzmark.echo.entity.EchoEntity; import top.zzmark.echo.util.CommandUtil; @@ -9,6 +12,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.HashMap; import java.util.Map; /** @@ -21,25 +25,50 @@ public class Controller { @RequestMapping("echo") - public EchoEntity echo(HttpServletRequest request, @RequestHeader(required = false) Map header, - @RequestParam(required = false) Map param, @RequestBody(required = false) Map body) + public EchoEntity echo(HttpServletRequest request, + @RequestHeader(required = false) Map header, + @RequestParam(required = false) Map param, + @RequestBody(required = false) Map body) throws IOException { + return EchoEntity.builder() .clientIp(IpUtil.getIpAddr(request)) .serverIp(InetAddress.getLocalHost().getHostAddress()) - .serverInfo(CommandUtil.run("cat /etc/os-release", 0)) .requestHeader(header) .requestBody(body) .requestParam(param) .build(); } + @GetMapping("env") + public Map env() throws IOException { + return System.getenv(); + } + + @GetMapping("systemInfo") + public Map systemInfo() throws IOException { + Map map = new HashMap<>(); + map.put("JvmSpecInfo", SystemUtil.getJvmSpecInfo()); + map.put("JvmInfo", SystemUtil.getJvmInfo()); + map.put("JavaSpecInfo", SystemUtil.getJvmSpecInfo()); + map.put("JavaInfo", SystemUtil.getJavaInfo()); + map.put("JavaRuntimeInfo", SystemUtil.getJavaRuntimeInfo()); + map.put("OsInfo", SystemUtil.getOsInfo()); + map.put("UserInfo", SystemUtil.getUserInfo()); + map.put("HostInfo", SystemUtil.getHostInfo()); + map.put("RuntimeInfo", SystemUtil.getRuntimeInfo().toString()); + CpuInfo cpuInfo = OshiUtil.getCpuInfo(); + map.put("cpuInfo", cpuInfo); + return map; + } + @GetMapping("ping") public String ping(@RequestParam("ip") String ip, @RequestParam(value = "s", required = false) Integer s) throws IOException { return CommandUtil.run("ping " + ip); } + @Deprecated @GetMapping("shell") public String shell(@RequestParam("shell") String shell, @RequestParam(value = "s", required = false) Integer s) throws IOException { diff --git a/src/main/java/top/zzmark/echo/controller/ExecController.java b/src/main/java/top/zzmark/echo/controller/ExecController.java new file mode 100644 index 0000000..86dc8ab --- /dev/null +++ b/src/main/java/top/zzmark/echo/controller/ExecController.java @@ -0,0 +1,120 @@ +package top.zzmark.echo.controller; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.io.*; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static cn.hutool.core.io.FileUtil.isWindows; + +/** + * @author mark_zhou[zz.mark06@gmail.com] + * @date 2021-08-19 11:56 + **/ +@RestController +public class ExecController { + + private static final Logger logger = LoggerFactory.getLogger(ExecController.class); + + @GetMapping("exec") + public String shell( + @RequestParam("shell") String shell, + @RequestParam(value = "s", required = false) Long s + ) throws IOException { + String preCmd = ""; + String encoding; + if (isWindows()) { + // 对 windows 需要点特别处理 + encoding = "gb2312"; + preCmd = "cmd /C "; + } else { + // linux 是否需要,得测试一下 + encoding = "utf-8"; + } + String[] exec = (preCmd + shell).split(" "); + return execToString(exec, encoding, s, TimeUnit.SECONDS); + } + + public static boolean exec(String[] cmdArray, String[] envp, File dir, + OutputStream stdout, OutputStream stderr, long timeout, TimeUnit timeUnit) throws IOException { + + StringBuilder sb = new StringBuilder(); + sb.append("execute : "); + for (String s : cmdArray) { + sb.append(s).append(" "); + } + sb.append("\n in ").append(dir.getAbsolutePath()); + logger.info(sb.toString()); + Process p = Runtime.getRuntime().exec(cmdArray, envp, dir); + + Thread _stdout = new Thread(new StreamReader(p.getInputStream(), stdout)); + Thread _stderr = new Thread(new StreamReader(p.getErrorStream(), stderr)); + _stdout.setDaemon(true); + _stderr.setDaemon(true); + + _stdout.start(); + _stderr.start(); + + boolean ret = false; + try { + ret = p.waitFor(timeout, timeUnit); + _stdout.join(timeUnit.toMillis(timeout)); + _stderr.join(timeUnit.toMillis(timeout)); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return ret; + } + + public static String execToString(String[] cmdArray, String encoding, long timeout, TimeUnit timeUnit) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try { + boolean ret = exec(cmdArray, getEnvs(), new File(".").getAbsoluteFile(), out, out, timeout, timeUnit); + out.close(); + if (ret) { + return out.toString(encoding); + } + return out.toString(encoding); + } catch (Exception e) { + logger.error("执行失败", e); + return e.getMessage(); + } + } + + public static String[] getEnvs() { + String[] envs = new String[System.getenv().size()]; + int c = 0; + for (Map.Entry entry : System.getenv().entrySet()) { + envs[c++] = entry.getKey() + "=" + entry.getValue(); + } + return envs; + } + + public static class StreamReader implements Runnable { + InputStream input; + OutputStream output; + + private StreamReader(InputStream input, OutputStream output) { + this.input = input; + this.output = output; + } + + @Override + public void run() { + try { + int c; + byte[] buff = new byte[1024]; + while ((c = input.read(buff)) > 0) { + output.write(buff, 0, c); + } + } catch (IOException e) { + logger.error("", e); + } + } + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties deleted file mode 100644 index 08c8822..0000000 --- a/src/main/resources/application.properties +++ /dev/null @@ -1,2 +0,0 @@ -spring.application.name=echo -server.port=60001 \ No newline at end of file diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml new file mode 100644 index 0000000..e9c5a8b --- /dev/null +++ b/src/main/resources/application.yml @@ -0,0 +1,33 @@ +spring.application.name: echo + +server: + port: 60001 + +management: + server: + port: 60002 # ActuatorWeb访问端口 + endpoint: + health: + show-details: always + probes: + enabled: true + prometheus: + enabled: true + endpoints: + jmx: + exposure: + include: '*' + web: + exposure: + include: '*' + metrics: + tags: + application: ${spring.application.name} + info: + git: # https://github.com/git-commit-id/git-commit-id-maven-plugin 显示 git 提交信息 + mode: full +logging: + pattern: + dateformat: 'yyyy-MM-dd HH:mm:ss.SSS,Asia/Shanghai' + file: + name: ./logs/${spring.application.name}.log diff --git a/src/main/resources/public/index.html b/src/main/resources/public/index.html new file mode 100644 index 0000000..35c7bab --- /dev/null +++ b/src/main/resources/public/index.html @@ -0,0 +1,137 @@ + + + + + + Echo + + + + +

+
+ + + + {{ item.val }} + + + + + + {{ item.val + }} + + + + + + + + + + + + + + 执行 + + + + {{ cmdRes }} + + + + +
+
+ + + + + +