前情提要
佬需要扩展我司调度平台(Dolphinscheduler)集群的节点数,故有了下面的脚本
部分脚本经过 AI 生成和自行调试,不能发的话管理可以删除
PS: Dolphinscheduler 是一个很不错的开源分布式调度平台,有大量调度任务的佬可以看看
就是这个了
GitHub - apache/dolphinscheduler: Apache DolphinScheduler is the modern data orchestration platform. Agile to create high performance workflow with low-code
创建用户
总结
sudo useradd user1
echo "user1@123" | sudo passwd --stdin user1
sudo sed -i '$auser1 ALL=(ALL) NOPASSWD: ALL' /etc/sudoers
设置 host 映射
总结
base=10.1.xxx.
HOSTS="${base}1 ${base}2 ${base}3"
USER="user1"
PASS="user1@2023" cat > hosts_to_add.tmp << EOF
10.1.xxx.1 node1
10.1.xxx.2 node2
10.1.xxx.3 node3
.... # 此处省略n个
EOF
CLEAN_NEW_NODES_PATTERN="node1|node2|node3|..." if ! command -v sshpass &> /dev/null; then echo "❌ 请先安装 sshpass: yum install sshpass -y" exit 1
fi echo "========================================" echo "开始批量修改 /etc/hosts" echo "========================================" for IP in $HOSTS; do echo ">> 正在处理节点: $IP"
sshpass -p "$PASS" scp -o StrictHostKeyChecking=no hosts_to_add.tmp "$USER@$IP:/tmp/hosts_chunk" if [ $? -ne 0 ]; then echo " ❌ 上传文件失败,跳过该节点" continue fi
REMOTE_CMD="
echo '$PASS' | sudo -S -p '' bash -c '
cp /etc/hosts /etc/hosts.bak_\$(date +%Y%m%d_%H%M) && \
sed -i -E \"/$CLEAN_NEW_NODES_PATTERN/d\" /etc/hosts && \
cat /tmp/hosts_chunk >> /etc/hosts && \
rm -f /tmp/hosts_chunk
'
"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" "$REMOTE_CMD" if [ $? -eq 0 ]; then echo " ✅ 修改成功"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" "grep -c 'node1' /etc/hosts" | xargs -I {} echo " 📊 验证: 包含 node1 的行数: {} (应为 1)" else echo " ❌ 修改失败 (可能是 sudo 密码错误或权限不足)" fi echo done rm -f hosts_to_add.tmp
echo "全部完成。"
配置节点间免密连接
总结
#!/bin/bash
base=10.1.xxx.
HOSTS="${base}1 ${base}2 ${base}3"
USER="user1"
PASS="user1@2023"
ALL_KEYS_FILE="./all_node_keys.tmp"
> "$ALL_KEYS_FILE" if ! command -v sshpass &> /dev/null; then echo "[Init] 正在安装 sshpass..." if [ -f /etc/redhat-release ]; then sudo yum install sshpass -y
elif [ -f /etc/debian_version ]; then sudo apt-get install sshpass -y
else echo "❌ 请先手动安装 sshpass" exit 1
fi fi echo "========================================" echo "阶段一:生成密钥并收集公钥" echo "========================================" for IP in $HOSTS; do echo ">> [收集] 正在处理节点: $IP"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" \
"if [ ! -f ~/.ssh/id_rsa ]; then ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa; fi"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" "cat ~/.ssh/id_rsa.pub" >> "$ALL_KEYS_FILE" echo "" >> "$ALL_KEYS_FILE" done if [ ! -s "$ALL_KEYS_FILE" ]; then echo "❌ 未收集到公钥,请检查网络或密码。" exit 1
fi echo "" echo "========================================" echo "阶段二:安全追加配置 (保留原有密钥)" echo "========================================" for IP in $HOSTS; do echo ">> [追加] 正在同步到节点: $IP"
sshpass -p "$PASS" scp -o StrictHostKeyChecking=no "$ALL_KEYS_FILE" "$USER@$IP:/tmp/new_keys_bundle"
REMOTE_CMD="
mkdir -p ~/.ssh && \
touch ~/.ssh/authorized_keys && \
sed -i -e '\$a\\' ~/.ssh/authorized_keys && \
cat /tmp/new_keys_bundle >> ~/.ssh/authorized_keys && \
sort -u ~/.ssh/authorized_keys -o ~/.ssh/authorized_keys && \
sed -i '/^$/d' ~/.ssh/authorized_keys && \
chmod 700 ~/.ssh && \
chmod 600 ~/.ssh/authorized_keys && \
rm -f /tmp/new_keys_bundle
"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" "$REMOTE_CMD" if [ $? -eq 0 ]; then echo " ✅ 成功追加并去重" else echo " ❌ 操作失败" fi done rm -f "$ALL_KEYS_FILE" echo "" echo "========================================" echo "全部完成!原有密钥已保留,新节点已互信。" echo "验证方法: ssh $USER@${base}1 'ssh ${base}2 date'" echo "========================================"
配置 Dolphinscheduler 的系统服务
总结
#!/bin/bash
base=10.1.xxx.
HOSTS="${base}1 ${base}2 ${base}3"
USER="user1"
PASS="user1@2023"
LOCAL_SERVICE_FILE="./ds_worker.service" if ! command -v sshpass &> /dev/null; then echo "❌ 请先安装 sshpass" exit 1
fi cat > "$LOCAL_SERVICE_FILE" << 'EOF'
[Unit]
Description=DolphinScheduler worker Service
After=network.target
[Service]
Type=forking
User=user1
Environment=JAVA_HOME=/mnt/vos-jddsq55k/software/jdk1.8.0_152
WorkingDirectory=/mnt/vos-jddsq55k/software/dss320
ExecStart=/mnt/vos-jddsq55k/software/dss320/bin/dolphinscheduler-daemon.sh start worker-server
ExecStop=/mnt/vos-jddsq55k/software/dss320/bin/dolphinscheduler-daemon.sh stop worker-server
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
Alias=ds-worker.service
EOF
echo "========================================" echo "批量部署 ds-worker 服务" echo "========================================" for IP in $HOSTS; do echo ">> 正在处理节点: $IP"
sshpass -p "$PASS" scp -o StrictHostKeyChecking=no "$LOCAL_SERVICE_FILE" "$USER@$IP:/tmp/ds_worker.service" if [ $? -ne 0 ]; then echo " ❌ 文件上传失败" continue fi
REMOTE_CMD="
echo '$PASS' | sudo -S -p '' bash -c '
mv /tmp/ds_worker.service /usr/lib/systemd/system/ds_worker.service && \
chmod 644 /usr/lib/systemd/system/ds_worker.service && \
systemctl daemon-reload && \
systemctl enable ds_worker && \
systemctl restart ds_worker
' && \
systemctl status ds_worker | grep 'Active:'
"
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" "$REMOTE_CMD" if [ $? -eq 0 ]; then echo " ✅ 部署并启动成功" else echo " ❌ 启动失败,请检查路径或权限" fi echo done rm -f "$LOCAL_SERVICE_FILE" echo "全部完成。"
配置日志清理任务
总结
#!/bin/bash
base=10.1.xxx.
HOSTS="${base}1 ${base}2 ${base}3"
USER="user1"
PASS="user1@2023"
TASK_MASTER="0 0 * * * find /mnt/vos-jddsq55k/software/dss320/master-server/logs -mtime +7 -exec rm -rf {} \;"
TASK_WORKER="0 0 * * * find /mnt/vos-jddsq55k/software/dss320/worker-server/logs -mtime +30 -exec rm -rf {} \;" if ! command -v sshpass &> /dev/null; then echo "❌ 请先安装 sshpass" exit 1
fi echo "========================================" echo "批量添加日志清理 Crontab 任务" echo "========================================" for IP in $HOSTS; do echo ">> 正在处理节点: $IP"
REMOTE_SCRIPT=$(cat << EOF
# 1. 导出当前 crontab 到临时文件 (如果没有任务则创建一个空文件)
crontab -l > /tmp/cron.tmp 2>/dev/null || touch /tmp/cron.tmp
# 2. 定义标志位,看是否有修改
MODIFIED=0
# 3. 检查并添加 Master 清理任务
# grep -F: 固定字符串匹配,忽略星号的特殊含义
if ! grep -Fq "master-server/logs" /tmp/cron.tmp; then
echo "$TASK_MASTER" >> /tmp/cron.tmp
echo " + 已添加 Master 日志清理任务"
MODIFIED=1
else
echo " = Master 任务已存在,跳过"
fi
# 4. 检查并添加 Worker 清理任务
if ! grep -Fq "worker-server/logs" /tmp/cron.tmp; then
echo "$TASK_WORKER" >> /tmp/cron.tmp
echo " + 已添加 Worker 日志清理任务"
MODIFIED=1
else
echo " = Worker 任务已存在,跳过"
fi
# 5. 如果有修改,则重新加载 Crontab
if [ \$MODIFIED -eq 1 ]; then
crontab /tmp/cron.tmp
echo " ✅ Crontab 更新成功"
else
echo " ok 无需变更"
fi
# 6. 清理临时文件
rm -f /tmp/cron.tmp
# 7. 验证:显示当前的 crontab 内容
echo " 👀 当前 Crontab 预览:"
crontab -l
EOF
)
sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "$USER@$IP" bash -c "'$REMOTE_SCRIPT'" echo done
一些其他常用 Linux 命令的总结
查看占用 CPU 最多进程
ps aux|head -1;ps aux|grep -v PID|sort -rn -k +3|head
ps -aux | sort -k3nr | head 5
linux 下获取占用内存资源最多
ps aux|head -1;ps aux|grep -v PID|sort -rn -k +4|head
ps -aux | sort -k4nr | head 5
查看程序运行时间
ps -eo pid,user,command,lstart,etime|grep xxx
ps -p PID -o lstart,etime,command
查看程序内存溢出记录
grep "Out of memory" /var/log/messages
查看开机自启服务列表
systemctl list-unit-files|grep enable
释放内存
echo 1 > /proc/sys/vm/drop_caches
按关键字搜索文件
grep -rl "key word" /path
批量替换文本内容
sed -i "s/demomanage.wbsjk.com/devrr.aoyuntech.com/g" `grep demomanage.wbsjk.com -rl .`
插入内容到文件头
sed -i '1i\export JAVA_HOME=\/usr\/local\/openjdk-8' /etc/profile
Redis 批量删除
redis-cli -h 172.17.48.144 -p 63791 -n 5 keys *BEAU*|xargs redis-cli -p 63791 -h 172.17.48.144 -n 5 del
删除大批量数据
ls|xargs -n 10 rm -rf ls
删除 3 天以前文件
upload_path=/tmp/upload
download_path=/tmp/download
find $upload_path -mtime +3|xargs -d '\n' rm -rf {} \;
find $download_path -mtime +3|xargs -d '\n' rm -rf {} \;
追加文件
cat >> ~/.bashrc << EOF
export JAVA_HOME=/opt/path/java/jdk1.8.0_152
export PATH=$JAVA_HOME/bin:$PATH:.
export CLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar:.
EOF
觉得有用的佬可以点个赞,打赏一下也是可以的哈哈
📌 转载信息
原作者:
zzzxdm
转载时间:
2026/1/9 10:07:17