actions
/
ssh-scp-ssh-pipelines
mirror of https://github.com/cross-the-world/ssh-scp-ssh-pipelines.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: actions:master
Branches Tags
actions:latest
actions:master
actions:v1.1.4
actions:v1.1.3
actions:v1.1.2
actions:v1.1.1
actions:v1.1.0
actions:v1.0.3
actions:v1.0.2
actions:v1.0.1
actions:v1.0.0
..
compare: actions:v1.0.0
Branches Tags
actions:latest
actions:master
actions:v1.1.4
actions:v1.1.3
actions:v1.1.2
actions:v1.1.1
actions:v1.1.0
actions:v1.0.3
actions:v1.0.2
actions:v1.0.1
actions:v1.0.0

No commits in common. "master" and "v1.0.0" have entirely different histories.
master ... v1.0.0

7 changed files with 193 additions and 267 deletions
Show Stats Expand all files Collapse all files

60
.github/workflows/deploy.yml vendored
View File

@ -1,11 +1,5 @@
name: scp files
on:
push:
branches:
- latest
env:
TARGET_DIR: /home/github/test
on: [push]
jobs:
build:
@ -17,54 +11,52 @@ jobs:
uses: actions/checkout@v1
- name: ssh scp ssh pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
env:
WELCOME: "ssh scp ssh pipelines"
LASTSSH: "Doing something after copying"
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
port: ${{ secrets.DC_PORT }}
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
port: {{ secrets.DC_PORT }}
connect_timeout: 10s
first_ssh: |-
first_ssh: |
rm -rf /home/github/test
ls -la
echo $WELCOME
ls -la \necho $WELCOME
mkdir -p /home/github/test/test1 &&
mkdir -p /home/github/test/test2 &&
scp: |-
scp: |
'./test/*' => /home/github/test/
./test/test1* => $TARGET_DIR/test1/
./test/test1* => /home/github/test/test1/
./test/test*.csv => "/home/github/test/test2/"
last_ssh: |-
last_ssh: |
echo $LASTSSH &&
(mkdir test1/test || true)
ls -la
|| ls -la
- name: scp ssh pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
env:
LASTSSH: "Doing something after copying"
WELCOME: "scp ssh pipelines"
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
scp: |-
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
scp: |
./test/test1* => /home/github/test/test1/
"." => "$TARGET_DIR/test3/"
last_ssh: |-
./test/test*.csv => "/home/github/test/test2/"
last_ssh: |
echo $LASTSSH
ls -la
- name: scp pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
env:
WELCOME: "scp pipelines"
TO_DIR: /home/github/test/test4
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
scp: |-
'.' => $TO_DIR
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
scp: |
'./test/*' => /home/github/test/

2
.gitignore vendored
View File

@ -1,4 +1,4 @@
.idea
test.*
test.sh
.env

28
Dockerfile
View File

@ -1,26 +1,22 @@
FROM python:3.8.3-slim-buster
FROM alpine:latest
LABEL "maintainer"="Scott Ng <thuongnht@gmail.com>"
LABEL "repository"="https://github.com/cross-the-world/ssh-scp-ssh-pipelines"
LABEL "version"="v1.1.0"
LABEL "version"="1.0.0"
LABEL "com.github.actions.name"="ssh-scp-ssh-pipelines"
LABEL "com.github.actions.description"="Pipeline: ssh -> scp -> ssh"
LABEL "com.github.actions.description"="Pipelines: ssh -> scp -> ssh"
LABEL "com.github.actions.icon"="terminal"
LABEL "com.github.actions.color"="gray-dark"
RUN apt-get update -y && \
apt-get install -y ca-certificates openssh-client openssl sshpass
RUN apk update && \
apk add ca-certificates && \
apk add --no-cache openssh-client openssl openssh sshpass && \
apk add --no-cache --upgrade bash openssh sshpass && \
rm -rf /var/cache/apk/*
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
COPY entrypoint.sh /entrypoint.sh
COPY test /opt/test
RUN chmod +x /entrypoint.sh
RUN mkdir -p /opt/tools
COPY entrypoint.sh /opt/tools/entrypoint.sh
RUN chmod +x /opt/tools/entrypoint.sh
COPY app.py /opt/tools/app.py
RUN chmod +x /opt/tools/app.py
ENTRYPOINT ["/opt/tools/entrypoint.sh"]
ENTRYPOINT ["/entrypoint.sh"]

41
README.md
View File

@ -1,15 +1,11 @@
# SSH SCP SSH Pipelines
[Github actions](https://help.github.com/en/actions/creating-actions/creating-a-docker-container-action)
Credit to [SSH SCP Action of Ali Najafizadeh](https://github.com/alinz/ssh-scp-action)
[SSH action](https://github.com/cross-the-world/ssh-pipeline)
[SCP action](https://github.com/cross-the-world/scp-pipeline)
This action allows doing in order
1. ssh if defined
2. scp if defined
3. ssh if defined
This action allows doing
1. ssh
2. scp
3. ssh
## Inputs
see the [action.yml](./action.yml) file for more detail imformation.
@ -57,20 +53,19 @@ e.g.
## Usages
see the [deploy.yml](./.github/workflows/deploy.yml) file for more detail imformation.
#### ssh scp ssh pipelines
```yaml
- name: ssh scp ssh pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
env:
WELCOME: "ssh scp ssh pipelines"
LASTSSH: "Doing something after copying"
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
port: ${{ secrets.DC_PORT }}
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
port: {{ secrets.DC_PORT }}
connect_timeout: 10s
first_ssh: |
rm -rf /home/github/test
@ -90,13 +85,13 @@ see the [deploy.yml](./.github/workflows/deploy.yml) file for more detail imform
#### scp ssh pipelines
```yaml
- name: scp ssh pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
env:
LASTSSH: "Doing something after copying"
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
scp: |
./test/test1* => /home/github/test/test1/
./test/test*.csv => "/home/github/test/test2/"
@ -108,11 +103,11 @@ see the [deploy.yml](./.github/workflows/deploy.yml) file for more detail imform
#### scp pipelines
```yaml
- name: scp pipelines
uses: cross-the-world/ssh-scp-ssh-pipelines@latest
uses: cross-the-world/ssh-scp-ssh-pipelines@v1.0.0
with:
host: ${{ secrets.DC_HOST }}
user: ${{ secrets.DC_USER }}
pass: ${{ secrets.DC_PASS }}
host: {{ secrets.DC_HOST }}
user: {{ secrets.DC_USER }}
pass: {{ secrets.DC_PASS }}
scp: |
'./test/*' => /home/github/test/
```

185
app.py
View File

@ -1,185 +0,0 @@
from os import environ, path
from glob import glob
import paramiko
import scp
import sys
import math
import re
import tempfile
import os
envs = environ
INPUT_HOST = envs.get("INPUT_HOST")
INPUT_PORT = int(envs.get("INPUT_PORT", "22"))
INPUT_USER = envs.get("INPUT_USER")
INPUT_PASS = envs.get("INPUT_PASS")
INPUT_KEY = envs.get("INPUT_KEY")
INPUT_CONNECT_TIMEOUT = envs.get("INPUT_CONNECT_TIMEOUT", "30s")
INPUT_SCP = envs.get("INPUT_SCP")
INPUT_FIRST_SSH = envs.get("INPUT_FIRST_SSH")
INPUT_LAST_SSH = envs.get("INPUT_LAST_SSH")
seconds_per_unit = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "M": 86400*30}
pattern_seconds_per_unit = re.compile(r'^(' + "|".join(['\\d+'+k for k in seconds_per_unit.keys()]) + ')$')
def convert_to_seconds(s):
if s is None:
return 30
if isinstance(s, str):
return int(s[:-1]) * seconds_per_unit[s[-1]] if pattern_seconds_per_unit.search(s) else 30
if (isinstance(s, int) or isinstance(s, float)) and not math.isnan(s):
return round(s)
return 30
strips = [" ", "\"", " ", "'", " "]
def strip_and_parse_envs(p):
if not p:
return None
for c in strips:
p = p.strip(c)
return path.expandvars(p) if p != "." else f"{path.realpath(p)}/*"
def connect(callback=None):
tmp = tempfile.NamedTemporaryFile(delete=False)
try:
ssh = paramiko.SSHClient()
p_key = None
if INPUT_KEY:
tmp.write(INPUT_KEY.encode())
tmp.close()
p_key = paramiko.RSAKey.from_private_key_file(filename=tmp.name)
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(INPUT_HOST, port=INPUT_PORT, username=INPUT_USER,
pkey=p_key, password=INPUT_PASS,
timeout=convert_to_seconds(INPUT_CONNECT_TIMEOUT))
except Exception as err:
print(f"Connect error\n{err}")
sys.exit(1)
else:
if callback:
callback(ssh)
finally:
os.unlink(tmp.name)
tmp.close()
# Define progress callback that prints the current percentage completed for the file
def progress(filename, size, sent):
sys.stdout.write(f"{filename}... {float(sent)/float(size)*100:.2f}%\n")
def ssh_process(ssh, input_ssh):
commands = [c.strip() for c in input_ssh.splitlines() if c is not None]
command_str = ""
l = len(commands)
for i in range(len(commands)):
c = path.expandvars(commands[i])
if c == "":
continue
if c.endswith('&&') or c.endswith('||') or c.endswith(';'):
c = c[0:-2] if i == (l-1) else c
else:
c = f"{c} &&" if i < (l-1) else c
command_str = f"{command_str} {c}"
command_str = command_str.strip()
print(command_str)
stdin, stdout, stderr = ssh.exec_command(command_str)
ssh_exit_status = stdout.channel.recv_exit_status()
out = "".join(stdout.readlines())
out = out.strip() if out is not None else None
if out:
print(f"Success: \n{out}")
err = "".join(stderr.readlines())
err = err.strip() if err is not None else None
if err:
print(f"Error: \n{err}")
if ssh_exit_status != 0:
print(f"ssh exit status: {ssh_exit_status}")
sys.exit(1)
pass
def scp_process(ssh, input_scp):
copy_list = []
for c in input_scp.splitlines():
if not c:
continue
l2r = c.split("=>")
if len(l2r) == 2:
local = strip_and_parse_envs(l2r[0])
remote = strip_and_parse_envs(l2r[1])
if local and remote:
copy_list.append({"l": local, "r": remote})
continue
print(f"SCP ignored {c.strip()}")
print(copy_list)
if len(copy_list) <= 0:
print("SCP no copy list found")
return
with scp.SCPClient(ssh.get_transport(), progress=progress, sanitize=lambda x: x) as conn:
for l2r in copy_list:
remote = l2r.get('r')
try:
ssh.exec_command(f"mkdir -p {remote}")
except Exception as err:
print(f"Remote mkdir error. Can't create {remote}\n{err}")
sys.exit(1)
for f in [f for f in glob(l2r.get('l'))]:
try:
conn.put(f, remote_path=remote, recursive=True)
print(f"{f} -> {remote}")
except Exception as err:
print(f"Scp error. Can't copy {f} on {remote}\n{err}")
sys.exit(1)
pass
def processes():
if INPUT_KEY is None and INPUT_PASS is None:
print("SSH-SCP-SSH invalid (Key/Passwd)")
return
if not INPUT_FIRST_SSH:
print("SSH-SCP-SSH no first_ssh input found")
else:
print("+++++++++++++++++++Pipeline: RUNNING FIRST SSH+++++++++++++++++++")
connect(lambda c: ssh_process(c, INPUT_FIRST_SSH))
if not INPUT_SCP:
print("SSH-SCP-SSH no scp input found")
else:
print("+++++++++++++++++++Pipeline: RUNNING SCP+++++++++++++++++++")
connect(lambda c: scp_process(c, INPUT_SCP))
if not INPUT_LAST_SSH:
print("SSH-SCP-SSH no last_ssh input found")
else:
print("+++++++++++++++++++Pipeline: RUNNING LAST SSH+++++++++++++++++++")
connect(lambda c: ssh_process(c, INPUT_LAST_SSH))
pass
if __name__ == '__main__':
processes()

138
entrypoint.sh Executable file → Normal file
View File

@ -1,10 +1,140 @@
#!/bin/bash
set -e
createKeyFile() {
local SSH_PATH="$HOME/.ssh"
mkdir -p "$SSH_PATH"
touch "$SSH_PATH/known_hosts"
echo "$INPUT_KEY" > "$SSH_PATH/id_rsa"
chmod 700 "$SSH_PATH"
chmod 600 "$SSH_PATH/known_hosts"
chmod 600 "$SSH_PATH/id_rsa"
eval $(ssh-agent)
ssh-add "$SSH_PATH/id_rsa"
ssh-keyscan -t rsa "$INPUT_HOST" >> "$SSH_PATH/known_hosts"
}
executeSSH() {
local USEPASS=$1
local LINES=$(echo -e $2)
local COMMAND=""
# holds all commands separated by semi-colon or keep "&&"
local COMMANDS=""
# this while read each commands in line and
# evaluate each line against all environment variables
while IFS=$'\n' read -r LINE; do
LINE=$(echo $LINE)
COMBINE="&&"
LASTCOMBINE="&&"
if [[ $LINE =~ ^.*\&\&$ ]]; then
LINE="$LINE true"
elif [[ $LINE =~ ^\&\&.*$ ]]; then
LINE="true $LINE"
elif [[ $LINE =~ ^.*\|\|$ ]]; then
LINE="$LINE false"
LASTCOMBINE="||"
elif [[ $LINE =~ ^\|\|.*$ ]]; then
LINE="false $LINE"
COMBINE="||"
fi
LINE=$(eval 'echo "$LINE"')
LINE=$(eval echo "$LINE")
LINE="$LINE $LASTCOMBINE"
if [ -z "$COMMANDS" ]; then
COMMANDS="$LINE"
else
# ref. https://unix.stackexchange.com/questions/459923/multiple-commands-in-sshpass
if [[ $COMMANDS =~ ^.*\&\&$ ]] || [[ $COMMANDS =~ ^.*\|\|$ ]]; then
COMMANDS="$COMMANDS $LINE"
else
COMMANDS="$COMMANDS $COMBINE $LINE"
fi
fi
done <<< "$LINES"
if [[ $COMMANDS =~ ^.*\&\&$ ]]; then
COMMANDS="$COMMANDS true"
elif [[ $COMMANDS =~ ^.*\|\|$ ]]; then
COMMANDS="$COMMANDS false"
fi
echo "$COMMANDS"
CMD="ssh"
if $USEPASS; then
CMD="sshpass -p $INPUT_PASS ssh"
fi
$CMD -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -p "${INPUT_PORT:-22}" "$INPUT_USER"@"$INPUT_HOST" "$COMMANDS" > /dev/stdout
}
executeSCP() {
local USEPASS=$1
local LINES=$(echo -e $2)
local COMMAND=
CMD="scp"
if $USEPASS; then
CMD="sshpass -p $INPUT_PASS scp"
fi
while IFS=$'\n' read -r LINE; do
delimiter="=>"
LINE=`echo $LINE`
s=$LINE$delimiter
arr=()
while [[ $s ]]; do
arr+=( "${s%%"$delimiter"*}" );
s=${s#*"$delimiter"};
done;
LOCAL=$(eval 'echo "${arr[0]}"')
LOCAL=$(eval echo "$LOCAL")
REMOTE=$(eval 'echo "${arr[1]}"')
REMOTE=$(eval echo "$REMOTE")
if [[ -z "${LOCAL}" ]] || [[ -z "${REMOTE}" ]]; then
echo "LOCAL/REMOTE can not be parsed $LINE"
else
echo "Copying $LOCAL ---> $REMOTE"
$CMD -r -o StrictHostKeyChecking=no -o ConnectTimeout=${INPUT_CONNECT_TIMEOUT:-30s} -P "${INPUT_PORT:-22}" $LOCAL "$INPUT_USER"@"$INPUT_HOST":$REMOTE > /dev/stdout
fi
done <<< "$LINES"
}
######################################################################################
echo "+++++++++++++++++++STARTING PIPELINES+++++++++++++++++++"
python3 /opt/tools/app.py
RET=$?
USEPASS=true
if [[ -z "${INPUT_KEY}" ]]; then
echo "+++++++++++++++++++Use password+++++++++++++++++++"
else
echo "+++++++++++++++++++Create Key File+++++++++++++++++++"
USEPASS=false
createKeyFile || false
fi
if ! [[ -z "${INPUT_FIRST_SSH}" ]]; then
echo "+++++++++++++++++++Step 1: RUNNING SSH+++++++++++++++++++"
executeSSH "$USEPASS" "$INPUT_FIRST_SSH" || false
fi
if ! [[ -z "${INPUT_SCP}" ]]; then
echo "+++++++++++++++++++Step 2: RUNNING SCP+++++++++++++++++++"
executeSCP "$USEPASS" "$INPUT_SCP" || false
fi
if ! [[ -z "${INPUT_LAST_SSH}" ]]; then
echo "+++++++++++++++++++Step 3: RUNNING SSH+++++++++++++++++++"
executeSSH "$USEPASS" "$INPUT_LAST_SSH" || false
fi
echo "+++++++++++++++++++END PIPELINES+++++++++++++++++++"
exit $RET

2
requirements.txt
View File

@ -1,2 +0,0 @@
paramiko
scp