From d7696331d077f4a7b3419ece2878636aea8f2495 Mon Sep 17 00:00:00 2001 From: redxef Date: Fri, 6 May 2022 15:28:11 +0200 Subject: [PATCH] Cleanup and fix of version output. --- Dockerfile | 4 +- check | 25 ------ common | 123 ------------------------- in | 42 --------- out | 47 ---------- src/check | 15 ++++ src/common | 158 +++++++++++++++++++++++++++++++++ src/in | 33 +++++++ src/out | 35 ++++++++ src/scripts/get_file_meta.sh | 23 +++++ src/scripts/get_file_meta.vars | 2 + src/scripts/get_files.sh | 10 +++ src/scripts/get_files.vars | 2 + src/scripts/put_files.sh | 17 ++++ src/scripts/put_files.vars | 4 + 15 files changed, 301 insertions(+), 239 deletions(-) delete mode 100755 check delete mode 100755 common delete mode 100755 in delete mode 100755 out create mode 100755 src/check create mode 100755 src/common create mode 100755 src/in create mode 100755 src/out create mode 100644 src/scripts/get_file_meta.sh create mode 100644 src/scripts/get_file_meta.vars create mode 100644 src/scripts/get_files.sh create mode 100644 src/scripts/get_files.vars create mode 100644 src/scripts/put_files.sh create mode 100644 src/scripts/put_files.vars diff --git a/Dockerfile b/Dockerfile index 2b131c2..e29591c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM alpine:latest -RUN apk --no-cache add bash openssh-client jq \ +RUN apk --no-cache add bash openssh-client jq gettext \ && adduser --no-create-home --uid 1000 -D user \ && mkdir -p /opt/resource/ USER 1000:1000 WORKDIR /opt/resource/ -COPY common check in out /opt/resource/ +COPY src/ /opt/resource/ diff --git a/check b/check deleted file mode 100755 index cc23d12..0000000 --- a/check +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env sh - -set -e -set -o pipefail - -[ -e /opt/resource/common ] && cd /opt/resource/ -. ./common - -version="$(compute_version "$(fetch_file_infos get_files)")" -if [ "$(echo "$version" | jq -r tostring)" = "$(get_version | jq -r .files | jq -r tostring)" ]; then - get_version - cleanup - exit 0 -fi - -jq -r tostring << EOF -{ - "version": { - "files": $version, - "time": $(date +%s) - } -} -EOF -cleanup -exit 0 diff --git a/common b/common deleted file mode 100755 index 4a1a510..0000000 --- a/common +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env sh - -INPUT_PATH= -CONFIG_PATH= -HOSTNAME= -PORT= -USER= -IDENTITY= -HOSTKEY= - -save_input() { - input_file="$(mktemp -tp /tmp)" - cat > "$input_file" <&0 - echo "$input_file" - return 0 -} - -read_config() { - HOSTNAME="$(jq -r .source.hostname < "$1")" - PORT="$(jq -r '.source.port // "22"' < "$1")" - USER="$(jq -r .source.user < "$1")" - IDENTITY="$(jq -r .source.identity < "$1")" - HOSTKEY="$(jq -r .source.hostkey < "$1")" - return 0 -} - -write_config() { - directory="$(mktemp -dp /tmp)" - config="$directory/config" - identity_file="$directory/identity" - knownhosts_file="$directory/knownhosts" - - { - echo "host $HOSTNAME" - echo " HostName $HOSTNAME" - echo " Port $PORT" - echo " User $USER" - echo " IdentityFile $identity_file" - echo " UserKnownHostsFile $knownhosts_file" - } > "$config" - echo "$IDENTITY" > "$identity_file" - echo "$HOSTKEY" > "$knownhosts_file" - chmod 600 "$identity_file" "$knownhosts_file" "$config" - echo "$config" - return 0 -} - -cleanup() { - rm -r /tmp/* -} - -INPUT_PATH="$(save_input)" -read_config "$INPUT_PATH" -CONFIG_PATH="$(write_config)" - -export INPUT_PATH -export CONFIG_PATH - -get_version() { - jq -r '.version // []' < "$INPUT_PATH" -} - -get_files() { - jq -r '.source.files?[]' < "$INPUT_PATH" -} - -get_command() { - jq -r '.params.command?' < "$INPUT_PATH" -} - -get_directory() { - jq -r '.params.directory // ""' < "$INPUT_PATH" -} - -get_workdir() { - jq -r '.params.workdir // "$HOME"' < "$INPUT_PATH" -} - -fetch_file_infos() { - files_file="$(mktemp -tp /tmp)" -ssh -F "$CONFIG_PATH" "$HOSTNAME" sh -s << EOF > "$files_file" - files="$("$1")" - echo "\$files" | while read -r file; do - echo "\$file" - [ -e "\$file" ] && echo true || echo false - [ -e "\$file" ] && echo "\$(stat --format=%Y "\$file")" || echo - done -EOF - echo "$files_file" - return 0 -} - -_compute_version() { - echo "[" - last_line= - line_file= - line_exists= - line_modify= - while read -r line; do - if [ -z "$line_file" ]; then - line_file="$line" - elif [ -z "$line_exists" ]; then - line_exists="$line" - elif [ -z "$line_modify" ]; then - line_modify="$line" - else - if "$line_exists"; then - echo '{"path":' "\"$line_file\"," '"modify":' "$line_modify" '}', - fi - line_file="$line" - line_exists= - line_modify= - fi - done < "$1" - if [ -n "$line_file" ] && [ -n "$line_exists" ] && [ -n "$line_modify" ] && "$line_exists"; then - echo '{"path":' "\"$line_file\"," '"modify":' "$line_modify" '}' - fi - echo "]" -} - -compute_version() { - _compute_version "$@" | jq -r tostring -} diff --git a/in b/in deleted file mode 100755 index b1c5546..0000000 --- a/in +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -[ -e /opt/resource/common ] && cd /opt/resource/ -. ./common - -cd "$1" -ssh -F "$CONFIG_PATH" "$HOSTNAME" sh -s << EOF | tar x - files="$(get_version | jq -r .[].path)" - archive="\$(mktemp -t)" - files_path="\$(mktemp -t)" - head --bytes=10240 /dev/zero > "\$archive" - echo "\$files" > "\$files_path" - error=false - while read -r filepath; do - if [ ! -e "\$filepath" ]; then - echo "File not found on remote: \$filepath" >&2 - error=true - break - fi - tar rf "\$archive" "\$filepath" - done < "\$files_path" - if ! "\$error"; then - cat "\$archive" - fi - rm "\$archive" - if "\$error"; then - exit 1 - else: - exit 0 - fi -EOF - -jq -r tostring << EOF -{ - "version": $(get_version) -} -EOF - -cleanup diff --git a/out b/out deleted file mode 100755 index e9469ba..0000000 --- a/out +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env sh - -set -e -set -o pipefail - -[ -e /opt/resource/common ] && cd /opt/resource/ -. ./common - -cd "$1" -if [ -n "$(get_directory)" ]; then - files="$(find . | sed -e '/^\.$/d' -e 's|^./||')" - tarfile="$(mktemp -tp /tmp)" - remote_tarfile="$(mktemp -u)" - tar -cf "$tarfile" ./* - scp -F "$CONFIG_PATH" "$tarfile" "$HOSTNAME:$remote_tarfile" -fi -ssh -F "$CONFIG_PATH" "$HOSTNAME" sh -s << EOF - set -e - if [ -n "$(get_directory)" ]; then - cd "$(get_directory)" - tar xf "$remote_tarfile" - rm "$remote_tarfile" - fi - if [ "$(get_command)" = "null" ]; then - exit 0 - fi - cd "$(get_workdir)" - $(get_command | jq .[] | tr '\n"' " '") >&2 - # echo "\$command_log" >&2 -EOF - -get_files_for_new_version() { - echo "$files" | xargs printf "$(get_directory)/%s\n" -} - -version="$(compute_version "$(fetch_file_infos get_files_for_new_version)")" - -jq -r tostring << EOF -{ - "version": { - "files": $version, - "time": $(date +%s) - } -} -EOF - -cleanup diff --git a/src/check b/src/check new file mode 100755 index 0000000..48c0200 --- /dev/null +++ b/src/check @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ -e /opt/resource/common ] && cd /opt/resource/ +. ./common + +jq -r tostring << EOF +[ + $(compute_version "$(fetch_file_meta)") +] +EOF +cleanup + +exit 0 diff --git a/src/common b/src/common new file mode 100755 index 0000000..79e387e --- /dev/null +++ b/src/common @@ -0,0 +1,158 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CLEANUP_FILES= + +tmpname() { + mktemp -u +} + +tmpfile() { + clean="$(mktemp -t)" + CLEANUP_FILES="$CLEANUP_FILES"$'\n'"$clean" + echo "$clean" + return 0 +} + +tmpdir() { + clean="$(mktemp -d)" + CLEANUP_FILES="$CLEANUP_FILES"$'\n'"$clean" + echo "$clean" + return 0 +} + +cleanup() { + if [ -n "$CLEANUP_FILES" ]; then + echo "$CLEANUP_FILES" | sed '//d' | xargs -d '\n' rm -r + fi + return 0 +} + +save_input() { + input_file="$(tmpfile)" + cat > "$input_file" <&0 + echo "$input_file" + return 0 +} +INPUT_PATH="$(save_input)" +export INPUT_PATH + +get_hostname() { + jq -r .source.hostname < "$INPUT_PATH" +} +get_port() { + jq -r '.source.port // "22"' < "$INPUT_PATH" +} +get_user() { + jq -r .source.user < "$INPUT_PATH" +} +get_identity() { + jq -r .source.identity < "$INPUT_PATH" +} +get_hostkey() { + jq -r .source.hostkey < "$INPUT_PATH" +} +get_files() { + jq -r '.source.files // []' < "$INPUT_PATH" | jq -r '.[]' +} +get_directory() { + jq -r '.source.directory // "$HOME"' < "$INPUT_PATH" | sed 's|/$||' +} +get_workdir() { + jq -r '.params.workdir // "$HOME"' < "$INPUT_PATH" | sed 's|/$||' +} +get_version() { + jq -r .version < "$INPUT_PATH" | jq -r tostring +} +get_command() { + jq -r '.params.command // ["true"]' < "$INPUT_PATH" | jq -r '.[]' | tr '\n"' " '" +} +HOSTNAME="$(get_hostname)" +PORT="$(get_port)" +USER="$(get_user)" +IDENTITY="$(get_identity)" +HOSTKEY="$(get_hostkey)" +FILES="$(get_files)" +DIRECTORY="$(get_directory)" +WORKDIR="$(get_workdir)" +VERSION="$(get_version)" +COMMAND="$(get_command)" + +export HOSTNAME +export PORT +export USER +export IDENTITY +export HOSTKEY +export FILES +export DIRECTORY +export WORKDIR +export VERSION +export COMMAND + +write_config() { + config="$(tmpfile)" + identity_file="$(tmpfile)" + knownhosts_file="$(tmpfile)" + + { + echo "host $HOSTNAME" + echo " HostName $HOSTNAME" + echo " Port $PORT" + echo " User $USER" + echo " IdentityFile $identity_file" + echo " UserKnownHostsFile $knownhosts_file" + } > "$config" + echo "$IDENTITY" > "$identity_file" + echo "$HOSTKEY" > "$knownhosts_file" + chmod 600 "$identity_file" "$knownhosts_file" "$config" + echo "$config" + return 0 +} +CONFIG_PATH="$(write_config)" +export CONFIG_PATH + +fetch_file_meta() { + meta_file="$(tmpfile)" + script="$(envsubst "$(< scripts/get_file_meta.vars)" < scripts/get_file_meta.sh)" + # echo "Executing script on remote" >&2 + # echo "$script" >&2 + ssh -F "$CONFIG_PATH" "$HOSTNAME" "sh -c '$script'" > "$meta_file" + echo "$meta_file" + return 0 +} + +_compute_version() { + echo "[" + line_file= + line_exists= + line_modify= + while read -r line; do + if [ -z "$line_file" ]; then + line_file="$line" + elif [ -z "$line_exists" ]; then + line_exists="$line" + elif [ -z "$line_modify" ]; then + line_modify="$line" + else + if "$line_exists"; then + echo '{"path":' "\"$line_file\"," '"modify":' "$line_modify" '}', + fi + line_file="$line" + line_exists= + line_modify= + fi + done < "$1" + if [ -n "$line_file" ] && [ -n "$line_exists" ] && [ -n "$line_modify" ] && "$line_exists"; then + echo '{"path":' "\"$line_file\"," '"modify":' "$line_modify" '}' + fi + echo "]" +} + +compute_version() { +jq -r tostring << EOF + { + "sha256sum": "$(_compute_version "$@" | jq -r tostring | sha256sum | tr -d ' -')" + } +EOF +} diff --git a/src/in b/src/in new file mode 100755 index 0000000..0fd40f1 --- /dev/null +++ b/src/in @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ -e /opt/resource/common ] && cd /opt/resource/ +. ./common + +computed_version="$(compute_version "$(fetch_file_meta)")" +if [ "$computed_version" != "$VERSION" ]; then + echo "Versions do not match, aborting" >&2 + echo "Requested version:" >&2 + echo "$VERSION" >&2 + echo "Have version:" >&2 + echo "$computed_version" >&2 + exit 1 +fi + +script="$(envsubst "$(< scripts/get_files.vars)" < "scripts/get_files.sh")" +if [ -n "$FILES" ]; then + ( + cd "$1" + ssh -F "$CONFIG_PATH" "$HOSTNAME" "sh -c '$script'" | tar x + ) +fi + +jq -r tostring << EOF + { + "version": $VERSION + } +EOF +cleanup + +exit 0 diff --git a/src/out b/src/out new file mode 100755 index 0000000..a5dc4dc --- /dev/null +++ b/src/out @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -euo pipefail + +[ -e /opt/resource/common ] && cd /opt/resource/ +. ./common + +extract=true +if [ -z "$FILES" ]; then + extract=false +fi +script="$(EXTRACT="$extract" envsubst "$(< scripts/put_files.vars)" < scripts/put_files.sh)" +( + cd "$1" + pipe="$(tmpname)" + mkfifo "$pipe" + ( + # shellcheck disable=SC2086 + IFS=$'\n' tar c $FILES > "$pipe" + ) & + background_1=$! + ssh -F "$CONFIG_PATH" "$HOSTNAME" "sh -c '$script'" < "$pipe" & + background_2=$! + rm "$pipe" + wait $background_1 $background_2 +) +computed_version="$(compute_version "$(fetch_file_meta)")" +jq -r tostring << EOF + { + "version": $computed_version + } +EOF +cleanup + +exit 0 diff --git a/src/scripts/get_file_meta.sh b/src/scripts/get_file_meta.sh new file mode 100644 index 0000000..4a0f99d --- /dev/null +++ b/src/scripts/get_file_meta.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env sh + +set -e + +directory="${DIRECTORY}" +files="${FILES}" + +if [ -z "$files" ]; then + exit 0 +fi + +( + echo "$files" | while read -r file; do + echo "$directory/$file" + if [ -e "$directory/$file" ]; then + echo true + echo "$(stat --format=%Y "$directory/$file")" + else + echo false + echo "# --PLACEHOLDER--" + fi + done +) diff --git a/src/scripts/get_file_meta.vars b/src/scripts/get_file_meta.vars new file mode 100644 index 0000000..ba406db --- /dev/null +++ b/src/scripts/get_file_meta.vars @@ -0,0 +1,2 @@ +${DIRECTORY} +${FILES} diff --git a/src/scripts/get_files.sh b/src/scripts/get_files.sh new file mode 100644 index 0000000..7e2c997 --- /dev/null +++ b/src/scripts/get_files.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh + +set -e + +directory="${DIRECTORY}" +files="${FILES}" +( + echo tar c $files >&2 + IFS=$'\n' tar c $files +) diff --git a/src/scripts/get_files.vars b/src/scripts/get_files.vars new file mode 100644 index 0000000..ba406db --- /dev/null +++ b/src/scripts/get_files.vars @@ -0,0 +1,2 @@ +${DIRECTORY} +${FILES} diff --git a/src/scripts/put_files.sh b/src/scripts/put_files.sh new file mode 100644 index 0000000..07f72bc --- /dev/null +++ b/src/scripts/put_files.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +set -e + +directory="${DIRECTORY}" +workdir="${WORKDIR}" +extract="${EXTRACT}" +( + if "$extract"; then + cd "$directory" + tar x + fi +) +( + cd "$workdir" + ${COMMAND} >&2 +) diff --git a/src/scripts/put_files.vars b/src/scripts/put_files.vars new file mode 100644 index 0000000..50186e7 --- /dev/null +++ b/src/scripts/put_files.vars @@ -0,0 +1,4 @@ +${DIRECTORY} +${EXTRACT} +${WORKDIR} +${COMMAND}