From 01abfba99d3c94b323402c2364637288cee258e1 Mon Sep 17 00:00:00 2001 From: Lukas Werner <lks.werner@fau.de> Date: Mon, 31 May 2021 14:39:45 +0200 Subject: [PATCH] Initial commit of run scripts and config --- runner_config/config.toml | 14 +++++ runner_scripts/root/config.sh | 19 +++++++ runner_scripts/root/run.sh | 92 +++++++++++++++++++++++++++++++ runner_scripts/user/config.sh | 19 +++++++ runner_scripts/user/run.sh | 100 ++++++++++++++++++++++++++++++++++ 5 files changed, 244 insertions(+) create mode 100644 runner_config/config.toml create mode 100755 runner_scripts/root/config.sh create mode 100755 runner_scripts/root/run.sh create mode 100755 runner_scripts/user/config.sh create mode 100755 runner_scripts/user/run.sh diff --git a/runner_config/config.toml b/runner_config/config.toml new file mode 100644 index 0000000..b3677a9 --- /dev/null +++ b/runner_config/config.toml @@ -0,0 +1,14 @@ +concurrent = 1 +check_interval = 0 + +[session_server] + session_timeout = 1800 + +[[runners]] + name = "xyz" + url = "https://i10git.cs.fau.de/" + token = "xx" + executor = "custom" + [runners.custom] + config_exec = "path/to/config.sh" + run_exec = "path/to/run.sh" diff --git a/runner_scripts/root/config.sh b/runner_scripts/root/config.sh new file mode 100755 index 0000000..1d9b786 --- /dev/null +++ b/runner_scripts/root/config.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euf -o pipefail +shopt -s inherit_errexit + +: "${CUSTOM_ENV_AUTH_USER:?"AUTH_USER CI/CD variable has not been set."}" +: "${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID:?"CI_CONCURRENT_PROJECT_ID CI/CD variable has not been set."}" + +cat << EOS +{ + + "builds_dir": "$WORK/gitlab-runner/builds/$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID", + "cache_dir": "$WORK/gitlab-runner/cache", + "builds_dir_is_shared": false, + "driver": { + "name": "Testserver Cx" + } +} +EOS + diff --git a/runner_scripts/root/run.sh b/runner_scripts/root/run.sh new file mode 100755 index 0000000..9ba01b0 --- /dev/null +++ b/runner_scripts/root/run.sh @@ -0,0 +1,92 @@ +#!/bin/bash -l +set -euf -o pipefail +shopt -s inherit_errexit + +module load slurm + +hash awk +hash chown +hash diff +hash getent +hash id +hash runuser +hash salloc +hash sed +hash srun +hash ssh-keygen + + +function error { + : "${RV:=${2:-$BUILD_FAILURE_EXIT_CODE}}" + echo "$1" >&2 + return "$RV" +} + +## User authentification + +: "${CUSTOM_ENV_AUTH_USER:?"AUTH_USER CI/CD variable has not been set."}" +: "${CUSTOM_ENV_AUTH_KEY:?"AUTH_KEY CI/CD variable has not been set."}" + +AUTH_USER=$CUSTOM_ENV_AUTH_USER +AUTH_USER_HOME=$(getent passwd "$AUTH_USER" | awk -F ":" '{print $6}') +AUTH_USER_SHELL=$(getent passwd "$AUTH_USER" | awk -F ":" '{print $7}') + +## Check if the user exists and the validity of its ID +id -u "$AUTH_USER" >/dev/null 2>&1 || error "User $AUTH_USER does not exist" +(( $(id -u "$AUTH_USER") >= 1000 )) || error "User $AUTH_USER ID within system reserved range." + +## Use a key pair to authenticate the user (private key has to be set as a GitLab CI/CD variable) +AUTH_KEY=$CUSTOM_ENV_AUTH_KEY +AUTH_PUB=$AUTH_USER_HOME/gitlab-runner/authorized_keys + +( +while read -r PUB +do + diff <(ssh-keygen -y -e -f /dev/stdin <<< "$AUTH_KEY") <(ssh-keygen -y -e -f /dev/stdin <<< "$PUB") && exit 0 +done < "$AUTH_PUB" +exit 1 +) || error "Authentication failed." + +### Env setup + +chown -R "$AUTH_USER" "$TMPDIR" + +BASE_DIR=$WORK/gitlab-runner/builds/${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID:?}/${CUSTOM_ENV_CI_PROJECT_NAMESPACE:?}/${CUSTOM_ENV_CI_PROJECT_NAME:?} + +runuser "$AUTH_USER" --command "mkdir -p $BASE_DIR" + +if [[ ("$2" == "step_script" || "$2" == "build_script") && ${CUSTOM_ENV_SUBMIT_TO_SLURM:-} ]] +then + ## The script is the one specified in the gitlab-ci.yml script directive and the SUBMIT_TO_SLURM variable is set, + + for E in $(env | grep -E "^CUSTOM_ENV_SLURM_") + do + export "${E#CUSTOM_ENV_}" + done + + : "${SLURM_JOB_NAME:="gitlab-ci-${CUSTOM_ENV_CI_PROJECT_NAME:?}-${CUSTOM_ENV_CI_PIPELINE_ID:?}-${CUSTOM_ENV_CI_JOB_ID:?}"}" + : "${SLURM_TIMELIMIT:="240"}" + : "${SLURM_TIME:=$SLURM_TIMELIMIT}" + + export SLURM_JOB_NAME + export SLURM_TIME + unset SLURM_TIMELIMIT + + # Generate salloc arguments from SLURM_* environment variables + SALLOC_OPTIONS=() + for E in $(env | grep -E "^SLURM_") + do + echo $E + SALLOC_OPTIONS+=("$(echo "${E#SLURM_}" | awk -F "=" '{gsub("_", "-", $1); print "--"tolower($1)"="$2}')") + done + + runuser --login "$AUTH_USER" --command "cp $1 $BASE_DIR.tmp/$CUSTOM_ENV_CI_JOB_ID.sh" + + exec salloc --quiet --uid "$AUTH_USER" --gid "$(id -g -n "$AUTH_USER")" --chdir "$BASE_DIR" "${SALLOC_OPTIONS[@]}" \ + srun --cpu-bind none --export=HOME="$AUTH_USER_HOME",SHELL="$AUTH_USER_SHELL" --wait 0 --kill-on-bad-exit=1 \ + "$AUTH_USER_SHELL" --login "$BASE_DIR.tmp/$CUSTOM_ENV_CI_JOB_ID.sh" + +else + runuser --login "$AUTH_USER" --command "cd \"$BASE_DIR\"; $1" +fi + diff --git a/runner_scripts/user/config.sh b/runner_scripts/user/config.sh new file mode 100755 index 0000000..1d9b786 --- /dev/null +++ b/runner_scripts/user/config.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euf -o pipefail +shopt -s inherit_errexit + +: "${CUSTOM_ENV_AUTH_USER:?"AUTH_USER CI/CD variable has not been set."}" +: "${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID:?"CI_CONCURRENT_PROJECT_ID CI/CD variable has not been set."}" + +cat << EOS +{ + + "builds_dir": "$WORK/gitlab-runner/builds/$CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID", + "cache_dir": "$WORK/gitlab-runner/cache", + "builds_dir_is_shared": false, + "driver": { + "name": "Testserver Cx" + } +} +EOS + diff --git a/runner_scripts/user/run.sh b/runner_scripts/user/run.sh new file mode 100755 index 0000000..3e328e3 --- /dev/null +++ b/runner_scripts/user/run.sh @@ -0,0 +1,100 @@ +#!/bin/bash -l +set -euf -o pipefail +shopt -s inherit_errexit + +module load slurm + +hash awk +hash chown +hash diff +hash getent +hash id +hash runuser +hash salloc +hash sed +hash srun +hash ssh-keygen + + +function error { + : "${RV:=${2:-$BUILD_FAILURE_EXIT_CODE}}" + echo "$1" >&2 + return "$RV" +} + +## User authentification + +: "${CUSTOM_ENV_AUTH_USER:?"AUTH_USER CI/CD variable has not been set."}" +: "${CUSTOM_ENV_AUTH_KEY:?"AUTH_KEY CI/CD variable has not been set."}" + +AUTH_USER=$CUSTOM_ENV_AUTH_USER +AUTH_USER_HOME=$(getent passwd "$AUTH_USER" | awk -F ":" '{print $6}') +AUTH_USER_SHELL=$(getent passwd "$AUTH_USER" | awk -F ":" '{print $7}') + +## Check if the user exists and the validity of its ID +id -u "$AUTH_USER" >/dev/null 2>&1 || error "User $AUTH_USER does not exist" +(( $(id -u "$AUTH_USER") >= 1000 )) || error "User $AUTH_USER ID within system reserved range." + +## Use a key pair to authenticate the user (private key has to be set as a GitLab CI/CD variable) +AUTH_KEY=$CUSTOM_ENV_AUTH_KEY +AUTH_PUB=$AUTH_USER_HOME/gitlab-runner/authorized_keys + +( +while read -r PUB +do + diff <(ssh-keygen -y -e -f /dev/stdin <<< "$AUTH_KEY") <(ssh-keygen -y -e -f /dev/stdin <<< "$PUB") && exit 0 +done < "$AUTH_PUB" +exit 1 +) || error "Authentication failed." + +### Env setup + +chown -R "$AUTH_USER" "$TMPDIR" + +BASE_DIR=$WORK/gitlab-runner/builds/${CUSTOM_ENV_CI_CONCURRENT_PROJECT_ID:?}/${CUSTOM_ENV_CI_PROJECT_NAMESPACE:?}/${CUSTOM_ENV_CI_PROJECT_NAME:?} + +mkdir -p $BASE_DIR + +if [[ ("$2" == "step_script" || "$2" == "build_script") && ${CUSTOM_ENV_SUBMIT_TO_SLURM:-} ]] +then + ## The script is the one specified in the gitlab-ci.yml script directive and the SUBMIT_TO_SLURM variable is set, + + for E in $(env | grep -E "^CUSTOM_ENV_SLURM_") + do + export "${E#CUSTOM_ENV_}" + done + + : "${SLURM_JOB_NAME:="gitlab-ci-${CUSTOM_ENV_CI_PROJECT_NAME:?}-${CUSTOM_ENV_CI_PIPELINE_ID:?}-${CUSTOM_ENV_CI_JOB_ID:?}"}" + : "${SLURM_TIMELIMIT:="240"}" + : "${SLURM_TIME:=$SLURM_TIMELIMIT}" + + export SLURM_JOB_NAME + export SLURM_TIME + unset SLURM_TIMELIMIT + + # Generate salloc arguments from SLURM_* environment variables + SALLOC_OPTIONS=() + for E in $(env | grep -E "^SLURM_") + do + echo $E + SALLOC_OPTIONS+=("$(echo "${E#SLURM_}" | awk -F "=" '{gsub("_", "-", $1); print "--"tolower($1)"="$2}')") + done + + ## If the GitLab runner executable is not in the $PATH, need to explicitly make it available in the SLURM script + # (here we assume it is in the user's home) + GITLAB_RUNNER_EXECUTABLE=$AUTH_USER_HOME/gitlab-runner + + cp $1 $BASE_DIR.tmp/$CUSTOM_ENV_CI_JOB_ID.sh + + exec salloc --quiet --chdir "$BASE_DIR" "${SALLOC_OPTIONS[@]}" \ + srun --cpu-bind none --export=HOME="$AUTH_USER_HOME",SHELL="$AUTH_USER_SHELL",PATH="$PATH:$GITLAB_RUNNER_EXECUTABLE" --wait 0 --kill-on-bad-exit=1 \ + "$AUTH_USER_SHELL" --login "$BASE_DIR.tmp/$CUSTOM_ENV_CI_JOB_ID.sh" + +else + ## just execute the script supplied in $1 + # if the GitLab runner executable is not in the $PATH, need to explicitly make it available here + # (here we assume it is in the user's home) + GITLAB_RUNNER_EXECUTABLE=$AUTH_USER_HOME/gitlab-runner + PATH=$PATH:$GITLAB_RUNNER_EXECUTABLE bash -l $1 +fi + -- GitLab