diff --git a/.gitignore b/.gitignore index 4ba6098..72a78b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /*.conf +/bin diff --git a/Makefile b/Makefile index 3ba81e5..df14ba6 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,30 @@ NAME = netoik-cicd VERSION = $(shell [ -d ".git" ] && git describe | sed "s/-/./g") BRANCH = $(shell [ -d ".git" ] && git branch --show-current) +TEMP_DIR = "${PWD}/.temp" RPM_SOURCEDIR = $(shell rpm --eval "%{_sourcedir}") SYSCONFDIR = $(shell rpm --eval "%{_sysconfdir}") UNITDIR = $(shell rpm --eval "%{_unitdir}") BINDIR = $(shell rpm --eval "%{_bindir}") +RUNDIR = $(shell rpm --eval "%{_rundir}") TMPDIR = $(shell rpm --eval "%{_tmppath}") .PHONY: build -build: bin/$(NAME)-deployer bin/$(NAME)-newtag +build: bin/$(NAME)-deployer bin/$(NAME)-runner bin/$(NAME)-pipeline bin/$(NAME)-deployer: src/deployer.sh - mkdir --parents bin - cp "$<" "$@" + install -D --no-target-directory "$<" "$@" -bin/$(NAME)-newtag: src/newtag.sh - mkdir --parents bin - cp "$<" "$@" +bin/$(NAME)-runner: src/runner.sh + install -D --no-target-directory "$<" "$@" + +bin/$(NAME)-pipeline: src/pipeline.sh + install -D --no-target-directory "$<" "$@" + +.PHONY: clean +clean: + rm --recursive --force ./bin .PHONY: name name: @@ -29,30 +36,63 @@ version: @echo "$(VERSION)" $(RPM_SOURCEDIR)/$(NAME)-%.tar.gz: * - git archive --format=tar.gz --output="$@" --prefix="$(NAME)-$(VERSION)/" --worktree-attributes --verbose "$(BRANCH)" + git archive --format=tar.gz \ + --output="$@" \ + --prefix="$(NAME)-$(VERSION)/" \ + --worktree-attributes \ + --verbose \ + "$(BRANCH)" .PHONY: tarball tarball: $(RPM_SOURCEDIR)/$(NAME)-$(VERSION).tar.gz .PHONY: install -install: - install -D --no-target-directory deployer.conf.sample "$(DESTDIR)$(SYSCONFDIR)/$(NAME)/deployer.conf" - install -D --target-directory="$(DESTDIR)$(SYSCONFDIR)/$(NAME)" deployer.conf.sample - install -D --target-directory="$(DESTDIR)$(SYSCONFDIR)/profile.d" profile/$(NAME)-git.sh - install -D --target-directory="$(DESTDIR)$(UNITDIR)" systemd/$(NAME)-deployer.service - install -D --target-directory="$(DESTDIR)$(BINDIR)" bin/$(NAME)-* - install --directory "$(DESTDIR)$(TMPDIR)/$(NAME)/deployer/request" - install --directory "$(DESTDIR)$(TMPDIR)/$(NAME)/deployer/response" +install: conf/*.sample systemd/*.service bin/* + install -D \ + --no-target-directory \ + "conf/$(NAME).conf.sample" \ + "$(DESTDIR)$(SYSCONFDIR)/$(NAME)/$(NAME).conf" + install -D \ + --target-directory="$(DESTDIR)$(SYSCONFDIR)/$(NAME)" \ + "conf/$(NAME).conf.sample" + install -D \ + --no-target-directory \ + conf/errors.conf.sample \ + "$(DESTDIR)$(SYSCONFDIR)/$(NAME)/errors.conf" + install -D \ + --target-directory="$(DESTDIR)$(SYSCONFDIR)/$(NAME)" \ + conf/errors.conf.sample + install -D \ + --no-target-directory \ + "systemd/deployer.service" \ + "$(DESTDIR)$(UNITDIR)/$(NAME)-deployer.service" + install -D \ + --no-target-directory \ + "systemd/runner.service" \ + "$(DESTDIR)$(UNITDIR)/$(NAME)-runner.service" + install -D \ + --target-directory="$(DESTDIR)$(BINDIR)" \ + "bin/$(NAME)-deployer" + install -D \ + --target-directory="$(DESTDIR)$(BINDIR)" \ + "bin/$(NAME)-runner" + install -D \ + --target-directory="$(DESTDIR)$(BINDIR)" \ + "bin/$(NAME)-pipeline" + install --directory "$(DESTDIR)$(RUNDIR)/$(NAME)/deployer" + install --directory "$(DESTDIR)$(RUNDIR)/$(NAME)/runner" + install --directory "$(DESTDIR)$(RUNDIR)/$(NAME)/pipeline" + install --directory "$(DESTDIR)$(TMPDIR)/$(NAME)/repositories" .PHONY: check_format check_format: - shfmt --diff . + shfmt --diff ./src .PHONY: check_linting check_linting: - shfmt --find . | xargs bash -o noexec - shfmt --find . | xargs shellcheck + shfmt --find ./src | xargs bash -o noexec + shfmt --find ./src | xargs shellcheck --external-sources .PHONY: unit_test unit_test: - @echo "not implemented yet" + bats ./tests diff --git a/conf/errors.conf.sample b/conf/errors.conf.sample new file mode 100644 index 0000000..6c8a106 --- /dev/null +++ b/conf/errors.conf.sample @@ -0,0 +1,58 @@ +# Config parsing errors. +err_deployer_sock_empty=11 +err_deployer_sock_dir_not_directory=12 +err_deployer_sock_dir_not_writable=13 +err_deployer_sock_already_in_use=14 +err_deployer_sock_not_exist=15 +err_repos_dir_empty=16 +err_repos_dir_not_directory=17 +err_repos_dir_not_accessible=18 +err_rpms_dir_empty=19 +err_rpms_dir_not_directory=20 +err_rpms_dir_not_accessible=21 +err_runner_username_empty=22 +err_runner_user_not_exist=23 +err_deployer_timeout_empty=24 +err_deployer_timeout_not_valid=25 +err_runner_sock_empty=26 +err_runner_sock_dir_not_directory=27 +err_runner_sock_dir_not_writable=28 +err_runner_sock_already_in_use=29 +err_runner_sock_not_exist=30 +err_deployer_username_empty=31 +err_deployer_user_not_exist=32 +err_runner_cloning_dir_empty=33 +err_runner_cloning_dir_not_directory=34 +err_runner_cloning_dir_not_writable=35 +err_runner_timeout_empty=36 +err_runner_timeout_not_valid=37 +err_git_runner_groupname_empty=38 +err_git_runner_group_not_exist=39 +err_runner_deployer_groupname_empty=40 +err_runner_deployer_group_not_exist=41 + +# JSON parsing errors +err_json_bad_format=51 +err_pkg_name_missing=52 +err_pkg_name_empty=53 +err_repo_dir_not_exist=54 +err_pkg_version_missing=55 +err_pkg_version_empty=56 +err_rpm_path_not_exist=57 +err_repo_name_missing=58 +err_repo_name_empty=59 +err_repo_hash_missing=60 +err_repo_hash_empty=61 +err_repo_tag_empty=62 +err_clone_repo=63 +err_checkout_hash=64 +err_check_format=65 +err_check_linting=66 +err_unit_test=67 +err_add_tag=68 +err_make_tarball=69 +err_rpm_build=70 + +# Operationnal errors. +err_rpm_upgrade=101 +err_rpm_install=102 diff --git a/conf/netoik-cicd.conf.sample b/conf/netoik-cicd.conf.sample new file mode 100644 index 0000000..df7680a --- /dev/null +++ b/conf/netoik-cicd.conf.sample @@ -0,0 +1,37 @@ +# Name of group unifying git and runner. +git_runner_groupname="netoik-cicd-git-runner" + +# Name of group unifying runner and deployer. +runner_deployer_groupname="netoik-cicd-runner-deployer" + +# Name of the user responsible of the deployer server. +deployer_username="netoik-cicd-deployer" + +# Location of unixsock file used to send requests to the deployer server. +deployer_sock="/run/netoik-cicd/deployer/deployer.sock" + +# Maximum number of seconds to wait for deployer response. +# Set to 0 to disable timeout. +deployer_timeout=30 + +# Name of the user responsible of the runner server. +runner_username="netoik-cicd-runner" + +# Location of unixsock file used to send requests to the runner server. +runner_sock="/run/netoik-cicd/runner/runner.sock" + +# Directory in which to clone git repositories. +runner_cloning_dir="/var/tmp/netoik-cicd/repositories" + +# Maximum number of seconds to wait for runner response. +# Set to 0 to disable timeout. +runner_timeout=120 + +# Directory in which to create pipeline sock files. +pipeline_sock_dir="/run/netoik-cicd/pipeline" + +# Directory containing all the git repositories. +repos_dir="/var/gogs/repositories/samuel" + +# Directory containing rpm packages. +rpms_dir="/var/netoik-cicd/runner/rpmbuild/RPMS" diff --git a/deployer.conf.sample b/deployer.conf.sample deleted file mode 100644 index b3521f7..0000000 --- a/deployer.conf.sample +++ /dev/null @@ -1,7 +0,0 @@ -REQUEST_DIR="/var/tmp/netoik-cicd/deployer/request" -RESPONSE_DIR="/var/tmp/netoik-cicd/deployer/response" -REPOS_DIR="/var/gogs/repositories/samuel" -RPMS_DIR="/home/git/rpmbuild/RPMS" -RPM_ARCH="x86_64" -RPM_RELEASE="1" -RPM_DIST="el8_5" diff --git a/netoik-cicd.spec b/netoik-cicd.spec index 94ac914..0b4b0e2 100644 --- a/netoik-cicd.spec +++ b/netoik-cicd.spec @@ -1,16 +1,16 @@ %define debug_package %{nil} -Name: netoik-cicd +Name: %(make name) Version: %(make version) Release: 1%{?dist} -Summary: Netoik Continuous Deployment tool +Summary: Netoik Continuous Integration & Deployment tool License: GPLv3 Source0: %{name}-%{version}.tar.gz -BuildArch: x86_64 -BuildRequires: make -Requires: bash,rpm-build,rpmdevtools,inotify-tools,shfmt,shellcheck +BuildArch: x86_64 +BuildRequires: make +Requires: bash,inotify-tools,nmap-ncat,rpm-build,rpmdevtools,ShellCheck %description Netoik Continuous Deployment tool @@ -25,36 +25,73 @@ Netoik Continuous Deployment tool %make_install %pre -# Build rpm setuptree if not already done. -runuser --login git --command "rpmdev-setuptree" +# Create users runner and deployer. +if ! getent group %{name}-git-runner; then + groupadd %{name}-git-runner + usermod --append --groups %{name}-git-runner git +fi +if ! getent group %{name}-runner-deployer; then + groupadd %{name}-runner-deployer +fi +if ! id %{name}-runner; then + useradd --create-home \ + --home-dir %{_var}/%{name}-runner \ + --shell=%{_bindir}/rpmdev-setuptree \ + --groups %{name}-git-runner,%{name}-runner-deployer \ + --user-group \ + %{name}-runner + runuser --login %{name}-runner +fi +if ! id %{name}-deployer; then + useradd --no-create-home \ + --home-dir %{_var}/%{name}/deployer \ + --shell=%{_sbindir}/nologin \ + --groups %{name}-runner-deployer,wheel \ + --user-group \ + %{name}-deployer +fi %post # Reload systemctl daemon and (re)start service. systemctl daemon-reload systemctl restart %{name}-deployer.service systemctl enable %{name}-deployer.service +systemctl restart %{name}-runner.service +systemctl enable %{name}-runner.service %preun # Stop service only if uninstalling. if [ $1 -eq 0 ]; then - systemctl disable --now %{name}-deployer.service + systemctl disable --now %{name}-deployer.service + rm --force %{_rundir}/%{name}/deployer/deployer.sock + systemctl disable --now %{name}-runner.service + rm --force %{_rundir}/%{name}/runner/runner.sock fi %postun -# Reload systemctl daemon only if uninstalling. +# Reload systemctl daemon only and remove users if uninstalling. if [ $1 -eq 0 ]; then - systemctl daemon-reload + systemctl daemon-reload + userdel %{name}-deployer || true + userdel --remove %{name}-runner || true + groupdel %{name}-git-runner || true + groupdel %{name}-runner-deployer || true fi %files %attr(755, root, root) %dir %{_sysconfdir}/%{name} -%attr(644, root, root) %config(noreplace) %{_sysconfdir}/%{name}/deployer.conf -%attr(644, root, root) %{_sysconfdir}/%{name}/deployer.conf.sample -%attr(644, root, root) %{_sysconfdir}/profile.d/%{name}-git.sh +%attr(644, root, root) %config(noreplace) %{_sysconfdir}/%{name}/%{name}.conf +%attr(644, root, root) %{_sysconfdir}/%{name}/%{name}.conf.sample +%attr(644, root, root) %config(noreplace) %{_sysconfdir}/%{name}/errors.conf +%attr(644, root, root) %{_sysconfdir}/%{name}/errors.conf.sample %attr(644, root, root) %{_unitdir}/%{name}-deployer.service +%attr(644, root, root) %{_unitdir}/%{name}-runner.service %attr(755, root, root) %{_bindir}/%{name}-deployer -%attr(755, root, root) %{_bindir}/%{name}-newtag +%attr(755, root, root) %{_bindir}/%{name}-runner +%attr(755, root, root) %{_bindir}/%{name}-pipeline +%attr(755, root, root) %dir %{_rundir}/%{name} +%attr(755, %{name}-deployer, %{name}-deployer) %dir %{_rundir}/%{name}/deployer +%attr(755, git, git) %dir %{_rundir}/%{name}/pipeline +%attr(755, %{name}-runner, %{name}-runner) %dir %{_rundir}/%{name}/runner %attr(755, root, root) %dir %{_tmppath}/%{name} -%attr(755, root, root) %dir %{_tmppath}/%{name}/deployer -%attr(775, root, git) %dir %{_tmppath}/%{name}/deployer/request -%attr(775, root, root) %dir %{_tmppath}/%{name}/deployer/response +%attr(755, %{name}-runner, %{name}-runner) %dir %{_tmppath}/%{name}/repositories diff --git a/profile/netoik-cicd-git.sh b/profile/netoik-cicd-git.sh deleted file mode 100644 index ea23b90..0000000 --- a/profile/netoik-cicd-git.sh +++ /dev/null @@ -1,6 +0,0 @@ -source "/etc/netoik-cicd/deployer.conf" - -if [ "$(id --user --name)" = "git" ]; then - NETOIK_CICD_DEPLOYER_RESPONSE_DIR="$REQUEST_DIR" - NETOIK_CICD_DEPLOYER_REQUEST_DIR="$RESPONSE_DIR" -fi diff --git a/src/deployer.sh b/src/deployer.sh index 9ebaf3d..3be4411 100755 --- a/src/deployer.sh +++ b/src/deployer.sh @@ -1,7 +1,17 @@ -#!/usr/bin/bash -# -# This binary is made to be run by root, it expects a request from git server, deploy (install or update) -# the related tpm package and send a response to git server. +#!/bin/bash + +################################################################################ +# # +# netoik-cicd-deployer # +# # +# This binary is made to be run by root, it expects a request from an # +# unprivileged user server, deploys (installs or updates) the related rpm # +# package and sends a response to the user. # +# # +# This script is coded with respect of google shell styleguide: # +# https://google.github.io/styleguide/shellguide.html # +# # +################################################################################ # Exit immediately if any command fails. set -e @@ -9,74 +19,396 @@ set -e # Exit with the last non-zero fail code. set -o pipefail -log() { - echo -e "[DEPLOYER] $(date --rfc-3339=s) - $1" -} +# Program constants. +readonly PACKAGE_NAME="netoik-cicd" +readonly PROGRAM_NAME="${PACKAGE_NAME}-deployer" -fail () { - if [ $# -eq 1 ]; then - echo "$1" 1>&2 +# Declare default variables. +readonly DEFAULT_CONFIG_FILE="/etc/${PACKAGE_NAME}/${PACKAGE_NAME}.conf" +readonly DEFAULT_ERRORS_FILE="/etc/${PACKAGE_NAME}/errors.conf" + +# Declare levels variables. +readonly VERBOSITY_LEVELS=( + [10]="CRITICAL" + [20]="ERROR" + [30]="WARN" + [40]="INFO" + [50]="DEBUG" +) +readonly DEFAULT_VERBOSITY_LEVEL=40 + +# RPM constants. +readonly RPM_RELEASE="1" +RPM_ARCH=$(rpm --eval "%{_arch}") +readonly RPM_ARCH +RPM_DIST=$(rpm --eval "%{dist}") +readonly RPM_DIST + +# Functions. +log() ( + level="$1" + msg="$2" + + if [ "${level}" -le "${verbosity_level}" ]; then + echo "${VERBOSITY_LEVELS[${level}]} - ${msg}" fi - exit 1 -} +) +log_critical() (log 10 "$1" >&2) +log_error() (log 20 "$1" >&2) +log_warn() (log 30 "$1" >&2) +log_info() (log 40 "$1") +log_debug() (log 50 "$1") -# Load config file. -[ $# -eq 1 ] || fail "Expecting 1 argument: config file." -source "$1" - -# Check variables in config file. -[ -d "$REQUEST_DIR" ] || fail "Directory does not exist REQUEST_DIR=$REQUEST_DIR in config file $1." -[ -d "$RESPONSE_DIR" ] || fail "Directory does not exist RESPONSE_DIR=$RESPONSE_DIR in config file $1." -[ -d "$REPOS_DIR" ] || fail "Directory does not exist REPOS_DIR=$REPOS_DIR in config file $1." -[ -d "$RPMS_DIR" ] || fail "Directory does not exist RPMS_DIR=$RPMS_DIR in config file $1." -[ -z "$RPM_ARCH" ] && fail "Empty value RPM_ARCH in config file $1." -[ -z "$RPM_RELEASE" ] && fail "Empty value RPM_RELEASE in config file $1." -[ -z "$RPM_DIST" ] && fail "Empty value RPM_DIST in config file $1." - -# First remove eventual old existing tmp files. -find "$REQUEST_DIR" -type f -delete -find "$RESPONSE_DIR" -type f -delete - -# Loop on every created request. -while read _ _ repo_name; do - - log "New request detected for repo $repo_name." - - # Read request file and remove it immediately. - repo_version=$(cat "$REQUEST_DIR/$repo_name") - rm "$REQUEST_DIR/$repo_name" - - # Check repo version not empty. - if [ -z "$repo_version" ]; then - echo -e "Content of $REQUEST_DIR/$repo_name must contain repo version but is empty\n1" > "$RESPONSE_DIR/$repo_name" - continue +fail() ( + msg="$1" + if [ $# -eq 2 ]; then + code="$2" + else + code=1 fi - # Check if repo does exist. - if [ ! -d "$REPOS_DIR/$repo_name.git" ]; then - echo -e "Repository $REPOS_DIR/$repo_name.git does not exist!\n1" > "$RESPONSE_DIR/$repo_name" - continue + log_critical "${msg}" + if [ "${code}" -gt 0 ]; then + exit "${code}" + else + log_error "Undefined error code for '${msg}'." + exit 1 fi - - # Check if repo package is already exisitng. - rpm_path="$RPMS_DIR/$RPM_ARCH/$repo_name-$repo_version-$RPM_RELEASE.$RPM_DIST.$RPM_ARCH.rpm" - log "Using rpm package at $rpm_path." - if [ ! -f "$rpm_path" ]; then - echo -e "RPM package $rpm_path does not exist!\n1" > "$RESPONSE_DIR/$repo_name" - continue +) + +usage() ( + echo "Usage: ${PROGRAM_NAME} [OPTION]... +Start deployer server, wait for unixsock requests to deploy rpm packages. + +Mandatory argumentes for long options are mandatory for short options too. + + -c, --conf= name of configuration file, default + to ${DEFAULT_CONFIG_FILE} + -d, --daemon run processing loop in background + -e, --errs= name of errors file, default to + ${DEFAULT_ERRORS_FILE} + -q, --quiet set level verbosity to WARN, default to INFO + -s, --stop stop listening after first request process + -t, --test just test config file and do not run start loop + -v, --verbose set level verbosity to DEBUG, default to INFO + + -h, --help display this help message and exit +" +) + +reply() ( + code="$1" + msg="$2" + + if [ "${code}" -gt 0 ]; then + log_error "${msg}" + else + log_info "${msg}" fi + if ! output="$(jq --null-input \ + --compact-output \ + --arg c "${code}" \ + --arg m "${msg}" \ + '{"code":$c|tonumber,"msg":$m}' | + ncat --unixsock "${response_sock}" 2>&1)"; then + log_error "Cannot write to sock '${response_sock}': ${output}." + fi +) + +process_request() ( + # Get response_sock from json. + log_debug "Get response sock file." + if ! response_sock="$(echo "${request}" | + jq --raw-output ".response_sock" 2>&1)"; then + log_error "Cannot parse json request '${request}': ${response_sock}." + return + fi + if [ "${response_sock}" = "null" ]; then + log_error "Missing key 'response_sock' in request '${request}'." + return + fi + if [ -z "${response_sock}" ]; then + log_error "Empty value for key 'response_sock' in request '${request}'." + return + fi + if [ ! -S "${response_sock}" ]; then + log_error \ + "Sock file '${response_sock}' does not exist in request '${request}'." + return + fi + log_debug "Sock file '${response_sock}' is reachable." + + # Get pkg_name from json. + log_debug "Get pkg_name from json." + if ! pkg_name="$(echo "${request}" | jq --raw-output ".pkg_name" 2>&1)"; then + reply "${err_json_bad_format}" \ + "Cannot parse json '${request}': ${pkg_name}." + return + fi + if [ "${pkg_name}" = "null" ]; then + reply "${err_pkg_name_missing}" "Missing key pkg_name in '${request}'." + return + fi + if [ -z "${pkg_name}" ]; then + reply "${err_pkg_name_empty}" \ + "Empty value for key pkg_name in '${request}'." + return + fi + log_debug "Got pkg name: '${pkg_name}'." + + # Get repository folder. + repo_dir="${repos_dir}/${pkg_name}.git" + if [ ! -d "${repo_dir}" ]; then + reply "${err_repo_dir_not_exist}" "Repository '${repo_dir}' does not exist." + return + fi + log_debug "Repository is present at '${repo_dir}'." + + # Get pkg_version from json. + log_debug "Get pkg_version from json." + if ! pkg_version="$(echo "${request}" | + jq --raw-output ".pkg_version" 2>&1)"; then + reply "${err_json_bad_format}" \ + "Cannot parse json '${request}': ${pkg_version}." + return + fi + if [ "${pkg_version}" = "null" ]; then + reply "${err_pkg_version_missing}" \ + "Missing key pkg_version in '${request}'." + return + fi + if [ -z "${pkg_version}" ]; then + reply "${err_pkg_version_empty}" \ + "Empty value for key pkg_version in '${request}'." + return + fi + log_debug "Got pkg version '${pkg_version}'." + + # Check if package does exist. + rpm_path="${rpms_dir}/${RPM_ARCH}/${pkg_name}-${pkg_version}-${RPM_RELEASE}${RPM_DIST}.${RPM_ARCH}.rpm" + if [ ! -f "${rpm_path}" ]; then + reply "${err_rpm_path_not_exist}" \ + "RPM package '${rpm_path}' does not exist!" + return + fi + log_debug "RPM path found at '${rpm_path}'." + # Upgrade package if already installed. - if rpm -q "$repo_name" 1>/dev/null 2>/dev/null; then - log "Upgrade package $repo_name to v$repo_version" - output=$(rpm --upgrade --verbose --hash "$rpm_path" 2>&1) || exit_code=$? - echo -e "$output\n$exit_code" > "$RESPONSE_DIR/$repo_name" - continue + log_debug "Check if pkg '${pkg_name}' is already installed." + if rpm --query "${pkg_name}" 1>/dev/null 2>/dev/null; then + log_debug "Package '${pkg_name}' already installed, so upgrade to v + ${pkg_version}" + if ! output="$(sudo rpm --upgrade \ + --verbose --hash "${rpm_path}" 2>&1)"; then + reply "${err_rpm_upgrade}" \ + "Cannot upgrade package '${pkg_name}' to v${pkg_version}: ${output}." + return + fi + log_debug "RPM package '${pkg_name}' upgraded to v${pkg_version}." fi # Install package if not already installed. - log "Install package $repo_name v$repo_version." - output=$(rpm --install --verbose --hash "$rpm_path" 2>&1) || exit_code=$? - echo -e "$output\n$exit_code" > "$RESPONSE_DIR/$repo_name" + log_debug "Package '${pkg_name}' is not already installed, so install + v${pkg_version}." + if ! output="$(sudo rpm --install --verbose --hash "${rpm_path}")"; then + reply "${err_rpm_install}" \ + "Cannot install package '${pkg_name}' v${pkg_version}: ${output}." + return + fi -done < <(inotifywait --monitor --event create "$REQUEST_DIR") + # Package deployed. + reply 0 "RPM package '${pkg_name}' v${pkg_version} has been deployed." +) + +process_loop() ( + # Loop on every request. + if "${keep_open}"; then + log_info "Start listening unixsock with keep-open at '${deployer_sock}'." + while read -r request; do + log_info "Process new request '${request}'." + process_request + done < <(ncat --listen --keep-open --unixsock "${deployer_sock}") + else + log_info "Start listening unixsock without keep-open at '${deployer_sock}'." + while read -r request; do + log_info "Process new request '${request}'." + process_request + done < <(ncat --listen --unixsock "${deployer_sock}") + fi + + # Kill all remaining subprocesses only if daemon. + log_info "End of loop." + if "${daemon}"; then + log_debug "Kill child jobs." + jobs -p | xargs kill 2>/dev/null || true + fi + + # Remove sock file + log_debug "Remove deployer unixsock file." + rm --force "${deployer_sock}" + log_info "End of process." +) + +main() ( + # Parse arguments. + daemon="false" + testing="false" + keep_open="true" + config_file="${DEFAULT_CONFIG_FILE}" + errors_file="${DEFAULT_ERRORS_FILE}" + verbosity_option="" + verbosity_level="${DEFAULT_VERBOSITY_LEVEL}" + if ! args="$(getopt --name "${PROGRAM_NAME}" \ + --options c:de:hqstv \ + --longoptions conf:,daemon,errs:,help,quiet,stop,test,verbose \ + -- "$@")"; then + usage + fail "Bad arguments." + fi + eval set -- "${args}" + + while true; do + case "$1" in + -c | --conf) + config_file="$2" + shift 2 + ;; + -d | --daemon) + daemon="true" + shift + ;; + -e | --errs) + errors_file="$2" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + -q | --quiet) + if [ "${verbosity_option}" ]; then + usage + fail "Options $1 and ${verbosity_option} are mutually exclusive." + fi + verbosity_option="$1" + ((verbosity_level -= 10)) + shift + ;; + -s | --stop) + keep_open="false" + shift + ;; + -t | --test) + testing="true" + shift + ;; + -v | --verbose) + if [ "${verbosity_option}" ]; then + usage + fail "Options $1 and ${verbosity_option} are mutually exclusive." + fi + verbosity_option="$1" + ((verbosity_level += 10)) + shift + ;; + --) + shift + break + ;; + *) + usage + fail "Unexpected option '$1'." + ;; + esac + done + if [ "$@" ]; then + usage + fail "Unexpected extra arguments '$*'." + fi + + # Load config file. + if [ ! -r "${config_file}" ]; then + fail "Config file '${config_file}' is not readable." + fi + log_info "Load config file '${config_file}'." + # shellcheck source=./conf/netoik-cicd.conf.sample + source "${config_file}" + + # Load errors file. + if [ ! -r "${errors_file}" ]; then + fail "Errors file '${errors_file}' is not reachable." + fi + log_info "Load errors file '${errors_file}'." + # shellcheck source=./conf/errors.conf.sample + source "${errors_file}" + + # Check variables in config file. + if [ -z "${deployer_sock}" ]; then + fail "Variable deployer_sock is empty." "${err_deployer_sock_empty}" + fi + if [ ! -d "$(dirname "${deployer_sock}")" ]; then + fail "Dirname of deployer_sock='${deployer_sock}' is not a directory." \ + "${err_deployer_sock_dir_not_directory}" + fi + if [ ! -w "$(dirname "${deployer_sock}")" ]; then + fail "Directory of deployer_sock='${deployer_sock}' is not writable." \ + "${err_deployer_sock_dir_not_writable}" + fi + if [ -e "${deployer_sock}" ]; then + fail "Sock deployer_sock='${deployer_sock}' is already in use." \ + "${err_deployer_sock_already_in_use}" + fi + + if [ -z "${repos_dir}" ]; then + fail "Variable repos_dir is empty." "${err_repos_dir_empty}" + fi + if [ ! -x "${repos_dir}" ]; then + fail "Directory repos_dir='${repos_dir}' is not accessible." \ + "${err_repos_dir_not_accessible}" + fi + + if [ -z "${rpms_dir}" ]; then + fail "Variable rpms_dir is empty." "${err_rpms_dir_empty}" + fi + + if [ -z "${runner_username}" ]; then + fail "Variable runner_username is empty." "${err_runner_username_empty}" + fi + if ! id --user --name "${runner_username}" 1>/dev/null; then + fail "Runner user '${runner_username}' does not exist." \ + "${err_runner_user_not_exist}" + fi + + if [ -z "${runner_deployer_groupname}" ]; then + fail "Variable runner_deployer_username is empty." "${err_runner_deployer_groupname_empty}" + fi + if ! getent group "${runner_deployer_groupname}"; then + fail "Runner-deployer group '${runner_deployer_groupname}' does not exist." \ + "${err_runner_deployer_group_not_exist}" + fi + + # Stop now if we are only testing config. + if "${testing}"; then + exit 0 + fi + + # Set right access in background after nc listen. + ( + inotifywait --event create "$(dirname "${deployer_sock}")" --quiet --quiet + chmod 775 "${deployer_sock}" + chgrp "${runner_deployer_groupname}" "${deployer_sock}" + ) & + + # Run process loop in background or foreground. + if "${daemon}"; then + log_info "Run process loop in background." + process_loop & + else + log_info "Run process loop in foreground." + process_loop + jobs -p | xargs kill 2>/dev/null || true + fi +) + +main "$@" diff --git a/src/newcommit.sh b/src/newcommit.sh deleted file mode 100644 index 0bd5acc..0000000 --- a/src/newcommit.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/bash -# -# This binary is made to be run by git server, it checks code validity. - -# Exit immediately if any command fails. -set -e - -# Exit with the last non-zero exit code. -set -o pipefail - -# Name of current gitops pipeline. -pipeline="NEWTAG" - -log () { - echo -e "[$pipeline] $(date --rfc-3339=s) - $1" -} - -fail () { - if [ $# -eq 1 ]; then - echo "$1" 1>&2 - fi - exit 1 -} - -# Check format. -log "Check format." -make check_format - -# Check linting. -log "Check linting." -make check_linting - -# Run unit tests. -log "Run unit tests." -make unit_test diff --git a/src/newtag.sh b/src/newtag.sh deleted file mode 100755 index c207501..0000000 --- a/src/newtag.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/bash -# -# This binary is made to be run by git server, it builds a rpm package and send a request to root in order -# to deploy the package on the server. - -# Exit immediately if any command fails. -set -e - -# Exit with the last non-zero exit code. -set -o pipefail - -# Name of current gitops pipeline. -pipeline="NEWTAG" - -log () { - echo -e "[$pipeline] $(date --rfc-3339=s) - $1" -} - -fail () { - if [ $# -eq 1 ]; then - echo "$1" 1>&2 - fi - exit 1 -} - -# Retrieve necessary details about package. -pkg_name=$(make name) -[ -z $pkg_name ] && fail "Empty result for target 'make name'." -pkg_version=$(make version) -[ -z $pkg_version ] && fail "Empty result for target 'make version'." - -# Check code validity. -log "Check code validity for $pkg_name v$pkg_version." -make check_format -make check_linting -make unit_test - -# Make tarball with source code. -log "Make source tarball for $pkg_name v$pkg_version." -make tarball - -# Build rpm package. -log "Build rpm package." -rpmbuild -bb "$pkg_name.spec" - -# Cleanup last response. -[ -f "$NETOIK_CICD_DEPLOYER_RESPONSE_DIR/$pkg_name" ] || touch "$NETOIK_CICD_DEPLOYER_RESPONSE_DIR/$pkg_name" -sed -i "d" "$NETOIK_CICD_DEPLOYER_RESPONSE_DIR/$pkg_name" - -# Send request to deployer with a little delay in background. -log "Install or update rpm package." -echo "$pkg_version" > "$NETOIK_CICD_DEPLOYER_REQUEST_DIR/$pkg_name" - -# Wait for response from deployer. -inotifywait --timeout 600 --event modify "$NETOIK_CICD_DEPLOYER_RESPONSE_DIR/$pkg_name" >/dev/null - -# Get content of the response. -while read line; do - [ -z "$previous" ] || echo "$previous" - previous="$line" -done < "$NETOIK_CICD_DEPLOYER_RESPONSE_DIR/$pkg_name" - -# Exit now with exit code found in response. -exit_code=$(printf "%d\n" "$previous") -exit $exit_code diff --git a/src/pipeline.sh b/src/pipeline.sh new file mode 100755 index 0000000..425143e --- /dev/null +++ b/src/pipeline.sh @@ -0,0 +1,253 @@ +#!/bin/bash + +################################################################################ +# # +# netoik-cicd-pipeline # +# # +# This binary is made to be run by git server, it asks runner server to check # +# code and eventually to deploy rpm package. # +# # +# This script is coded with respect of google shell styleguide: # +# https://google.github.io/styleguide/shellguide.html # +# # +################################################################################ + +# Exit immediately if any command fails. +set -e + +# Exit with the last non-zero fail code. +set -o pipefail + +# Program constants. +readonly PACKAGE_NAME="netoik-cicd" +readonly PROGRAM_NAME="${PACKAGE_NAME}-pipeline" + +# Declare default variables. +readonly DEFAULT_CONFIG_FILE="/etc/${PACKAGE_NAME}/${PACKAGE_NAME}.conf" +readonly DEFAULT_ERRORS_FILE="/etc/${PACKAGE_NAME}/errors.conf" + +fail() ( + msg="$1" + if [ $# -eq 2 ]; then + code="$2" + else + code=1 + fi + + echo "${msg}" >&2 + if [ "${code}" -gt 0 ]; then + exit "${code}" + else + echo "Undefined error code for '${msg}'." >&2 + exit 1 + fi +) + +usage() ( + echo "Usage: ${PROGRAM_NAME} [OPTION]... PIPELINE +Asks runner server to check code and eventually deploy rpm package. See section Pipelines for positionnal argument PIPELINE. + +Mandatory argumentes for long options are mandatory for short options too. + + -c, --conf= name of configuration file, default + to ${DEFAULT_CONFIG_FILE} + -e, --errs= name of errors file, default to + ${DEFAULT_ERRORS_FILE} + -H, --hash= repo hash to checkout + -n, --name= repo name to clone + -t, --test just test config file and exit + -T, --tag= repo tag to deploy + + -h, --help display this help message and exit + +Pipelines: + newcommit + should be called when a new commit is pushed to repo, it will clone repo and + check code validity + it needs options -H|--hash, -n|--name + newtag + should be called when a new tag is pushed to repo, it will clone repo, + check code validity and deploy rpm package + it needs options -H|--hash, -n|--name, -T|--tag +" +) + +main() ( + # Parse arguments. + testing="false" + config_file="${DEFAULT_CONFIG_FILE}" + errors_file="${DEFAULT_ERRORS_FILE}" + if ! args="$(getopt --name "${PROGRAM_NAME}" \ + --options c:e:H:hn:tT: \ + --longoptions conf:,errs:,hash:,help,name:,test,tag:,help \ + -- "$@")"; then + usage + fail "Bad arguments." + fi + eval set -- "${args}" + + while true; do + case "$1" in + -c | --conf) + config_file="$2" + shift 2 + ;; + -e | --errs) + errors_file="$2" + shift 2 + ;; + -H | --hash) + repo_hash="$2" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + -n | --name) + repo_name="$2" + shift 2 + ;; + -t | --test) + testing="true" + shift + ;; + -T | --tag) + repo_tag="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + usage + fail "Unexpected option '$1'." + ;; + esac + done + if [ $# -ne 1 ]; then + usage + fail "Missing positionnal argument PIPELINE." + fi + pipeline="$1" + + # Generate pipeline tmp sock. + tsp="$(date +%s)" + random="$(echo "${RANDOM}" | md5sum | head --bytes 32)" + rd_pipeline_sock_dir="${pipeline_sock_dir}/${pipeline}-${tsp}-${random}" + rd_pipeline_sock="${pipeline_sock_dir}/pipeline.sock" + + # Check pipeline with options. + case "${pipeline}" in + newcommit) + if [ -z "${repo_name}" ]; then + usage + fail "Missing option -n|--name for pipeline '${pipeline}'." + fi + if [ -z "${repo_hash}" ]; then + usage + fail "Missing option -H|--hash for pipeline '${pipeline}'." + fi + runner_request="$(jq --null-input --compact-output \ + --arg s "${rd_pipeline_sock}" \ + --arg n "${repo_name}" \ + --arg h "${repo_hash}" \ + '{"response_sock":$s,"repo_name":$n,"repo_hash":$h}')" + ;; + newtag) + if [ -z "${repo_name}" ]; then + usage + fail "Missing option -n|--name for pipeline '${pipeline}'." + fi + if [ -z "${repo_hash}" ]; then + usage + fail "Missing option -H|--hash for pipeline '${pipeline}'." + fi + if [ -z "${repo_tag}" ]; then + usage + fail "Missing option -T|--tag for pipeline '${pipeline}'." + fi + runner_request="$(jq --null-input --compact-output \ + --arg s "${rd_pipeline_sock}" \ + --arg n "${repo_name}" \ + --arg h "${repo_hash}" \ + --arg t "${repo_tag}" \ + '{"response_sock":$s,"repo_name":$n,"repo_hash":$h,"repo_tag":$t}')" + ;; + *) + usage + fail "Invalid pipeline '$1'." + ;; + esac + + # Load config file. + if [ ! -r "${config_file}" ]; then + fail "Config file '${config_file}' is not readable." + fi + # shellcheck source=./conf/netoik-cicd.conf.sample + source "${config_file}" + + # Load errors file. + if [ ! -r "${errors_file}" ]; then + fail "Errors file '${errors_file}' is not reachable." + fi + # shellcheck source=./conf/errors.conf.sample + source "${errors_file}" + + # Check variables in config file. + if [ -z "${runner_sock}" ]; then + fail "Variable runner_sock is empty." "${err_runner_sock_empty}" + fi + if [ ! -S "${runner_sock}" ]; then + fail "Sock runner_sock='${runner_sock}' does not exist." \ + "${err_runner_sock_not_exist}" + fi + + if [ -z "${runner_timeout}" ]; then + fail "Variable runner_timeout is empty." "${err_runner_timeout_empty}" + fi + if [ "${runner_timeout}" -lt 0 ]; then + fail "Runner timeout is not valid: ${runner_timeout}." \ + "${err_runner_timeout_not_valid}" + fi + + if [ -z "${git_runner_groupname}" ]; then + fail "Variable git_runner_groupname is empty." "${err_git_runner_groupname_empty}" + fi + if ! getent group "${git_runner_groupname}"; then + fail "Git-runner group '${git_runner_groupname}' does not exist." \ + "${err_git_runner_group_not_exist}" + fi + + # Stop now if testing. + if "${testing}"; then + exit 0 + fi + + # Send request to runner. + mkdir "${rd_pipeline_sock_dir}" + ( + inotifywait --quiet --quiet --event create "${rd_pipeline_sock_dir}" + chmod 775 "${rd_pipeline_sock}" + chgrp "${git_runner_groupname}" "${rd_pipeline_sock}" + echo "${runner_request}" | ncat --unixsock "${runner_sock}" + ) & + + # Wait for runner response. + if [ "${runner_timeout}" -gt 0 ]; then + response="$(timeout "${runner_timeout}" ncat --listen \ + --unixsock "${rd_pipeline_sock}")" + else + response="$(ncat --listen --unixsock "${rd_pipeline_sock}")" + fi + + # Remove random directory. + rm --recursive "${rd_pipeline_sock_dir}" + + # Display response. + echo -e "$(echo "${response}" | jq .msg)" + exit "$(echo "${response}" | jq .code)" +) + +main "$@" diff --git a/src/runner.sh b/src/runner.sh new file mode 100755 index 0000000..41394a0 --- /dev/null +++ b/src/runner.sh @@ -0,0 +1,508 @@ +#!/bin/bash + +################################################################################ +# # +# netoik-cicd-runner # +# # +# This binary is made to be run by an unprivileged user, it clones a git repo, # +# checks code validity, builds eventually a rpm package and asks deployer # +# to deploy it. # +# # +# This script is coded with respect of google shell styleguide: # +# https://google.github.io/styleguide/shellguide.html # +# # +################################################################################ + +# Exit immediately if any command fails. +set -e + +# Exit with the last non-zero fail code. +set -o pipefail + +# Program constants. +readonly PACKAGE_NAME="netoik-cicd" +readonly PROGRAM_NAME="${PACKAGE_NAME}-runner" + +# Declare default variables. +readonly DEFAULT_CONFIG_FILE="/etc/${PACKAGE_NAME}/${PACKAGE_NAME}.conf" +readonly DEFAULT_ERRORS_FILE="/etc/${PACKAGE_NAME}/errors.conf" + +# Declare levels variables. +readonly VERBOSITY_LEVELS=( + [10]="CRITICAL" + [20]="ERROR" + [30]="WARN" + [40]="INFO" + [50]="DEBUG" +) +readonly DEFAULT_VERBOSITY_LEVEL=40 + +# Functions. +log() ( + level="$1" + msg="$2" + + if [ "${level}" -le "${verbosity_level}" ]; then + echo "${VERBOSITY_LEVELS[${level}]} - ${msg}" + fi +) +log_critical() (log 10 "$1" >&2) +log_error() (log 20 "$1" >&2) +log_warn() (log 30 "$1" >&2) +log_info() (log 40 "$1") +log_debug() (log 50 "$1") + +fail() ( + msg="$1" + if [ $# -eq 2 ]; then + code="$2" + else + code=1 + fi + + log_critical "${msg}" + if [ "${code}" -gt 0 ]; then + exit "${code}" + else + log_error "Undefined error code for '${msg}'." + exit 1 + fi +) + +usage() ( + echo "Usage: ${PROGRAM_NAME} [OPTION]... +Start runner server, wait for unixsock requests to clone git repo, run tests +and build rpm package. + +Mandatory argumentes for long options are mandatory for short options too. + + -c, --conf= name of configuration file, default + to ${DEFAULT_CONFIG_FILE} + -d, --daemon run processing loop in background + -e, --errs= name of errors file, default to + ${DEFAULT_ERRORS_FILE} + -q, --quiet set level verbosity to WARN, default to INFO + -s, --stop stop listening after first request process + -t, --test just test config file and do not run start loop + -v, --verbose set level verbosity to DEBUG, default to INFO + + -h, --help display this help message and exit +" +) + +reply() ( + code="$1" + msg="$2" + + if [ "${code}" -gt 0 ]; then + log_error "${msg}" + else + log_info "${msg}" + fi + + if ! output="$(jq --null-input \ + --compact-output \ + --arg c "${code}" \ + --arg m "${msg}" \ + '{"code":$c|tonumber,"msg":$m}' | + ncat --unixsock "${response_sock}" 2>&1)"; then + log_error "Cannot write to sock '${response_sock}': ${output}." + fi +) + +fwd_reply() ( + read -r json + + if ! output="$(echo "${json}" | ncat --unixsock 2>&1)"; then + log_error "Cannot write to sock '${response_sock}': ${output}." + fi +) + +process_request() ( + # Get repo_name from json. + log_debug "Get response sock file." + if ! response_sock="$(echo "${request}" | + jq --raw-output ".response_sock" 2>&1)"; then + log_error "Cannot parse json request '${request}': ${response_sock}." + return + fi + if [ "${response_sock}" = "null" ]; then + log_error "Missing key 'response_sock' in request '${request}'." + return + fi + if [ -z "${response_sock}" ]; then + log_error "Empty value for key 'response_sock' in request '${request}'." + return + fi + if [ ! -S "${response_sock}" ]; then + log_error \ + "Sock file '${response_sock}' does not exist in request '${request}'." + return + fi + log_debug "Sock file '${response_sock}' is reachable." + + # Get repo_name from json. + log_debug "Get repo_name from json." + if ! repo_name="$(echo "${request}" | + jq --raw-output ".repo_name" 2>&1)"; then + reply "${err_json_bad_format}" \ + "Cannot parse json '${request}': ${repo_name}." + return + fi + if [ "${repo_name}" = "null" ]; then + reply "${err_repo_name_missing}" "Missing key repo_name in '${request}'." + return + fi + if [ -z "${repo_name}" ]; then + reply "${err_repo_name_empty}" \ + "Empty value for key repo_name in '${request}'." + return + fi + log_debug "Got pkg name: '${repo_name}'." + + # Get repository folder. + repo_dir="${repos_dir}/${repo_name}.git" + if [ ! -d "${repo_dir}" ]; then + reply "${err_repo_dir_not_exist}" "Repository '${repo_dir}' does not exist." + return + fi + log_debug "Repository is present at '${repo_dir}'." + + # Get repo_hash from json. + log_debug "Get repo_hash from json." + if ! repo_hash=$(echo "${request}" | jq --raw-output ".repo_hash" 2>&1); then + reply "${err_json_bad_format}" \ + "Cannot parse json '${request}': ${repo_hash}." + return + fi + if [ "${repo_hash}" = "null" ]; then + reply "${err_repo_hash_missing}" "Missing key repo_hash in '${request}'." + return + fi + if [ -z "${repo_hash}" ]; then + reply "${err_repo_hash_empty}" \ + "Empty value for key repo_hash in '${request}'." 1>&2 + return + fi + + # Get repo_tag from json. + log_debug "Get repo_tag from json." + if ! repo_tag=$(echo "${request}" | jq --raw-output ".repo_tag"); then + reply "${err_json_bad_format}" \ + "Cannot parse json '${request}': ${repo_tag}." + return + fi + if [ -z "${repo_tag}" ]; then + reply "${err_repo_tag_empty}" \ + "Empty value for key repo_tag in '${request}'." + return + fi + + # Clone repo and move into it. + rand="$(echo "${RANDOM}" | md5sum | head --bytes 32)" + repo_clone="${runner_cloning_dir}/${repo_name}-${repo_hash}-${rand}" + log_info "Clone '$repo_name'." + if ! output="$(git clone "$repo_dir" "$repo_clone" 2>&1)"; then + reply "${err_clone_repo}" "Cannot clone repo '${repo_name}': ${output}." + return + fi + cd "$repo_clone" + + # Checkout git hash. + log_info "Checkout hash $repo_hash." + if ! output="$(git checkout "$repo_hash" 2>&1)"; then + reply "${err_checkout_hash}" \ + "Cannot checkout hash '${repo_hash}': ${output}." + return + fi + + # Check code validity. + log_info "Check code validity." + if ! output="$(make check_format 2>&1)"; then + reply "${err_check_format}" "Check format error: ${output}." + return + fi + if ! output="$(make check_linting 2>&1)"; then + reply "${err_check_linting}" "Check linting error: ${output}." + return + fi + if ! output="$(make unit_test 2>&1)"; then + reply "${err_unit_test}" "Unit test error: ${output}." + return + fi + + # Stop now if no tag specified. + if [ "$repo_tag" = "null" ]; then + reply 0 "Hash '${repo_hash}' OK for repo '${repo_name}'." + return + fi + + # Build rpm package. + echo "Build RPM package." + if ! output="$(git tag --message="$repo_tag" "$repo_tag" 2>&1)"; then + reply "${err_add_tag}" "Cannot add git tag '${repo_tag}': ${output}." + return + fi + if ! output="$(make tarball 2>&1)"; then + reply "${err_make_tarball}" \ + "Cannot make tarball for tag '${repo_tag}': ${output}." + return + fi + if ! output="$(rpmbuild -bb "$repo_name.spec" 2>&1)"; then + reply "${err_rpm_build}" \ + "Cannot build rpm for tag '${repo_tag}': ${output}." + return + fi + + # Deploy rpm package. + log_info "Deploy RPM package." + deployer_request="$(jq --null-input \ + --compact-output \ + --arg s "${repo_clone}/runner.sock" \ + --arg n "${repo_name}" \ + --arg v "$(make version)" \ + '{"response_sock":$s,"pkg_name":$n,"pkg_version":$v}')" + ( + inotifywait --event create "${repo_clone}" 1>/dev/null 2>/dev/null + chmod 775 "${repo_clone}/runner.sock" + chgrp "${runner_deployer_groupname}" "${repo_clone}/runner.sock" + echo "${deployer_request}" | ncat --unixsock "${deployer_sock}" + ) & + + # Wait for deployer response. + if [ "${deployer_timeout}" -gt 0 ]; then + timeout "${deployer_timeout}" ncat --listen --unixsock runner.sock | + fwd_reply + else + ncat --listen --unixsock runner.sock | fwd_reply + fi +) + +process_loop() ( + # Loop on every request. + if "${keep_open}"; then + log_info "Start listening unixsock with keep-open at '${runner_sock}'." + while read -r request; do + log_info "Process new request '${request}'." + process_request + done < <(ncat --listen --keep-open --unixsock "${runner_sock}") + else + log_info "Start listening unixsock without keep-open at '${runner_sock}'." + while read -r request; do + log_info "Process new request '${request}'." + process_request + done < <(ncat --listen --unixsock "${runner_sock}") + fi + + # Kill all remaining subprocesses only if daemon. + log_info "End of loop." + if "${daemonize}"; then + log_debug "Kill child jobs." + jobs -p | xargs kill 2>/dev/null || true + fi + + # Remove sock file. + log_debug "Remove deployer unixsock file." + rm --force "${runner_sock}" + log_info "End of process." +) + +main() ( + # Parse arguments. + daemonize="false" + testing="false" + keep_open="true" + config_file="${DEFAULT_CONFIG_FILE}" + errors_file="${DEFAULT_ERRORS_FILE}" + verbosity_option="" + verbosity_level="${DEFAULT_VERBOSITY_LEVEL}" + if ! args="$(getopt --name "${PROGRAM_NAME}" \ + --options c:de:hqstv \ + --longoptions conf:,daemon,errs:,help,quiet,stop,test,verbose \ + -- "$@")"; then + usage + fail "Bad arguments." + fi + eval set -- "${args}" + + while true; do + case "$1" in + -c | --conf) + config_file="$2" + shift 2 + ;; + -d | --daemon) + daemonize="true" + shift + ;; + -e | --errs) + errors_file="$2" + shift 2 + ;; + -h | --help) + usage + exit 0 + ;; + -q | --quiet) + if [ "${verbosity_option}" ]; then + usage + fail "Options $1 and ${verbosity_option} are mutually exclusive." + fi + verbosity_option="$1" + ((verbosity_level -= 10)) + shift + ;; + -s | --stop) + keep_open="false" + shift + ;; + -t | --test) + testing="true" + shift + ;; + -v | --verbose) + if [ "${verbosity_option}" ]; then + usage + fail "Options $1 and ${verbosity_option} are mutually exclusive." + fi + verbosity_option="$1" + ((verbosity_level += 10)) + shift + ;; + --) + shift + break + ;; + *) + usage + fail "Unexpected option '$1'." + ;; + esac + done + if [ "$@" ]; then + usage + fail "Unexpected extra arguments '$*'." + fi + + # Load config file. + if [ ! -r "${config_file}" ]; then + fail "Config file '${config_file}' is not readable." + fi + log_info "Load config file '${config_file}'." + # shellcheck source=./conf/netoik-cicd.conf.sample + source "${config_file}" + + # Load errors file. + if [ ! -r "${errors_file}" ]; then + fail "Errors file '${errors_file}' is not reachable." + fi + log_info "Load errors file '${errors_file}'." + # shellcheck source=./conf/errors.conf.sample + source "${errors_file}" + + # Check variables in config file. + if [ -z "${deployer_sock}" ]; then + fail "Variable deployer_sock is empty." "${err_deployer_sock_empty}" + fi + if [ ! -S "${deployer_sock}" ]; then + fail "Sock deployer_sock='${deployer_sock}' does not exist." \ + "${err_deployer_sock_not_exist}" + fi + + if [ -z "${deployer_timeout}" ]; then + fail "Variable deployer_timeout is empty." "${err_deployer_timeout_empty}" + fi + if [ "${deployer_timeout}" -lt 0 ]; then + fail "Deployer timeout is not valid: ${deployer_timeout}." \ + "${err_deployer_timeout_not_valid}" + fi + + if [ -z "${runner_sock}" ]; then + fail "Variable runner_sock is empty." "${err_runner_sock_empty}" + fi + if [ ! -d "$(dirname "${runner_sock}")" ]; then + fail "Dirname of runner_sock='${runner_sock}' is not a directory." \ + "${err_runner_sock_dir_not_directory}" + fi + if [ ! -w "$(dirname "${runner_sock}")" ]; then + fail "Directory of runner_sock='${runner_sock}' is not writable." \ + "${err_runner_sock_dir_not_writable}" + fi + if [ -e "${runner_sock}" ]; then + fail "Sock deployer_sock='${runner_sock}' is already in use." \ + "${err_runner_sock_already_in_use}" + fi + + if [ -z "${runner_cloning_dir}" ]; then + fail "Variable runner_cloning_dir is empty." \ + "${err_runner_cloning_dir_empty}" + fi + if [ ! -d "${runner_cloning_dir}" ]; then + fail "File runner_cloning_dir='${runner_cloning_dir}' is not a directory." \ + "${err_runner_cloning_dir_not_directory}" + fi + if [ ! -w "${runner_cloning_dir}" ]; then + fail "Dir runner_cloning_dir='${runner_cloning_dir}' is not writable." \ + "${err_runner_cloning_dir_not_writable}" + fi + + if [ -z "${repos_dir}" ]; then + fail "Variable repos_dir is empty." "${err_repos_dir_empty}" + fi + if [ ! -d "${repos_dir}" ]; then + fail "File repos_dir='${repos_dir}' is not a directory." \ + "${err_repos_dir_not_directory}" + fi + if [ ! -x "${repos_dir}" ]; then + fail "Directory repos_dir='${repos_dir}' is not accessible." \ + "${err_repos_dir_not_accessible}" + fi + + if [ -z "${deployer_username}" ]; then + fail "Variable deployer_username is empty." "${err_deployer_username_empty}" + fi + if ! id --user --name "${deployer_username}" 1>/dev/null; then + fail "Deployer user '${deployer_username}' does not exist." \ + "${err_deployer_user_not_exist}" + fi + + if [ -z "${runner_deployer_groupname}" ]; then + fail "Variable runner_deployer_groupname is empty." "${err_runner_deployer_groupname_empty}" + fi + if ! getent group "${runner_deployer_groupname}"; then + fail "Runner-deployer group '${runner_deployer_groupname}' does not exist." \ + "${err_runner_deployer_group_not_exist}" + fi + + if [ -z "${git_runner_groupname}" ]; then + fail "Variable git_runner_groupname is empty." "${err_git_runner_groupname_empty}" + fi + if ! getent group "${git_runner_groupname}"; then + fail "Git-runner group '${git_runner_groupname}' does not exist." \ + "${err_git_runner_group_not_exist}" + fi + + # Stop now if we are only testing config. + if "${testing}"; then + exit 0 + fi + + # Set right access in background after nc listen. + ( + inotifywait --event create "$(dirname "${runner_sock}")" --quiet --quiet + chmod 775 "${runner_sock}" + chgrp "${git_runner_groupname}" "${runner_sock}" + ) & + + # Run process loop in background or foreground. + if "${daemonize}"; then + log_info "Run process loop in background." + process_loop & + else + log_info "Run process loop in foreground." + process_loop + jobs -p | xargs kill 2>/dev/null || true + fi +) + +main "$@" diff --git a/systemd/netoik-cicd-deployer.service b/systemd/deployer.service similarity index 58% rename from systemd/netoik-cicd-deployer.service rename to systemd/deployer.service index 4edb116..e7fe829 100644 --- a/systemd/netoik-cicd-deployer.service +++ b/systemd/deployer.service @@ -3,9 +3,9 @@ Description=Netoik automatic deployer After=network.target [Service] -User=root -Group=root -ExecStart=/usr/bin/netoik-cicd-deployer /etc/netoik-cicd/deployer.conf +User=netoik-cicd-deployer +Group=netoik-cicd-deployer +ExecStart=/usr/bin/netoik-cicd-deployer Restart=always [Install] diff --git a/systemd/runner.service b/systemd/runner.service new file mode 100644 index 0000000..45be478 --- /dev/null +++ b/systemd/runner.service @@ -0,0 +1,12 @@ +[Unit] +Description=Netoik automatic runner +After=network.target netoik-cicd-deployer + +[Service] +User=netoik-cicd-runner +Group=netoik-cicd-runner +ExecStart=/usr/bin/netoik-cicd-runner +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/tests/deployer.bats b/tests/deployer.bats new file mode 100644 index 0000000..028fa1c --- /dev/null +++ b/tests/deployer.bats @@ -0,0 +1,106 @@ +# +# Deployer unit tests using BATS framework. +# + +bats_require_minimum_version 1.5.0 + +setup() { + source "${PWD}/tests/tests.conf" + make build >/dev/null + DESTDIR="${temp}" make install >/dev/null + install --no-target-directory \ + "${PWD}/tests/tests.conf" \ + "${temp}${sysconf_dir}/${name}/${name}.conf" + install --directory "${temp}${var_dir}/${name}/repositories/test.git" + install --directory "${temp}${var_dir}/${name}/RPMS" + source "${errs_file}" +} + +teardown() { + rm --recursive --force "${temp}" + make clean >/dev/null +} + +@test "run with bad arguments" { + run -1 "${deployer_bin}" -z + run -1 "${deployer_bin}" --zzz + run -1 "${deployer_bin}" zzz + + run -1 "${deployer_bin}" --conf + run -1 "${deployer_bin}" --conf=/path/to/nowhere + + run -1 "${deployer_bin}" --errs + run -1 "${deployer_bin}" --errs=/path/to/nowhere + + run -1 "${deployer_bin}" --verbose --quiet +} + +@test "run without loop" { + "${deployer_bin}" --conf="${conf_file}" --errs="${errs_file}" --test --verbose +} + +send_request() ( + request="$1" + + "${deployer_bin}" --conf="${conf_file}" --errs="${errs_file}" \ + --daemon --stop --verbose || return $? + ( + inotifywait --event create --quiet --quiet "${response_sock_dir}" + echo "${request}" | ncat --unixsock "${deployer_sock}" + ) & + return "$(ncat --listen --unixsock "${response_sock}" | jq .code)" +) + +@test "run with err_pkg_name_missing" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_pkg_name_missing}" + [ "${status}" -eq "${err_pkg_name_missing}" ] +} + +@test "run with err_pkg_name_empty" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"pkg_name":""}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_pkg_name_empty}" + [ "${status}" -eq "${err_pkg_name_empty}" ] +} + +@test "run with err_repo_dir_not_exist" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"pkg_name":"not_exist"}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_dir_not_existyy}" + [ "${status}" -eq "${err_repo_dir_not_exist}" ] +} + +@test "run with err_pkg_version_missing" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"pkg_name":"test"}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_pkg_version_missing}" + [ "${status}" -eq "${err_pkg_version_missing}" ] +} + +@test "run with err_pkg_version_empty" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"pkg_name":"test","pkg_version":""}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_pkg_version_empty}" + [ "${status}" -eq "${err_pkg_version_empty}" ] +} + +@test "run with err_rpm_path_not_exist" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"pkg_name":"test","pkg_version":"0.1"}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_rpm_path_not_exist}" + [ "${status}" -eq "${err_rpm_path_not_exist}" ] +} diff --git a/tests/pipeline.bats b/tests/pipeline.bats new file mode 100644 index 0000000..6ba38ce --- /dev/null +++ b/tests/pipeline.bats @@ -0,0 +1,61 @@ +# +# Pipeline uni tests using BATS framework. +# + +bats_require_minimum_version 1.5.0 + +setup() { + source "${PWD}/tests/tests.conf" + make build >/dev/null + DESTDIR="${temp}" make install >/dev/null + install --no-target-directory \ + "${PWD}/tests/tests.conf" \ + "${temp}${sysconf_dir}/${name}/${name}.conf" + source "${errs_file}" +} + +teardown() { + rm --recursive --force "${temp}" + make clean >/dev/null +} + +@test "run with bad arguments" { + run -1 "${pipeline_bin}" + run -1 "${pipeline_bin}" bad_pipeline + + run -1 "${pipeline_bin}" newcommit + run -1 "${pipeline_bin}" -H hash newcommit + run -1 "${pipeline_bin}" -n name newcommit + + run -1 "${pipeline_bin}" newtag + run -1 "${pipeline_bin}" -H hash newtag + run -1 "${pipeline_bin}" -n name newtag + run -1 "${pipeline_bin}" -T tag newtag + run -1 "${pipeline_bin}" -H hash -n name newtag + run -1 "${pipeline_bin}" -H hash -T tag newtag + run -1 "${pipeline_bin}" -n name -T tag newtag + + run -1 "${pipeline_bin}" -c /path/to/nowhere -H hash -n name newcommit + run -1 "${pipeline_bin}" -e /path/to/nowhere -H hash -n name newcommit +} + +@test "run without runner sock" { + run "${pipeline_bin}" --conf="${conf_file}" --errs="${errs_file}" \ + --test --hash hash --name name newcommit + echo "status: got ${status}, expected ${err_runner_sock_not_exist}" + [ "${status}" -eq "${err_runner_sock_not_exist}" ] +} + +@test "run newcommit ok" { + ncat --listen --unixsock "${runner_sock}" & + "${pipeline_bin}" --conf="${conf_file}" --errs="${errs_file}" \ + --test --hash hash --name name newcommit + echo "done" | ncat --unixsock "${runner_sock}" +} + +@test "run newtag ok" { + ncat --listen --unixsock "${runner_sock}" & + "${pipeline_bin}" --conf="${conf_file}" --errs="${errs_file}" \ + --test --hash hash --name name --tag tag newtag + echo "done" | ncat --unixsock "${runner_sock}" +} diff --git a/tests/runner.bats b/tests/runner.bats new file mode 100644 index 0000000..f532451 --- /dev/null +++ b/tests/runner.bats @@ -0,0 +1,110 @@ +# +# Deployer unit tests using BATS framework. +# + +bats_require_minimum_version 1.5.0 + +setup() { + source "${PWD}/tests/tests.conf" + make build >/dev/null + DESTDIR="${temp}" make install >/dev/null + install --no-target-directory \ + "${PWD}/tests/tests.conf" \ + "${temp}${sysconf_dir}/${name}/${name}.conf" + install --directory "${temp}${var_dir}/${name}/repositories/test.git" + install --directory "${temp}${var_dir}/${name}/RPMS" + source "${errs_file}" +} + +teardown() { + rm --recursive --force "${temp}" + make clean >/dev/null +} + +@test "run with bad arguments" { + run -1 "${runner_bin}" -z + run -1 "${runner_bin}" --zzz + run -1 "${runner_bin}" zzz + + run -1 "${runner_bin}" --conf + run -1 "${runner_bin}" --conf=/path/to/nowhere + + run -1 "${runner_bin}" --errs + run -1 "${runner_bin}" --errs=/path/to/nowhere + + run -1 "${runner_bin}" --verbose --quiet +} + +@test "run without loop" { + ncat --listen --unixsock "${deployer_sock}" & + "${runner_bin}" --conf="${conf_file}" --errs="${errs_file}" --test --verbose + echo "done" | ncat --unixsock "${deployer_sock}" +} + +send_request() ( + request="$1" + + ncat --listen --unixsock "${deployer_sock}" & + "${runner_bin}" --conf="${conf_file}" --errs="${errs_file}" \ + --daemon --stop --verbose || return $? + ( + inotifywait --quiet --quiet --event create "${response_sock_dir}" + echo "${request}" | ncat --unixsock "${runner_sock}" + ) & + echo "done" | ncat --unixsock "${deployer_sock}" + return "$(ncat --listen --unixsock "${response_sock}" | jq .code)" +) + +@test "run with err_repo_name_missing" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_name_missing}" + [ "${status}" -eq "${err_repo_name_missing}" ] +} + +@test "run with err_repo_name_empty" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"repo_name":""}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_name_empty}" + [ "${status}" -eq "${err_repo_name_empty}" ] +} + +@test "run with err_repo_dir_not_exist" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"repo_name":"not_exist"}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_dir_not_exist}" + [ "${status}" -eq "${err_repo_dir_not_exist}" ] +} + +@test "run with err_repo_hash_missing" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"repo_name":"test"}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_hash_missing}" + [ "${status}" -eq "${err_repo_hash_missing}" ] +} + +@test "run with err_repo_hash_empty" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"repo_name":"test","repo_hash":""}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_hash_empty}" + [ "${status}" -eq "${err_repo_hash_empty}" ] +} + +@test "run with err_repo_tag_empty" { + request="$(jq --null-input --compact-output \ + --arg s "${response_sock}" \ + '{"response_sock":$s,"repo_name":"test","repo_hash":"hash","repo_tag":""}')" + run send_request "${request}" + echo "status: got ${status}, expected ${err_repo_tag_empty}" + [ "${status}" -eq "${err_repo_tag_empty}" ] +} diff --git a/tests/tests.conf b/tests/tests.conf new file mode 100644 index 0000000..57c810b --- /dev/null +++ b/tests/tests.conf @@ -0,0 +1,60 @@ +temp="${PWD}/.temp" +name="$(make --silent name)" + +bin_dir="$(rpm --eval '%{_bindir}')" +run_dir="$(rpm --eval '%{_rundir}')" +sysconf_dir="$(rpm --eval '%{_sysconfdir}')" +tmp_dir="$(rpm --eval '%{_tmppath}')" +var_dir="$(rpm --eval '%{_var}')" + +deployer_bin="${temp}${bin_dir}/${name}-deployer" +pipeline_bin="${temp}${bin_dir}/${name}-pipeline" +runner_bin="${temp}${bin_dir}/${name}-runner" +conf_file="${temp}${sysconf_dir}/${name}/${name}.conf" +errs_file="${temp}${sysconf_dir}/${name}/errors.conf" + +# Name of group unifying git and runner. +git_runner_groupname="$(id --user --name)" + +# Name of group unifying runner and deployer. +runner_deployer_groupname="$(id --user --name)" + +# Name of the user responsible of the deployer server. +deployer_username="$(id --user --name)" + +# Location of unixsock file used to send requests to the deployer server. +deployer_sock="${temp}${run_dir}/${name}/deployer/deployer.sock" + +# Maximum number of seconds to wait for deployer response. +# Set to 0 to disable timeout. +deployer_timeout=30 + +# Name of the user responsible of the runner server. +runner_username="$(id --user --name)" + +# Location of unixsock file used to send requests to the runner server. +runner_sock="${temp}${run_dir}/${name}/runner/runner.sock" + +# Directory in which to clone git repositories. +runner_cloning_dir="${temp}${tmp_dir}/${name}/repositories" + +# Maximum number of seconds to wait for runner response. +# Set to 0 to disable timeout. +runner_timeout=120 + +# Directory in which to create pipeline sock files. +pipeline_sock_dir="${temp}${run_dir}/${name}/pipeline" + +# Directory containing all the git repositories. +repos_dir="${temp}${var_dir}/${name}/repositories" + +# Directory containing rpm packages. +rpms_dir="${temp}${var_dir}/${name}/RPMS" + +# Some other vars. +pipeline_sock="${temp}${run_dir}/${name}/pipeline/test/test.sock" +deployer_sock_dir="$(dirname "${deployer_sock}")" +runner_sock_dir="$(dirname "${runner_sock}")" +pipeline_sock_dir="$(dirname "${pipeline_sock}")" +response_sock="${temp}${run_dir}/${name}/response.sock" +response_sock_dir="$(dirname "${response_sock}")"